@bli-cockpit/cli 0.2.70 → 0.2.72
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/doctor-disk-words.js +59 -0
- package/dist/commands/doctor-pipeline.js +15 -21
- package/dist/commands/heartbeat.js +35 -0
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/status.js +8 -1
- package/dist/commands/sync-followups.js +64 -1
- package/dist/commands/sync.js +13 -2
- package/dist/disk-retention.js +5 -1
- package/dist/disk-usage.js +62 -9
- package/dist/evidence-redelivery-plan.js +277 -0
- package/dist/evidence-redelivery.js +280 -0
- package/dist/local-state-status.js +34 -0
- package/package.json +4 -4
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
export function days(ms) {
|
|
2
|
+
return (ms / 86_400_000).toFixed(1);
|
|
3
|
+
}
|
|
4
|
+
/**
|
|
5
|
+
* What the drain did, in the sentence a person reads (BLI-3797).
|
|
6
|
+
*
|
|
7
|
+
* Until this ticket the row said "run `cockpit sync` to deliver them" about
|
|
8
|
+
* objects no sync would ever offer again: the live sync only sends what it
|
|
9
|
+
* collected THIS tick, so 1.8 GB of orphaned evidence sat behind a fix sentence
|
|
10
|
+
* that could not come true. The drain is real now, `--fix` spends it here, and
|
|
11
|
+
* the row reports what it actually managed rather than issuing an instruction.
|
|
12
|
+
*/
|
|
13
|
+
export function redeliveryLine(result) {
|
|
14
|
+
if (result.reason === "no_device_session") {
|
|
15
|
+
return "they cannot be re-offered until this machine signs in — run `cockpit login`";
|
|
16
|
+
}
|
|
17
|
+
if (result.reason === "disabled") {
|
|
18
|
+
return "re-delivery is switched off on this machine (COCKPIT_DISABLE_EVIDENCE_REDELIVERY=1)";
|
|
19
|
+
}
|
|
20
|
+
if (result.uploaded > 0 || result.reused > 0) {
|
|
21
|
+
return `re-delivered ${result.uploaded + result.reused} of them (${mib(result.uploaded_bytes)} MB); ${result.deferred} left for the next tick`;
|
|
22
|
+
}
|
|
23
|
+
if (result.failed > 0) {
|
|
24
|
+
return `re-offering them was refused: ${result.failure_reasons.join(", ") || "unknown"} — the next tick retries under backoff`;
|
|
25
|
+
}
|
|
26
|
+
if (result.held > 0) {
|
|
27
|
+
return `${result.held} are held by delivery backoff and retry on a later tick`;
|
|
28
|
+
}
|
|
29
|
+
return "the next `cockpit sync` re-offers them";
|
|
30
|
+
}
|
|
31
|
+
export function diskRowMessage(footprint, capBytes) {
|
|
32
|
+
const staging = footprint.staging;
|
|
33
|
+
const parts = [
|
|
34
|
+
`staging ${mib(staging.total_bytes)} MB (${mib(staging.committed_bytes)} committed / ${mib(staging.uncommitted_bytes)} uncommitted / ${mib(staging.unknown_bytes)} unknown) against a ${mib(capBytes)} MB cap`,
|
|
35
|
+
];
|
|
36
|
+
// BLI-3797: "1,916.9 uncommitted" is a number, not a finding. The row now
|
|
37
|
+
// names WHY the biggest share of it has not landed and HOW LONG those bytes
|
|
38
|
+
// have been here — on the reference Mac the answer was
|
|
39
|
+
// `reconciled_unknown_to_server` on 234 objects whose oldest was 45 days old,
|
|
40
|
+
// which is a different problem from a delivery that failed this morning.
|
|
41
|
+
const dominant = staging.dominant_uncommitted_reason;
|
|
42
|
+
if (dominant) {
|
|
43
|
+
parts.push(`oldest undelivered ${days(dominant.oldest_disk_age_ms)}d; biggest reason ${dominant.reason} on ${dominant.count} object(s), ${mib(dominant.bytes)} MB`);
|
|
44
|
+
}
|
|
45
|
+
// BLI-3619's second half: "unknown" means the local ledger's own capped
|
|
46
|
+
// memory cannot say, never that delivery failed — and the one command that
|
|
47
|
+
// actually answers it is named right here, not left for a person to find.
|
|
48
|
+
if (staging.unknown_count > 0) {
|
|
49
|
+
parts.push(`${staging.unknown_count} object(s) unknown to this laptop's own ledger — run \`cockpit clean --reconcile\` to ask the server`);
|
|
50
|
+
}
|
|
51
|
+
parts.push(`logs ${mib(footprint.logs.total_bytes)} MB`, `spool ${mib(footprint.spool_bytes)} MB`);
|
|
52
|
+
for (const vault of footprint.vaults) {
|
|
53
|
+
parts.push(`${vault.name} ${mib(vault.byte_size)} MB (a one-off; \`cockpit clean --all-committed\` removes it only if every file in it is accepted)`);
|
|
54
|
+
}
|
|
55
|
+
return parts.join("; ");
|
|
56
|
+
}
|
|
57
|
+
export function mib(bytes) {
|
|
58
|
+
return (bytes / (1024 * 1024)).toFixed(1);
|
|
59
|
+
}
|
|
@@ -7,9 +7,11 @@ import { describeError } from "../health-detail.js";
|
|
|
7
7
|
import { runStagingPrune } from "../disk-prune.js";
|
|
8
8
|
import { retentionOptionsFromEnv } from "../disk-retention.js";
|
|
9
9
|
import { readDiskFootprint } from "../disk-usage.js";
|
|
10
|
+
import { runEvidenceRedelivery } from "../evidence-redelivery.js";
|
|
10
11
|
import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths } from "../local-state.js";
|
|
11
12
|
import { runRawEvidenceLocalGc, rawEvidenceGcSummary } from "../raw-evidence-gc.js";
|
|
12
13
|
import { runBackfillCommand } from "./backfill.js";
|
|
14
|
+
import { diskRowMessage, mib, redeliveryLine } from "./doctor-disk-words.js";
|
|
13
15
|
import { doctorRoots } from "./doctor-access.js";
|
|
14
16
|
import { asRecord, fail, needsFix, ok, skipped } from "./doctor-report.js";
|
|
15
17
|
/**
|
|
@@ -126,6 +128,18 @@ export async function checkDiskState(context) {
|
|
|
126
128
|
}
|
|
127
129
|
export async function fixDiskState(context) {
|
|
128
130
|
const paths = getCollectorRuntimePaths(context.command.homeDir);
|
|
131
|
+
// BLI-3797, BEFORE the prune: the prune can only delete what the ledger
|
|
132
|
+
// vouches for, so on a machine held over the cap by undelivered evidence it
|
|
133
|
+
// frees nothing at all. Deliver first, then sweep what delivering made
|
|
134
|
+
// durable. This never throws and never decides the row's verdict — a drain
|
|
135
|
+
// that could not run leaves the prune to do exactly what it did before.
|
|
136
|
+
const redelivery = await runEvidenceRedelivery({
|
|
137
|
+
homeDir: context.command.homeDir,
|
|
138
|
+
dashboardUrl: context.command.dashboardUrl,
|
|
139
|
+
env: context.io.env,
|
|
140
|
+
fetch: context.io.fetch,
|
|
141
|
+
});
|
|
142
|
+
const redelivered = redeliveryLine(redelivery);
|
|
129
143
|
const pruned = await runStagingPrune(paths, {
|
|
130
144
|
env: context.io.env,
|
|
131
145
|
force: true,
|
|
@@ -144,27 +158,7 @@ export async function fixDiskState(context) {
|
|
|
144
158
|
// Deliberately still `ok`: staging over the cap because evidence has not been
|
|
145
159
|
// accepted yet is the collector working, not a machine to repair. The row
|
|
146
160
|
// names the blockage and the one command that goes further.
|
|
147
|
-
return ok("disk-bounded", "staging_cap_blocked_by_uncommitted", `freed ${mib(pruned.deleted_bytes)} MB; ${message}; ${pruned.cap_blocked_count} object(s) the upload ledger cannot vouch for are holding the rest —
|
|
148
|
-
}
|
|
149
|
-
function diskRowMessage(footprint, capBytes) {
|
|
150
|
-
const staging = footprint.staging;
|
|
151
|
-
const parts = [
|
|
152
|
-
`staging ${mib(staging.total_bytes)} MB (${mib(staging.committed_bytes)} committed / ${mib(staging.uncommitted_bytes)} uncommitted / ${mib(staging.unknown_bytes)} unknown) against a ${mib(capBytes)} MB cap`,
|
|
153
|
-
];
|
|
154
|
-
// BLI-3619's second half: "unknown" means the local ledger's own capped
|
|
155
|
-
// memory cannot say, never that delivery failed — and the one command that
|
|
156
|
-
// actually answers it is named right here, not left for a person to find.
|
|
157
|
-
if (staging.unknown_count > 0) {
|
|
158
|
-
parts.push(`${staging.unknown_count} object(s) unknown to this laptop's own ledger — run \`cockpit clean --reconcile\` to ask the server`);
|
|
159
|
-
}
|
|
160
|
-
parts.push(`logs ${mib(footprint.logs.total_bytes)} MB`, `spool ${mib(footprint.spool_bytes)} MB`);
|
|
161
|
-
for (const vault of footprint.vaults) {
|
|
162
|
-
parts.push(`${vault.name} ${mib(vault.byte_size)} MB (a one-off; \`cockpit clean --all-committed\` removes it only if every file in it is accepted)`);
|
|
163
|
-
}
|
|
164
|
-
return parts.join("; ");
|
|
165
|
-
}
|
|
166
|
-
function mib(bytes) {
|
|
167
|
-
return (bytes / (1024 * 1024)).toFixed(1);
|
|
161
|
+
return ok("disk-bounded", "staging_cap_blocked_by_uncommitted", `freed ${mib(pruned.deleted_bytes)} MB; ${message}; ${pruned.cap_blocked_count} object(s) the upload ledger cannot vouch for are holding the rest — ${redelivered} — or \`cockpit clean --all-committed\` to drop every accepted copy now`);
|
|
168
162
|
}
|
|
169
163
|
function capturedIo(io, forward) {
|
|
170
164
|
const stdoutChunks = [];
|
|
@@ -25,6 +25,7 @@ import os from "node:os";
|
|
|
25
25
|
import path from "node:path";
|
|
26
26
|
import { COLLECTOR_HEARTBEAT_SCHEMA_VERSION, MemoryInstallReceiptSchema, memoryInstallGaps, setupReceiptGaps, } from "@bli-cockpit/telemetry-core";
|
|
27
27
|
import { readCachedSetupReceipt } from "./setup-receipt.js";
|
|
28
|
+
import { readStagingInventory } from "../disk-usage.js";
|
|
28
29
|
import { describeError } from "../health-detail.js";
|
|
29
30
|
import { shouldSuppressFleetReceipts, } from "../dev-build.js";
|
|
30
31
|
import { getCollectorRuntimePaths, readLocalCollectorSessionFile, LOCAL_COLLECTOR_VERSION, } from "../local-state.js";
|
|
@@ -111,10 +112,44 @@ export function buildCollectorHeartbeat(options) {
|
|
|
111
112
|
...(typeof options.facts.sessionsPendingUpload === "number"
|
|
112
113
|
? { sessions_pending_upload: options.facts.sessionsPendingUpload }
|
|
113
114
|
: {}),
|
|
115
|
+
...(typeof options.facts.stagingUncommittedBytes === "number"
|
|
116
|
+
? { staging_uncommitted_bytes: options.facts.stagingUncommittedBytes }
|
|
117
|
+
: {}),
|
|
118
|
+
...(options.facts.stagingUncommittedReason
|
|
119
|
+
? { staging_uncommitted_reason: options.facts.stagingUncommittedReason }
|
|
120
|
+
: {}),
|
|
114
121
|
...(options.memoryInstall ? { memory_install: options.memoryInstall } : {}),
|
|
115
122
|
...(options.setupReceipt ? { setup_receipt: options.setupReceipt } : {}),
|
|
116
123
|
};
|
|
117
124
|
}
|
|
125
|
+
/**
|
|
126
|
+
* What this machine is still holding that never reached storage (BLI-3797).
|
|
127
|
+
*
|
|
128
|
+
* One extra staging walk per tick, on purpose: `cockpit ops` had no way to see
|
|
129
|
+
* that a green, syncing, up-to-date laptop was sitting on 1.8 GB of evidence the
|
|
130
|
+
* server had never received, and a fleet-wide failure nobody can see is the
|
|
131
|
+
* BLI-2528 shape exactly. The walk is bounded by the pack tree the prune keeps
|
|
132
|
+
* under a 2 GB cap, and it is best-effort: a read that fails returns nulls and
|
|
133
|
+
* says so, so the heartbeat omits both fields rather than claiming zero.
|
|
134
|
+
*/
|
|
135
|
+
export async function readHeartbeatStagingFacts(options) {
|
|
136
|
+
const paths = getCollectorRuntimePaths(options.homeDir);
|
|
137
|
+
try {
|
|
138
|
+
const inventory = await readStagingInventory(paths, options.now ?? new Date());
|
|
139
|
+
const dominant = inventory.dominant_uncommitted_reason;
|
|
140
|
+
return {
|
|
141
|
+
bytes: inventory.uncommitted_bytes,
|
|
142
|
+
reason: dominant ? `${dominant.reason}:${dominant.count}`.slice(0, 120) : null,
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
catch (error) {
|
|
146
|
+
console.error("[heartbeat] the staging tree could not be read; this tick reports no backlog figure", JSON.stringify({
|
|
147
|
+
reason: "staging_inventory_unreadable",
|
|
148
|
+
...describeError(error),
|
|
149
|
+
}));
|
|
150
|
+
return { bytes: null, reason: null };
|
|
151
|
+
}
|
|
152
|
+
}
|
|
118
153
|
/**
|
|
119
154
|
* The cached memory receipt for this tick, or null with a named reason
|
|
120
155
|
* (BLI-3729).
|
|
@@ -15,7 +15,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
15
15
|
}
|
|
16
16
|
|
|
17
17
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
18
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
18
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.72");
|
|
19
19
|
return 0;
|
|
20
20
|
}
|
|
21
21
|
|
package/dist/commands/status.js
CHANGED
|
@@ -72,7 +72,14 @@ export async function runStatus(command, io) {
|
|
|
72
72
|
writeConnectedBlock(io, setupReceipt);
|
|
73
73
|
return 0;
|
|
74
74
|
}
|
|
75
|
-
|
|
75
|
+
// The staging walk is opt-in (BLI-3797) and this is the surface that pays for
|
|
76
|
+
// it: one person asking one question. The multi-repo branch above deliberately
|
|
77
|
+
// does not — the backlog is a MACHINE fact, so N repos would walk one tree N
|
|
78
|
+
// times to print the same number.
|
|
79
|
+
const status = await inspectLocalCollectorStatus({
|
|
80
|
+
...command,
|
|
81
|
+
includeStagingBacklog: true,
|
|
82
|
+
});
|
|
76
83
|
if (command.json) {
|
|
77
84
|
writeLine(io.stdout, JSON.stringify({
|
|
78
85
|
...status,
|
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
* What the sync tick does AFTER collection's own outcome is decided and
|
|
3
3
|
* reported: keep this machine's CLI current on npm `latest` (BLI-2601), put a
|
|
4
4
|
* broken scheduler registration back (BLI-2721), keep BLI Memory registered
|
|
5
|
-
* with both agent hosts (BLI-3580),
|
|
5
|
+
* with both agent hosts (BLI-3580), re-offer staged evidence that never landed
|
|
6
|
+
* (BLI-3797), and stop the laptop filling up (BLI-3619).
|
|
6
7
|
*
|
|
7
8
|
* Split out of commands/sync.ts (BLI-3578), moved verbatim. They belong
|
|
8
9
|
* together because they share one rule, and it is the reason both are called
|
|
@@ -20,6 +21,7 @@ import { getCollectorRuntimePaths, LOCAL_COLLECTOR_VERSION, } from "../local-sta
|
|
|
20
21
|
import { runAutostartSelfHeal, } from "../autostart-self-heal.js";
|
|
21
22
|
import { runStagingPrune } from "../disk-prune.js";
|
|
22
23
|
import { runEvidenceReconcile, } from "../evidence-reconcile-client.js";
|
|
24
|
+
import { runEvidenceRedelivery, } from "../evidence-redelivery.js";
|
|
23
25
|
import { envWithNodeRuntimeOnPath, runScheduledSelfUpdate, } from "../scheduled-self-update.js";
|
|
24
26
|
/** How many reconcile batches (of up to 500 hashes each) one sync tick may spend. */
|
|
25
27
|
const RECONCILE_BATCHES_PER_TICK = 1;
|
|
@@ -215,12 +217,20 @@ function memoryInstallEvent(outcome) {
|
|
|
215
217
|
* costs one cheap local read and no network call at all, so it never competes
|
|
216
218
|
* with collection for the tick's time. A reconcile failure never blocks the
|
|
217
219
|
* prune that follows it.
|
|
220
|
+
*
|
|
221
|
+
* BLI-3797 sits between them: the reconcile has just established which staged
|
|
222
|
+
* objects the SERVER says it never received, so the redelivery drain re-offers
|
|
223
|
+
* exactly those, bounded, before the prune runs and can delete whatever landed.
|
|
224
|
+
* Neither of the two can block the prune, and none of the three can fail a tick.
|
|
218
225
|
*/
|
|
219
226
|
export async function runStagingPruneAfterSync(command, io, dashboardUrl, options = {}) {
|
|
220
227
|
const events = [];
|
|
221
228
|
const reconciled = await runReconcileFollowUp(command, io, dashboardUrl, options);
|
|
222
229
|
if (reconciled)
|
|
223
230
|
events.push(reconcileEvent(reconciled));
|
|
231
|
+
const redelivered = await runRedeliveryFollowUp(command, io, dashboardUrl, options);
|
|
232
|
+
if (redelivered)
|
|
233
|
+
events.push(redeliveryEvent(redelivered));
|
|
224
234
|
let result;
|
|
225
235
|
try {
|
|
226
236
|
result = await runStagingPrune(getCollectorRuntimePaths(command.homeDir), {
|
|
@@ -269,6 +279,59 @@ async function runReconcileFollowUp(command, io, dashboardUrl, options) {
|
|
|
269
279
|
});
|
|
270
280
|
return result.reason === "nothing_unknown" ? null : result;
|
|
271
281
|
}
|
|
282
|
+
/**
|
|
283
|
+
* BLI-3797, between the reconcile above and the prune below, and in that order
|
|
284
|
+
* for a reason: reconcile turns `unknown` into a server-backed answer, this
|
|
285
|
+
* re-offers what that answer says never landed, and the prune then deletes
|
|
286
|
+
* whatever this just made durable. Running the drain first would ask the server
|
|
287
|
+
* about hashes it is about to be told the truth about; running it after the
|
|
288
|
+
* prune would leave a tick's worth of freed cap unused.
|
|
289
|
+
*
|
|
290
|
+
* Never throws (`runEvidenceRedelivery` already never does); returns `null` for
|
|
291
|
+
* the boring, common case — nothing on this disk is undelivered — so the steady
|
|
292
|
+
* state costs no receipt, the same rule the reconcile follow-up above and the
|
|
293
|
+
* prune's daily throttle below both follow.
|
|
294
|
+
*/
|
|
295
|
+
async function runRedeliveryFollowUp(command, io, dashboardUrl, options) {
|
|
296
|
+
const result = await runEvidenceRedelivery({
|
|
297
|
+
homeDir: command.homeDir,
|
|
298
|
+
dashboardUrl,
|
|
299
|
+
env: io.env,
|
|
300
|
+
fetch: io.fetch,
|
|
301
|
+
...(options.now ? { now: options.now } : {}),
|
|
302
|
+
});
|
|
303
|
+
return result.reason === "nothing_uncommitted" ? null : result;
|
|
304
|
+
}
|
|
305
|
+
/** Counts and byte totals only; no pack id and no hash travels in a receipt. */
|
|
306
|
+
function redeliveryEvent(result) {
|
|
307
|
+
const detail = [
|
|
308
|
+
`offered ${result.offered} object(s), ${result.offered_bytes}B, across ${result.packs} pack(s)`,
|
|
309
|
+
`uploaded ${result.uploaded} (${result.uploaded_bytes}B), reused ${result.reused}, failed ${result.failed}`,
|
|
310
|
+
`held ${result.held}, deferred ${result.deferred} (${result.deferred_bytes}B)`,
|
|
311
|
+
result.failure_reasons.length > 0
|
|
312
|
+
? `failure_reasons ${result.failure_reasons.join(",")}`
|
|
313
|
+
: null,
|
|
314
|
+
]
|
|
315
|
+
.filter((part) => Boolean(part))
|
|
316
|
+
.join("; ");
|
|
317
|
+
if (result.status === "fail") {
|
|
318
|
+
return {
|
|
319
|
+
step: "evidence_redelivery",
|
|
320
|
+
status: "fail",
|
|
321
|
+
error_code: result.reason,
|
|
322
|
+
error_detail: detail,
|
|
323
|
+
};
|
|
324
|
+
}
|
|
325
|
+
if (result.status === "skipped") {
|
|
326
|
+
return {
|
|
327
|
+
step: "evidence_redelivery",
|
|
328
|
+
status: "skipped",
|
|
329
|
+
error_code: result.reason,
|
|
330
|
+
error_detail: detail,
|
|
331
|
+
};
|
|
332
|
+
}
|
|
333
|
+
return { step: "evidence_redelivery", status: "ok", error_detail: detail };
|
|
334
|
+
}
|
|
272
335
|
function prunedNothing() {
|
|
273
336
|
return {
|
|
274
337
|
status: "skipped",
|
package/dist/commands/sync.js
CHANGED
|
@@ -2,7 +2,7 @@ import { writeLine } from "./cli-io.js";
|
|
|
2
2
|
import { attributedSyncRunStatus, cursorStatusLine, displayTicketId, rawEvidenceSyncLine, shortSha, worktreeSyncRow, writeAgentSessionSummary, } from "./collection-report.js";
|
|
3
3
|
import { collectionRootConsentAliases } from "./collection-roots.js";
|
|
4
4
|
import { discoverCommandWorktrees } from "./local-discovery.js";
|
|
5
|
-
import { sendCollectorHeartbeatBestEffort, } from "./heartbeat.js";
|
|
5
|
+
import { sendCollectorHeartbeatBestEffort, readHeartbeatStagingFacts, } from "./heartbeat.js";
|
|
6
6
|
import { classifySyncFailureRecords, classifySyncHealthError, redactedSyncErrorDetail, reportInstallEventsBestEffort, } from "./install-receipts.js";
|
|
7
7
|
import { runAttributedWorktreeSync, } from "./session-sync.js";
|
|
8
8
|
import { runAutostartSelfHealAfterSync, runMemoryInstallAfterSync, runScheduledSelfUpdateAfterSync, runStagingPruneAfterSync, } from "./sync-followups.js";
|
|
@@ -98,11 +98,22 @@ export async function runSync(command, io) {
|
|
|
98
98
|
*/
|
|
99
99
|
async function sendSyncHeartbeat(command, io, dashboardUrl, facts) {
|
|
100
100
|
const roots = await resolveSyncCollectionRoots(command).catch(() => []);
|
|
101
|
+
// BLI-3797: the backlog figure rides the same check-in as the root labels, so
|
|
102
|
+
// `cockpit ops` learns about undelivered evidence on the SAME tick that proves
|
|
103
|
+
// the machine is alive. Best-effort: nulls omit the fields rather than
|
|
104
|
+
// reporting a zero nobody measured.
|
|
105
|
+
const staging = await readHeartbeatStagingFacts({
|
|
106
|
+
homeDir: command.homeDir,
|
|
107
|
+
}).catch(() => ({ bytes: null, reason: null }));
|
|
101
108
|
await sendCollectorHeartbeatBestEffort({
|
|
102
109
|
homeDir: command.homeDir,
|
|
103
110
|
dashboardUrl,
|
|
104
111
|
roots,
|
|
105
|
-
facts
|
|
112
|
+
facts: {
|
|
113
|
+
...facts,
|
|
114
|
+
stagingUncommittedBytes: staging.bytes,
|
|
115
|
+
stagingUncommittedReason: staging.reason,
|
|
116
|
+
},
|
|
106
117
|
io,
|
|
107
118
|
}).catch((error) => {
|
|
108
119
|
// The sender already swallows everything it knows about; this is the net
|
package/dist/disk-retention.js
CHANGED
|
@@ -131,7 +131,11 @@ function pushDeletion(plan, object, reason) {
|
|
|
131
131
|
function addToBucket(bucket, object) {
|
|
132
132
|
bucket.count += 1;
|
|
133
133
|
bucket.bytes += object.byte_size;
|
|
134
|
-
|
|
134
|
+
// Report the age of the BYTES, not of the last answer about them (BLI-3797).
|
|
135
|
+
// `age_ms` on a reconciled object is the age of the reconcile reply, which
|
|
136
|
+
// made `oldest_uncommitted_age_ms` read 0.94 days for a 45-day-old backlog.
|
|
137
|
+
// Deletion still keys off `age_ms` above — this bucket is report-only.
|
|
138
|
+
bucket.oldest_age_ms = Math.max(bucket.oldest_age_ms, object.disk_age_ms || object.age_ms);
|
|
135
139
|
}
|
|
136
140
|
function removeFromBucket(bucket, object) {
|
|
137
141
|
bucket.count -= 1;
|
package/dist/disk-usage.js
CHANGED
|
@@ -104,6 +104,7 @@ export async function readStagingInventory(paths, now = new Date()) {
|
|
|
104
104
|
inventory.packs.push(pack);
|
|
105
105
|
accumulatePack(inventory, pack);
|
|
106
106
|
}
|
|
107
|
+
rankUncommittedReasons(inventory);
|
|
107
108
|
return inventory;
|
|
108
109
|
}
|
|
109
110
|
/**
|
|
@@ -150,15 +151,23 @@ async function readPack(dir, packId, options) {
|
|
|
150
151
|
manifest_present: manifest !== null,
|
|
151
152
|
};
|
|
152
153
|
for (const relative of await listPackFiles(dir)) {
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
154
|
+
// One stat, two facts: how big the staged copy is and WHEN it was written.
|
|
155
|
+
// The mtime is the only unfalsifiable record of how long undelivered
|
|
156
|
+
// evidence has been sitting here (BLI-3797) — every other timestamp on the
|
|
157
|
+
// machine records when something last had an opinion about it.
|
|
158
|
+
const info = await fs.stat(path.join(dir, relative)).catch(() => null);
|
|
159
|
+
const size = info?.isFile() ? info.size : 0;
|
|
157
160
|
if (relative === "manifest.json") {
|
|
158
161
|
pack.manifest_bytes += size;
|
|
159
162
|
continue;
|
|
160
163
|
}
|
|
161
|
-
|
|
164
|
+
const stagedOnDiskAt = info && Number.isFinite(info.mtimeMs)
|
|
165
|
+
? new Date(info.mtimeMs).toISOString()
|
|
166
|
+
: null;
|
|
167
|
+
pack.objects.push(await classifyObject(dir, packId, relative, size, manifest, {
|
|
168
|
+
...options,
|
|
169
|
+
stagedOnDiskAt,
|
|
170
|
+
}));
|
|
162
171
|
}
|
|
163
172
|
return pack;
|
|
164
173
|
}
|
|
@@ -167,7 +176,13 @@ async function classifyObject(dir, packId, relative, size, manifest, options) {
|
|
|
167
176
|
(await hashUnnamedFile(path.join(dir, relative), size, options.budget));
|
|
168
177
|
const stagedAt = options.stagedAt.get(`${packId}/${relative}`) ?? null;
|
|
169
178
|
const committedAt = hash ? options.ledger.committed.get(hash) : undefined;
|
|
170
|
-
const base = {
|
|
179
|
+
const base = {
|
|
180
|
+
pack_id: packId,
|
|
181
|
+
relative_path: relative,
|
|
182
|
+
byte_size: size,
|
|
183
|
+
staged_on_disk_at: options.stagedOnDiskAt,
|
|
184
|
+
disk_age_ms: ageMs(options.stagedOnDiskAt, options.now),
|
|
185
|
+
};
|
|
171
186
|
if (committedAt) {
|
|
172
187
|
return {
|
|
173
188
|
...base,
|
|
@@ -359,13 +374,49 @@ function accumulatePack(inventory, pack) {
|
|
|
359
374
|
}
|
|
360
375
|
inventory.uncommitted_count += 1;
|
|
361
376
|
inventory.uncommitted_bytes += object.byte_size;
|
|
362
|
-
|
|
377
|
+
tallyUncommittedReason(inventory, object);
|
|
378
|
+
// The age of the BYTES, not of the answer about them (BLI-3797). Falls back
|
|
379
|
+
// to `decided_at` only when the file has no readable mtime, which is the
|
|
380
|
+
// one case where the disk itself cannot say.
|
|
381
|
+
const stagedAt = object.staged_on_disk_at ?? object.decided_at;
|
|
382
|
+
if (stagedAt &&
|
|
363
383
|
(!inventory.oldest_uncommitted_at ||
|
|
364
|
-
|
|
365
|
-
inventory.oldest_uncommitted_at =
|
|
384
|
+
stagedAt.localeCompare(inventory.oldest_uncommitted_at) < 0)) {
|
|
385
|
+
inventory.oldest_uncommitted_at = stagedAt;
|
|
366
386
|
}
|
|
367
387
|
}
|
|
368
388
|
}
|
|
389
|
+
/**
|
|
390
|
+
* One more object against its reason label. Kept as a mutable array rather than
|
|
391
|
+
* a Map so the inventory stays a plain serializable value — it rides `--json`
|
|
392
|
+
* output and a receipt, and a Map would silently become `{}` in both.
|
|
393
|
+
*/
|
|
394
|
+
function tallyUncommittedReason(inventory, object) {
|
|
395
|
+
const existing = inventory.uncommitted_reasons.find((tally) => tally.reason === object.reason);
|
|
396
|
+
const tally = existing ?? {
|
|
397
|
+
reason: object.reason,
|
|
398
|
+
count: 0,
|
|
399
|
+
bytes: 0,
|
|
400
|
+
oldest_disk_age_ms: 0,
|
|
401
|
+
};
|
|
402
|
+
tally.count += 1;
|
|
403
|
+
tally.bytes += object.byte_size;
|
|
404
|
+
tally.oldest_disk_age_ms = Math.max(tally.oldest_disk_age_ms, object.disk_age_ms || object.age_ms);
|
|
405
|
+
if (!existing)
|
|
406
|
+
inventory.uncommitted_reasons.push(tally);
|
|
407
|
+
}
|
|
408
|
+
/**
|
|
409
|
+
* Rank the reasons by BYTES, then name the biggest one.
|
|
410
|
+
*
|
|
411
|
+
* Bytes, not object count, because the question the ranking answers is "what is
|
|
412
|
+
* holding this laptop over its cap" — 234 objects at 1,836 MB and 17 at 81 MB
|
|
413
|
+
* are not the same finding, and counting rows would have called them close.
|
|
414
|
+
*/
|
|
415
|
+
function rankUncommittedReasons(inventory) {
|
|
416
|
+
inventory.uncommitted_reasons.sort((left, right) => right.bytes - left.bytes || right.count - left.count);
|
|
417
|
+
inventory.dominant_uncommitted_reason =
|
|
418
|
+
inventory.uncommitted_reasons[0] ?? null;
|
|
419
|
+
}
|
|
369
420
|
function emptyStagingInventory() {
|
|
370
421
|
return {
|
|
371
422
|
pack_count: 0,
|
|
@@ -376,6 +427,8 @@ function emptyStagingInventory() {
|
|
|
376
427
|
uncommitted_count: 0,
|
|
377
428
|
uncommitted_bytes: 0,
|
|
378
429
|
oldest_uncommitted_at: null,
|
|
430
|
+
uncommitted_reasons: [],
|
|
431
|
+
dominant_uncommitted_reason: null,
|
|
379
432
|
unknown_count: 0,
|
|
380
433
|
unknown_bytes: 0,
|
|
381
434
|
manifest_bytes: 0,
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Rebuilding an upload offer for evidence that is already on this disk.
|
|
3
|
+
*
|
|
4
|
+
* BLI-3797. The live sync only ever offers what THIS tick collected: it reads
|
|
5
|
+
* a transcript, hashes it, stages a copy, uploads that copy, and moves on. If
|
|
6
|
+
* the upload never happens — the process is killed between staging and
|
|
7
|
+
* delivery, the envelope is refused, the source file grows so next tick mints a
|
|
8
|
+
* different hash into a different pack — the staged bytes are orphaned. Nothing
|
|
9
|
+
* offers them again, ever. `disk-retention.ts` will not delete them (correctly:
|
|
10
|
+
* the ledger cannot vouch for them), so they sit until the disk fills.
|
|
11
|
+
*
|
|
12
|
+
* Measured on the reference Mac 2026-09-06, not reasoned about: 251 objects and
|
|
13
|
+
* 1,916.9 MB uncommitted against a 2,048 MB cap, 234 of them (1,835.8 MB)
|
|
14
|
+
* carrying `reconciled_unknown_to_server` — the SERVER was asked and answered
|
|
15
|
+
* that it has never held those hashes. `raw-evidence-staging.json` held six
|
|
16
|
+
* delivery-attempt rows in total, so those objects had not failed delivery;
|
|
17
|
+
* they had never been offered. The oldest pack was written 2026-07-23, 45 days
|
|
18
|
+
* earlier, and `cockpit doctor`'s fix sentence said "run `cockpit sync` to
|
|
19
|
+
* deliver them" — a sentence that could not come true.
|
|
20
|
+
*
|
|
21
|
+
* The repair is possible because each pack's own `manifest.json` already
|
|
22
|
+
* carries everything an offer needs: `object_key`, `content_hash_sha256`,
|
|
23
|
+
* `byte_size`, `media_type` and `redacted_summary` per file, and the pack's
|
|
24
|
+
* `work_context_id` / `session_id` / `operator_id` / `repo_basename` / `branch`
|
|
25
|
+
* beside them. This module rebuilds a `RawEvidenceUploadFile` from those
|
|
26
|
+
* recorded facts and NEVER re-derives them: the bytes on disk were sanitized
|
|
27
|
+
* when they were staged, and re-reading the original source would both cost the
|
|
28
|
+
* IO and risk offering content the redactor of the day would have masked
|
|
29
|
+
* differently.
|
|
30
|
+
*
|
|
31
|
+
* Metadata only, as everywhere in this family: pack ids, pack-relative paths,
|
|
32
|
+
* hashes, byte counts, reason labels. Never content, never an operator path.
|
|
33
|
+
*/
|
|
34
|
+
import fs from "node:fs/promises";
|
|
35
|
+
import path from "node:path";
|
|
36
|
+
import { RAW_EVIDENCE_BUCKET, RAW_EVIDENCE_RETENTION_MODE, } from "./adapters/raw-evidence-manifest.js";
|
|
37
|
+
import { RAW_EVIDENCE_DIR } from "./disk-usage.js";
|
|
38
|
+
import { describeError } from "./health-detail.js";
|
|
39
|
+
import { LOCAL_COLLECTOR_VERSION } from "./local-state.js";
|
|
40
|
+
import { deliveryHold, } from "./raw-evidence-staging.js";
|
|
41
|
+
/**
|
|
42
|
+
* How much undelivered backlog one tick may re-offer.
|
|
43
|
+
*
|
|
44
|
+
* A tick is fifteen minutes and the backlog is measured in gigabytes, so this
|
|
45
|
+
* is a drain rate, not a ceiling on what will eventually be sent: 512 MB a tick
|
|
46
|
+
* clears the reference Mac's 1.9 GB in four ticks (one hour) without competing
|
|
47
|
+
* with the tick's own collection, which owns a separate 2 GB budget. The object
|
|
48
|
+
* count bounds the number of `begin` round-trips for the opposite case — a
|
|
49
|
+
* backlog of thousands of small objects.
|
|
50
|
+
*/
|
|
51
|
+
export const REDELIVERY_MAX_BYTES_PER_TICK = 512 * 1024 * 1024;
|
|
52
|
+
export const REDELIVERY_MAX_OBJECTS_PER_TICK = 64;
|
|
53
|
+
/**
|
|
54
|
+
* Which undelivered objects this tick will re-offer, oldest bytes first.
|
|
55
|
+
*
|
|
56
|
+
* Oldest first is deliberate: the objects that have been undelivered longest
|
|
57
|
+
* are the ones most likely to be the only remaining copy of a session whose
|
|
58
|
+
* source transcript has since been rotated away by its agent host, and they are
|
|
59
|
+
* also the ones the disk cap will start refusing to keep. A newest-first drain
|
|
60
|
+
* would leave a 45-day-old orphan permanently at the back of the queue.
|
|
61
|
+
*
|
|
62
|
+
* `unknown` objects are deliberately NOT planned. "The ledger cannot say" is
|
|
63
|
+
* `cockpit clean --reconcile`'s question, it already runs one bounded batch per
|
|
64
|
+
* tick ahead of this, and it turns each answer into `committed` or
|
|
65
|
+
* `uncommitted` — so an unknown object reaches this planner on a later tick,
|
|
66
|
+
* with the server's own answer behind it, rather than being uploaded again on a
|
|
67
|
+
* guess.
|
|
68
|
+
*/
|
|
69
|
+
export async function planEvidenceRedelivery(options) {
|
|
70
|
+
const plan = {
|
|
71
|
+
packs: [],
|
|
72
|
+
object_count: 0,
|
|
73
|
+
bytes: 0,
|
|
74
|
+
held_count: 0,
|
|
75
|
+
held_bytes: 0,
|
|
76
|
+
unplannable: {},
|
|
77
|
+
deferred_count: 0,
|
|
78
|
+
deferred_bytes: 0,
|
|
79
|
+
};
|
|
80
|
+
const maxBytes = options.maxBytes ?? REDELIVERY_MAX_BYTES_PER_TICK;
|
|
81
|
+
const maxObjects = options.maxObjects ?? REDELIVERY_MAX_OBJECTS_PER_TICK;
|
|
82
|
+
const root = path.join(options.stateDir, RAW_EVIDENCE_DIR);
|
|
83
|
+
const candidates = uncommittedOldestFirst(options.inventory);
|
|
84
|
+
const manifests = new Map();
|
|
85
|
+
const packPlans = new Map();
|
|
86
|
+
for (const object of candidates) {
|
|
87
|
+
if (!object.content_hash) {
|
|
88
|
+
countUnplannable(plan, "no_content_hash");
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
if (deliveryHold(options.staging, object.content_hash, options.now)) {
|
|
92
|
+
plan.held_count += 1;
|
|
93
|
+
plan.held_bytes += object.byte_size;
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
if (plan.object_count >= maxObjects ||
|
|
97
|
+
plan.bytes + object.byte_size > maxBytes) {
|
|
98
|
+
plan.deferred_count += 1;
|
|
99
|
+
plan.deferred_bytes += object.byte_size;
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
if (!manifests.has(object.pack_id)) {
|
|
103
|
+
manifests.set(object.pack_id, await readPackManifest(path.join(root, object.pack_id), object.pack_id));
|
|
104
|
+
}
|
|
105
|
+
const manifest = manifests.get(object.pack_id) ?? null;
|
|
106
|
+
if (!manifest) {
|
|
107
|
+
countUnplannable(plan, "manifest_unreadable");
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
const row = manifest.files.get(object.relative_path);
|
|
111
|
+
if (!row) {
|
|
112
|
+
countUnplannable(plan, "no_manifest_row");
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
let packPlan = packPlans.get(object.pack_id);
|
|
116
|
+
if (!packPlan) {
|
|
117
|
+
packPlan = {
|
|
118
|
+
pack_id: object.pack_id,
|
|
119
|
+
provenance: manifest.provenance,
|
|
120
|
+
files: [],
|
|
121
|
+
bytes: 0,
|
|
122
|
+
};
|
|
123
|
+
packPlans.set(object.pack_id, packPlan);
|
|
124
|
+
plan.packs.push(packPlan);
|
|
125
|
+
}
|
|
126
|
+
packPlan.files.push({
|
|
127
|
+
pointer: {
|
|
128
|
+
raw_evidence_pointer_id: row.object_key,
|
|
129
|
+
privacy_classification: "remote_durable_raw_evidence",
|
|
130
|
+
retention_policy: {
|
|
131
|
+
mode: RAW_EVIDENCE_RETENTION_MODE,
|
|
132
|
+
privacy_classification: "remote_durable_raw_evidence",
|
|
133
|
+
},
|
|
134
|
+
storage_scope: "remote_object",
|
|
135
|
+
storage_bucket: RAW_EVIDENCE_BUCKET,
|
|
136
|
+
object_key: row.object_key,
|
|
137
|
+
content_hash_sha256: row.content_hash_sha256,
|
|
138
|
+
byte_size: row.byte_size,
|
|
139
|
+
media_type: row.media_type,
|
|
140
|
+
redacted_summary: row.redacted_summary,
|
|
141
|
+
},
|
|
142
|
+
local_path: path.join(root, object.pack_id, ...object.relative_path.split("/").filter(Boolean)),
|
|
143
|
+
kind: row.kind,
|
|
144
|
+
codex_session_id: null,
|
|
145
|
+
});
|
|
146
|
+
packPlan.bytes += object.byte_size;
|
|
147
|
+
plan.object_count += 1;
|
|
148
|
+
plan.bytes += object.byte_size;
|
|
149
|
+
}
|
|
150
|
+
return plan;
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Every uncommitted object, oldest staged bytes first.
|
|
154
|
+
*
|
|
155
|
+
* Sorted on `disk_age_ms` (the file's own mtime) rather than `age_ms`, because
|
|
156
|
+
* `age_ms` for a reconciled object is the age of the reconcile ANSWER — the
|
|
157
|
+
* same confusion that had 45-day-old orphans reading as one day old.
|
|
158
|
+
*/
|
|
159
|
+
function uncommittedOldestFirst(inventory) {
|
|
160
|
+
return inventory.packs
|
|
161
|
+
.flatMap((pack) => pack.objects)
|
|
162
|
+
.filter((object) => object.state === "uncommitted")
|
|
163
|
+
.sort((left, right) => (right.disk_age_ms || right.age_ms) - (left.disk_age_ms || left.age_ms));
|
|
164
|
+
}
|
|
165
|
+
function countUnplannable(plan, reason) {
|
|
166
|
+
plan.unplannable[reason] = (plan.unplannable[reason] ?? 0) + 1;
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* A pack's recorded provenance and file rows.
|
|
170
|
+
*
|
|
171
|
+
* Every field is read back from what the collector WROTE when it staged the
|
|
172
|
+
* pack; nothing is re-derived from the machine's current state. A pack staged
|
|
173
|
+
* six weeks ago under a work context that no longer exists still knows which
|
|
174
|
+
* repo, branch, operator and work context it belonged to, and that is the only
|
|
175
|
+
* honest label for those bytes.
|
|
176
|
+
*/
|
|
177
|
+
async function readPackManifest(dir, packId) {
|
|
178
|
+
const raw = await fs
|
|
179
|
+
.readFile(path.join(dir, "manifest.json"), "utf8")
|
|
180
|
+
.catch(() => null);
|
|
181
|
+
if (!raw)
|
|
182
|
+
return null;
|
|
183
|
+
try {
|
|
184
|
+
const parsed = JSON.parse(raw);
|
|
185
|
+
const provenance = provenanceFromManifest(parsed, packId);
|
|
186
|
+
if (!provenance)
|
|
187
|
+
return null;
|
|
188
|
+
const files = new Map();
|
|
189
|
+
for (const entry of asArray(parsed["files"])) {
|
|
190
|
+
const row = manifestFileRow(entry);
|
|
191
|
+
if (row)
|
|
192
|
+
files.set(row.relative_path, row.value);
|
|
193
|
+
}
|
|
194
|
+
return { provenance, files };
|
|
195
|
+
}
|
|
196
|
+
catch (error) {
|
|
197
|
+
console.error("[evidence-redelivery] pack manifest unreadable; its objects cannot be re-offered", JSON.stringify({
|
|
198
|
+
reason: "manifest_unreadable",
|
|
199
|
+
pack_id: packId,
|
|
200
|
+
...describeError(error),
|
|
201
|
+
}));
|
|
202
|
+
return null;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
function provenanceFromManifest(manifest, packId) {
|
|
206
|
+
const workContextId = asText(manifest["work_context_id"]);
|
|
207
|
+
const sessionId = asText(manifest["session_id"]);
|
|
208
|
+
const operatorId = asText(manifest["operator_id"]);
|
|
209
|
+
const repo = asText(manifest["repo_basename"]) ?? asText(manifest["repo_label"]);
|
|
210
|
+
if (!workContextId || !sessionId || !operatorId || !repo) {
|
|
211
|
+
console.error("[evidence-redelivery] pack manifest names no work context; its objects cannot be re-offered", JSON.stringify({
|
|
212
|
+
reason: "manifest_provenance_incomplete",
|
|
213
|
+
pack_id: packId,
|
|
214
|
+
has_work_context: Boolean(workContextId),
|
|
215
|
+
has_session: Boolean(sessionId),
|
|
216
|
+
has_operator: Boolean(operatorId),
|
|
217
|
+
has_repo: Boolean(repo),
|
|
218
|
+
}));
|
|
219
|
+
return null;
|
|
220
|
+
}
|
|
221
|
+
const repoLabel = asText(manifest["repo_label"]);
|
|
222
|
+
const worktreeLabel = asText(manifest["worktree_label"]);
|
|
223
|
+
return {
|
|
224
|
+
capture_source: "collector_runtime",
|
|
225
|
+
// The version that is re-offering, not the one that staged: this IS a
|
|
226
|
+
// delivery by today's collector, and pretending otherwise would misreport
|
|
227
|
+
// which build put the object in the bucket.
|
|
228
|
+
capture_adapter_version: LOCAL_COLLECTOR_VERSION,
|
|
229
|
+
collector_version: LOCAL_COLLECTOR_VERSION,
|
|
230
|
+
repo,
|
|
231
|
+
// A pack staged on a detached HEAD or a deleted worktree can have no branch
|
|
232
|
+
// recorded. `unknown` is a label, not a guess at which branch it was.
|
|
233
|
+
branch: asText(manifest["branch"]) ?? "unknown",
|
|
234
|
+
...(repoLabel ? { repo_label: repoLabel } : {}),
|
|
235
|
+
...(worktreeLabel ? { worktree_label: worktreeLabel } : {}),
|
|
236
|
+
operator_id: operatorId,
|
|
237
|
+
session_id: sessionId,
|
|
238
|
+
work_context_id: workContextId,
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
function manifestFileRow(entry) {
|
|
242
|
+
if (!entry || typeof entry !== "object")
|
|
243
|
+
return null;
|
|
244
|
+
const row = entry;
|
|
245
|
+
const relativePath = asText(row["relative_path"]);
|
|
246
|
+
const objectKey = asText(row["object_key"]);
|
|
247
|
+
const hash = asText(row["content_hash_sha256"]);
|
|
248
|
+
const mediaType = asText(row["media_type"]);
|
|
249
|
+
const summary = asText(row["redacted_summary"]);
|
|
250
|
+
const byteSize = row["byte_size"];
|
|
251
|
+
if (!relativePath ||
|
|
252
|
+
!objectKey ||
|
|
253
|
+
!hash ||
|
|
254
|
+
hash.length !== 64 ||
|
|
255
|
+
!mediaType ||
|
|
256
|
+
!summary ||
|
|
257
|
+
typeof byteSize !== "number") {
|
|
258
|
+
return null;
|
|
259
|
+
}
|
|
260
|
+
return {
|
|
261
|
+
relative_path: relativePath,
|
|
262
|
+
value: {
|
|
263
|
+
object_key: objectKey,
|
|
264
|
+
content_hash_sha256: hash,
|
|
265
|
+
byte_size: byteSize,
|
|
266
|
+
media_type: mediaType,
|
|
267
|
+
redacted_summary: summary,
|
|
268
|
+
kind: asText(row["kind"]) ?? "unknown",
|
|
269
|
+
},
|
|
270
|
+
};
|
|
271
|
+
}
|
|
272
|
+
function asArray(value) {
|
|
273
|
+
return Array.isArray(value) ? value : [];
|
|
274
|
+
}
|
|
275
|
+
function asText(value) {
|
|
276
|
+
return typeof value === "string" && value.trim() ? value : null;
|
|
277
|
+
}
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Offering undelivered staged evidence again, on the next tick and every tick
|
|
3
|
+
* after, until it lands or names why it cannot.
|
|
4
|
+
*
|
|
5
|
+
* BLI-3797. `evidence-redelivery-plan.ts` says WHICH bytes and rebuilds the
|
|
6
|
+
* offer; this file spends it against the same chunked upload client the live
|
|
7
|
+
* sync uses, writes each success into the commit ledger and each failure into
|
|
8
|
+
* the delivery-attempt history (so the existing 15-min → 6-h backoff applies
|
|
9
|
+
* and one doomed object is not re-offered 96 times a day), and says what
|
|
10
|
+
* happened on every branch.
|
|
11
|
+
*
|
|
12
|
+
* Four properties, the same ones the reconcile and prune follow-ups keep:
|
|
13
|
+
*
|
|
14
|
+
* - **It never throws and never fails a tick.** A drain that cannot run is a
|
|
15
|
+
* log line and a receipt, never a reason collection did not happen.
|
|
16
|
+
* - **It never deletes.** Nothing here removes a staged byte; the prune that
|
|
17
|
+
* follows it decides that, and it only ever takes what the ledger vouches
|
|
18
|
+
* for — which now includes whatever this just delivered.
|
|
19
|
+
* - **It is bounded.** One drain rate per tick (see the plan module), so the
|
|
20
|
+
* backlog clears in hours without competing with the tick's own collection.
|
|
21
|
+
* - **It says so on both branches.** A tick with nothing undelivered is silent
|
|
22
|
+
* (the steady state, 96 times a day); every other outcome, including a clean
|
|
23
|
+
* drain, gets one `[evidence-redelivery]` line with counts and reasons.
|
|
24
|
+
*/
|
|
25
|
+
import { uploadRawEvidenceFilesChunked, } from "./evidence-upload-client.js";
|
|
26
|
+
import { planEvidenceRedelivery, } from "./evidence-redelivery-plan.js";
|
|
27
|
+
import { markObjectCommitted, readRawEvidenceCursor, writeRawEvidenceCursor, } from "./cursors/raw-evidence-cursor.js";
|
|
28
|
+
import { readStagingInventory } from "./disk-usage.js";
|
|
29
|
+
import { describeError } from "./health-detail.js";
|
|
30
|
+
import { getCollectorRuntimePaths, readLocalCollectorSessionFile, } from "./local-state.js";
|
|
31
|
+
import { clearDeliveryAttempt, readRawEvidenceStagingState, recordDeliveryFailure, writeRawEvidenceStagingState, } from "./raw-evidence-staging.js";
|
|
32
|
+
/**
|
|
33
|
+
* The drain the sync tick runs and `cockpit doctor`'s disk fix reuses.
|
|
34
|
+
* Never throws.
|
|
35
|
+
*/
|
|
36
|
+
export async function runEvidenceRedelivery(options) {
|
|
37
|
+
const env = options.env ?? process.env;
|
|
38
|
+
const now = options.now ?? new Date();
|
|
39
|
+
if (env["COCKPIT_DISABLE_EVIDENCE_REDELIVERY"] === "1") {
|
|
40
|
+
return emptyResult("disabled", "skipped");
|
|
41
|
+
}
|
|
42
|
+
const paths = getCollectorRuntimePaths(options.homeDir);
|
|
43
|
+
try {
|
|
44
|
+
const inventory = options.inventory ?? (await readStagingInventory(paths, now));
|
|
45
|
+
if (inventory.uncommitted_count === 0) {
|
|
46
|
+
// The steady state, 96 ticks a day. Saying so every time would bury the
|
|
47
|
+
// ticks that matter; `cockpit doctor`'s disk row still says it out loud
|
|
48
|
+
// to a person who asks.
|
|
49
|
+
return emptyResult("nothing_uncommitted", "ok");
|
|
50
|
+
}
|
|
51
|
+
const staging = await readRawEvidenceStagingState(paths.state_dir);
|
|
52
|
+
const plan = await planEvidenceRedelivery({
|
|
53
|
+
stateDir: paths.state_dir,
|
|
54
|
+
inventory,
|
|
55
|
+
staging,
|
|
56
|
+
now,
|
|
57
|
+
...(options.maxBytes != null ? { maxBytes: options.maxBytes } : {}),
|
|
58
|
+
...(options.maxObjects != null ? { maxObjects: options.maxObjects } : {}),
|
|
59
|
+
});
|
|
60
|
+
if (plan.object_count === 0) {
|
|
61
|
+
const result = emptyPlanResult(plan);
|
|
62
|
+
reportRedelivery(result, plan);
|
|
63
|
+
return result;
|
|
64
|
+
}
|
|
65
|
+
const session = await readLocalCollectorSessionFile(paths).catch(() => null);
|
|
66
|
+
if (!session ||
|
|
67
|
+
session.session_state !== "valid" ||
|
|
68
|
+
typeof session.device_token !== "string" ||
|
|
69
|
+
!session.device_token) {
|
|
70
|
+
// Not a failure of the backlog: this machine simply cannot speak to the
|
|
71
|
+
// dashboard right now. The bytes stay, the reason is named, and the next
|
|
72
|
+
// tick after a `cockpit login` picks them straight back up.
|
|
73
|
+
const result = {
|
|
74
|
+
...emptyPlanResult(plan),
|
|
75
|
+
status: "skipped",
|
|
76
|
+
reason: "no_device_session",
|
|
77
|
+
};
|
|
78
|
+
reportRedelivery(result, plan);
|
|
79
|
+
return result;
|
|
80
|
+
}
|
|
81
|
+
const outcome = await deliverPlan({
|
|
82
|
+
plan,
|
|
83
|
+
dashboardUrl: options.dashboardUrl,
|
|
84
|
+
deviceToken: session.device_token,
|
|
85
|
+
fetchImpl: options.fetch ?? fetch,
|
|
86
|
+
now,
|
|
87
|
+
});
|
|
88
|
+
await recordOutcomes(paths, staging, outcome.outcomes, now);
|
|
89
|
+
const result = {
|
|
90
|
+
status: outcome.failed > 0 && outcome.uploaded === 0 ? "fail" : "ok",
|
|
91
|
+
reason: "redelivered",
|
|
92
|
+
offered: plan.object_count,
|
|
93
|
+
offered_bytes: plan.bytes,
|
|
94
|
+
uploaded: outcome.uploaded,
|
|
95
|
+
uploaded_bytes: outcome.uploadedBytes,
|
|
96
|
+
reused: outcome.reused,
|
|
97
|
+
failed: outcome.failed,
|
|
98
|
+
held: plan.held_count,
|
|
99
|
+
deferred: plan.deferred_count,
|
|
100
|
+
deferred_bytes: plan.deferred_bytes,
|
|
101
|
+
packs: plan.packs.length,
|
|
102
|
+
failure_reasons: outcome.failureReasons,
|
|
103
|
+
};
|
|
104
|
+
reportRedelivery(result, plan);
|
|
105
|
+
return result;
|
|
106
|
+
}
|
|
107
|
+
catch (error) {
|
|
108
|
+
console.error("[evidence-redelivery] the drain could not run; nothing was re-offered", JSON.stringify({ reason: "redelivery_threw", ...describeError(error) }));
|
|
109
|
+
return { ...emptyResult("redelivery_threw", "fail") };
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* One `uploadRawEvidenceFilesChunked` call per pack, because provenance is a
|
|
114
|
+
* PACK fact: the repo, branch, operator, session and work context these bytes
|
|
115
|
+
* belonged to were recorded when they were staged, and sending one pack's
|
|
116
|
+
* objects under another pack's labels would file six-week-old evidence against
|
|
117
|
+
* today's ticket.
|
|
118
|
+
*/
|
|
119
|
+
async function deliverPlan(options) {
|
|
120
|
+
const result = {
|
|
121
|
+
outcomes: [],
|
|
122
|
+
uploaded: 0,
|
|
123
|
+
uploadedBytes: 0,
|
|
124
|
+
reused: 0,
|
|
125
|
+
failed: 0,
|
|
126
|
+
failureReasons: [],
|
|
127
|
+
};
|
|
128
|
+
for (const pack of options.plan.packs) {
|
|
129
|
+
const uploaded = await deliverOnePack(pack, options);
|
|
130
|
+
for (const outcome of uploaded) {
|
|
131
|
+
result.outcomes.push(outcome);
|
|
132
|
+
if (outcome.upload_state === "uploaded") {
|
|
133
|
+
result.uploaded += 1;
|
|
134
|
+
result.uploadedBytes += outcome.pointer.byte_size ?? 0;
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
if (outcome.upload_state === "reused_existing") {
|
|
138
|
+
result.reused += 1;
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
result.failed += 1;
|
|
142
|
+
const reason = outcome.reason ?? "unknown";
|
|
143
|
+
if (!result.failureReasons.includes(reason)) {
|
|
144
|
+
result.failureReasons.push(reason);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
return result;
|
|
149
|
+
}
|
|
150
|
+
async function deliverOnePack(pack, options) {
|
|
151
|
+
try {
|
|
152
|
+
const uploaded = await uploadRawEvidenceFilesChunked({
|
|
153
|
+
fetchImpl: options.fetchImpl,
|
|
154
|
+
dashboardUrl: options.dashboardUrl,
|
|
155
|
+
deviceToken: options.deviceToken,
|
|
156
|
+
provenance: pack.provenance,
|
|
157
|
+
generatedAt: options.now.toISOString(),
|
|
158
|
+
files: pack.files,
|
|
159
|
+
});
|
|
160
|
+
return uploaded.outcomes;
|
|
161
|
+
}
|
|
162
|
+
catch (error) {
|
|
163
|
+
// The upload client already reports its own transport failures; this is the
|
|
164
|
+
// net for anything it does not, and it must not abandon the packs behind
|
|
165
|
+
// this one. Each file is reported failed so the backoff history records the
|
|
166
|
+
// attempt rather than re-offering all of them again in fifteen minutes.
|
|
167
|
+
console.error("[evidence-redelivery] a pack's re-offer threw; the rest of the drain continues", JSON.stringify({
|
|
168
|
+
reason: "pack_redelivery_threw",
|
|
169
|
+
pack_id: pack.pack_id,
|
|
170
|
+
object_count: pack.files.length,
|
|
171
|
+
...describeError(error),
|
|
172
|
+
}));
|
|
173
|
+
return pack.files.map((file) => ({
|
|
174
|
+
pointer: file.pointer,
|
|
175
|
+
object_key: file.pointer.object_key ?? "",
|
|
176
|
+
codex_session_id: null,
|
|
177
|
+
kind: file.kind ?? "unknown",
|
|
178
|
+
upload_state: "upload_failed",
|
|
179
|
+
reason: "pack_redelivery_threw",
|
|
180
|
+
uploaded_chunk_count: 0,
|
|
181
|
+
}));
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Write what happened where the next tick will read it: the commit ledger for
|
|
186
|
+
* anything that landed, the delivery-attempt history for anything that did not.
|
|
187
|
+
*
|
|
188
|
+
* Both writes happen even when the run partly failed, and the ledger is written
|
|
189
|
+
* FIRST — a delivered object whose ledger row was lost would be re-offered
|
|
190
|
+
* forever, which is the failure this whole module exists to end.
|
|
191
|
+
*/
|
|
192
|
+
async function recordOutcomes(paths, staging, outcomes, now) {
|
|
193
|
+
const cursor = await readRawEvidenceCursor(paths);
|
|
194
|
+
let ledgerChanged = false;
|
|
195
|
+
let stagingChanged = false;
|
|
196
|
+
for (const outcome of outcomes) {
|
|
197
|
+
const hash = outcome.pointer.content_hash_sha256;
|
|
198
|
+
if (!hash)
|
|
199
|
+
continue;
|
|
200
|
+
if (outcome.upload_state === "upload_failed") {
|
|
201
|
+
recordDeliveryFailure(staging, hash, {
|
|
202
|
+
reason: outcome.reason ?? "unknown",
|
|
203
|
+
attemptedAt: now,
|
|
204
|
+
byteSize: outcome.pointer.byte_size ?? 0,
|
|
205
|
+
});
|
|
206
|
+
stagingChanged = true;
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
209
|
+
markObjectCommitted(cursor, hash, {
|
|
210
|
+
object_key: outcome.object_key,
|
|
211
|
+
byte_size: outcome.pointer.byte_size ?? 0,
|
|
212
|
+
committed_at: now.toISOString(),
|
|
213
|
+
});
|
|
214
|
+
ledgerChanged = true;
|
|
215
|
+
if (clearDeliveryAttempt(staging, hash))
|
|
216
|
+
stagingChanged = true;
|
|
217
|
+
}
|
|
218
|
+
if (ledgerChanged)
|
|
219
|
+
await writeRawEvidenceCursor(paths, cursor);
|
|
220
|
+
if (stagingChanged) {
|
|
221
|
+
await writeRawEvidenceStagingState(paths.state_dir, staging);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* One line per non-steady-state tick. Counts, byte totals and reason labels
|
|
226
|
+
* only — no pack id, no hash, no path, per the logging contract.
|
|
227
|
+
*/
|
|
228
|
+
function reportRedelivery(result, plan) {
|
|
229
|
+
const fields = {
|
|
230
|
+
reason: result.reason,
|
|
231
|
+
status: result.status,
|
|
232
|
+
offered: result.offered,
|
|
233
|
+
offered_bytes: result.offered_bytes,
|
|
234
|
+
uploaded: result.uploaded,
|
|
235
|
+
uploaded_bytes: result.uploaded_bytes,
|
|
236
|
+
reused: result.reused,
|
|
237
|
+
failed: result.failed,
|
|
238
|
+
held_by_backoff: result.held,
|
|
239
|
+
deferred_to_next_tick: result.deferred,
|
|
240
|
+
deferred_bytes: result.deferred_bytes,
|
|
241
|
+
packs: result.packs,
|
|
242
|
+
unplannable: plan.unplannable,
|
|
243
|
+
failure_reasons: result.failure_reasons,
|
|
244
|
+
};
|
|
245
|
+
if (result.status === "fail") {
|
|
246
|
+
console.error("[evidence-redelivery] every re-offered object was refused", JSON.stringify(fields));
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
console.error(result.uploaded > 0
|
|
250
|
+
? "[evidence-redelivery] undelivered staged evidence landed"
|
|
251
|
+
: "[evidence-redelivery] undelivered staged evidence was not re-offered this tick", JSON.stringify(fields));
|
|
252
|
+
}
|
|
253
|
+
function emptyPlanResult(plan) {
|
|
254
|
+
const nothingPlannable = plan.held_count > 0 && plan.deferred_count === 0
|
|
255
|
+
? "all_held_by_backoff"
|
|
256
|
+
: "nothing_plannable";
|
|
257
|
+
return {
|
|
258
|
+
...emptyResult(nothingPlannable, "ok"),
|
|
259
|
+
held: plan.held_count,
|
|
260
|
+
deferred: plan.deferred_count,
|
|
261
|
+
deferred_bytes: plan.deferred_bytes,
|
|
262
|
+
};
|
|
263
|
+
}
|
|
264
|
+
function emptyResult(reason, status) {
|
|
265
|
+
return {
|
|
266
|
+
status,
|
|
267
|
+
reason,
|
|
268
|
+
offered: 0,
|
|
269
|
+
offered_bytes: 0,
|
|
270
|
+
uploaded: 0,
|
|
271
|
+
uploaded_bytes: 0,
|
|
272
|
+
reused: 0,
|
|
273
|
+
failed: 0,
|
|
274
|
+
held: 0,
|
|
275
|
+
deferred: 0,
|
|
276
|
+
deferred_bytes: 0,
|
|
277
|
+
packs: 0,
|
|
278
|
+
failure_reasons: [],
|
|
279
|
+
};
|
|
280
|
+
}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import os from "node:os";
|
|
2
2
|
import path from "node:path";
|
|
3
|
+
import { readStagingInventory } from "./disk-usage.js";
|
|
4
|
+
import { describeError } from "./health-detail.js";
|
|
3
5
|
import { summarizeLocalUploadSpool } from "./spool/local-spool.js";
|
|
4
6
|
import { summarizeInstallEventOutbox } from "./spool/install-event-outbox.js";
|
|
5
7
|
import { readRawEvidenceStagingState, summarizeStuckEvidence, } from "./raw-evidence-staging.js";
|
|
@@ -55,6 +57,10 @@ export async function inspectLocalCollectorStatus(options = {}) {
|
|
|
55
57
|
stuck_evidence_max_attempts: stuckEvidence.max_attempts,
|
|
56
58
|
stuck_evidence_oldest_failure_at: stuckEvidence.oldest_first_failed_at,
|
|
57
59
|
stuck_evidence_reasons: stuckEvidence.reasons,
|
|
60
|
+
staging_uncommitted_count: reading.staging?.uncommitted_count ?? null,
|
|
61
|
+
staging_uncommitted_bytes: reading.staging?.uncommitted_bytes ?? null,
|
|
62
|
+
staging_uncommitted_reason: reading.staging?.dominant_uncommitted_reason?.reason ?? null,
|
|
63
|
+
staging_oldest_uncommitted_at: reading.staging?.oldest_uncommitted_at ?? null,
|
|
58
64
|
details: describeCollectorStatus(reading),
|
|
59
65
|
};
|
|
60
66
|
}
|
|
@@ -76,7 +82,20 @@ async function readCollectorStatus(options, now) {
|
|
|
76
82
|
readRawEvidenceStagingState(paths.state_dir),
|
|
77
83
|
]);
|
|
78
84
|
const stuckEvidence = summarizeStuckEvidence(stagingState, now);
|
|
85
|
+
// Best-effort by construction: a status read must never fail because the
|
|
86
|
+
// staging tree could not be walked, and an absent backlog reads as "this read
|
|
87
|
+
// did not ask", never as "there is none".
|
|
88
|
+
const staging = options.includeStagingBacklog
|
|
89
|
+
? await readStagingInventory(paths, now).catch((error) => {
|
|
90
|
+
console.error("[collector status] the staging tree could not be read; the undelivered backlog is unknown", JSON.stringify({
|
|
91
|
+
reason: "staging_inventory_unreadable",
|
|
92
|
+
...describeError(error),
|
|
93
|
+
}));
|
|
94
|
+
return null;
|
|
95
|
+
})
|
|
96
|
+
: null;
|
|
79
97
|
return {
|
|
98
|
+
staging,
|
|
80
99
|
paths,
|
|
81
100
|
config,
|
|
82
101
|
session,
|
|
@@ -134,6 +153,18 @@ function describeCollectorStatus(reading) {
|
|
|
134
153
|
if (healthOutbox.pending_count > 0) {
|
|
135
154
|
details.push(`Collector health retry pending: ${healthOutbox.pending_count} sanitized receipt(s) queued since ${healthOutbox.oldest_created_at ?? "unknown"}.`);
|
|
136
155
|
}
|
|
156
|
+
// BLI-3797: the line above only sees objects whose delivery FAILED. This one
|
|
157
|
+
// sees evidence that is on this disk and not in the bucket, however it got
|
|
158
|
+
// that way — which is the only question that answers "has everything landed?".
|
|
159
|
+
if (reading.staging && reading.staging.uncommitted_count > 0) {
|
|
160
|
+
const dominant = reading.staging.dominant_uncommitted_reason;
|
|
161
|
+
details.push(`Staged evidence not yet accepted: ${reading.staging.uncommitted_count} object(s), ${mib(reading.staging.uncommitted_bytes)} MB, oldest staged ${reading.staging.oldest_uncommitted_at ?? "unknown"}${dominant
|
|
162
|
+
? ` — biggest reason ${dominant.reason} on ${dominant.count} object(s)`
|
|
163
|
+
: ""}. \`cockpit sync\` re-offers them each tick; \`cockpit doctor --fix\` does it now.`);
|
|
164
|
+
}
|
|
165
|
+
else if (reading.staging) {
|
|
166
|
+
details.push("Staged evidence: every object on this disk is accepted.");
|
|
167
|
+
}
|
|
137
168
|
if (stuckEvidence.stuck_object_count > 0) {
|
|
138
169
|
details.push(`Raw evidence stuck: ${stuckEvidence.stuck_object_count} object(s) have never been accepted (${stuckEvidence.held_object_count} waiting on backoff, worst ${stuckEvidence.max_attempts} attempt(s) since ${stuckEvidence.oldest_first_failed_at ?? "unknown"}) — reasons: ${stuckEvidence.reasons.join(", ") || "unknown"}.`);
|
|
139
170
|
}
|
|
@@ -155,6 +186,9 @@ export function classifyCollectorFreshness(context, lastUploadSuccessAt, now) {
|
|
|
155
186
|
return "stale";
|
|
156
187
|
return now.getTime() - latestActivityAt <= 5 * 60 * 1000 ? "fresh" : "stale";
|
|
157
188
|
}
|
|
189
|
+
function mib(bytes) {
|
|
190
|
+
return (bytes / (1024 * 1024)).toFixed(1);
|
|
191
|
+
}
|
|
158
192
|
function latestTimestamp(values) {
|
|
159
193
|
const timestamps = values
|
|
160
194
|
.map((value) => Date.parse(value ?? ""))
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bli-cockpit/cli",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.72",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -27,8 +27,8 @@
|
|
|
27
27
|
"test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
|
-
"@bli-cockpit/memory-mcp": "0.1.
|
|
31
|
-
"@bli-cockpit/mcp": "0.1.
|
|
32
|
-
"@bli-cockpit/telemetry-core": "0.1.
|
|
30
|
+
"@bli-cockpit/memory-mcp": "0.1.11",
|
|
31
|
+
"@bli-cockpit/mcp": "0.1.12",
|
|
32
|
+
"@bli-cockpit/telemetry-core": "0.1.32"
|
|
33
33
|
}
|
|
34
34
|
}
|