@bli-cockpit/cli 0.2.69 → 0.2.72
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/doctor-disk-words.js +59 -0
- package/dist/commands/doctor-pipeline.js +15 -21
- package/dist/commands/heartbeat.js +64 -1
- package/dist/commands/memory-hook-counts.js +61 -0
- package/dist/commands/ops-render-memory.js +34 -0
- package/dist/commands/ops-render.js +1 -1
- package/dist/commands/ops.js +24 -6
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/status.js +8 -1
- package/dist/commands/sync-followups.js +64 -1
- package/dist/commands/sync.js +13 -2
- package/dist/disk-retention.js +5 -1
- package/dist/disk-usage.js +62 -9
- package/dist/evidence-redelivery-plan.js +277 -0
- package/dist/evidence-redelivery.js +280 -0
- package/dist/local-state-status.js +34 -0
- package/package.json +4 -4
package/dist/disk-usage.js
CHANGED
|
@@ -104,6 +104,7 @@ export async function readStagingInventory(paths, now = new Date()) {
|
|
|
104
104
|
inventory.packs.push(pack);
|
|
105
105
|
accumulatePack(inventory, pack);
|
|
106
106
|
}
|
|
107
|
+
rankUncommittedReasons(inventory);
|
|
107
108
|
return inventory;
|
|
108
109
|
}
|
|
109
110
|
/**
|
|
@@ -150,15 +151,23 @@ async function readPack(dir, packId, options) {
|
|
|
150
151
|
manifest_present: manifest !== null,
|
|
151
152
|
};
|
|
152
153
|
for (const relative of await listPackFiles(dir)) {
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
154
|
+
// One stat, two facts: how big the staged copy is and WHEN it was written.
|
|
155
|
+
// The mtime is the only unfalsifiable record of how long undelivered
|
|
156
|
+
// evidence has been sitting here (BLI-3797) — every other timestamp on the
|
|
157
|
+
// machine records when something last had an opinion about it.
|
|
158
|
+
const info = await fs.stat(path.join(dir, relative)).catch(() => null);
|
|
159
|
+
const size = info?.isFile() ? info.size : 0;
|
|
157
160
|
if (relative === "manifest.json") {
|
|
158
161
|
pack.manifest_bytes += size;
|
|
159
162
|
continue;
|
|
160
163
|
}
|
|
161
|
-
|
|
164
|
+
const stagedOnDiskAt = info && Number.isFinite(info.mtimeMs)
|
|
165
|
+
? new Date(info.mtimeMs).toISOString()
|
|
166
|
+
: null;
|
|
167
|
+
pack.objects.push(await classifyObject(dir, packId, relative, size, manifest, {
|
|
168
|
+
...options,
|
|
169
|
+
stagedOnDiskAt,
|
|
170
|
+
}));
|
|
162
171
|
}
|
|
163
172
|
return pack;
|
|
164
173
|
}
|
|
@@ -167,7 +176,13 @@ async function classifyObject(dir, packId, relative, size, manifest, options) {
|
|
|
167
176
|
(await hashUnnamedFile(path.join(dir, relative), size, options.budget));
|
|
168
177
|
const stagedAt = options.stagedAt.get(`${packId}/${relative}`) ?? null;
|
|
169
178
|
const committedAt = hash ? options.ledger.committed.get(hash) : undefined;
|
|
170
|
-
const base = {
|
|
179
|
+
const base = {
|
|
180
|
+
pack_id: packId,
|
|
181
|
+
relative_path: relative,
|
|
182
|
+
byte_size: size,
|
|
183
|
+
staged_on_disk_at: options.stagedOnDiskAt,
|
|
184
|
+
disk_age_ms: ageMs(options.stagedOnDiskAt, options.now),
|
|
185
|
+
};
|
|
171
186
|
if (committedAt) {
|
|
172
187
|
return {
|
|
173
188
|
...base,
|
|
@@ -359,13 +374,49 @@ function accumulatePack(inventory, pack) {
|
|
|
359
374
|
}
|
|
360
375
|
inventory.uncommitted_count += 1;
|
|
361
376
|
inventory.uncommitted_bytes += object.byte_size;
|
|
362
|
-
|
|
377
|
+
tallyUncommittedReason(inventory, object);
|
|
378
|
+
// The age of the BYTES, not of the answer about them (BLI-3797). Falls back
|
|
379
|
+
// to `decided_at` only when the file has no readable mtime, which is the
|
|
380
|
+
// one case where the disk itself cannot say.
|
|
381
|
+
const stagedAt = object.staged_on_disk_at ?? object.decided_at;
|
|
382
|
+
if (stagedAt &&
|
|
363
383
|
(!inventory.oldest_uncommitted_at ||
|
|
364
|
-
|
|
365
|
-
inventory.oldest_uncommitted_at =
|
|
384
|
+
stagedAt.localeCompare(inventory.oldest_uncommitted_at) < 0)) {
|
|
385
|
+
inventory.oldest_uncommitted_at = stagedAt;
|
|
366
386
|
}
|
|
367
387
|
}
|
|
368
388
|
}
|
|
389
|
+
/**
|
|
390
|
+
* One more object against its reason label. Kept as a mutable array rather than
|
|
391
|
+
* a Map so the inventory stays a plain serializable value — it rides `--json`
|
|
392
|
+
* output and a receipt, and a Map would silently become `{}` in both.
|
|
393
|
+
*/
|
|
394
|
+
function tallyUncommittedReason(inventory, object) {
|
|
395
|
+
const existing = inventory.uncommitted_reasons.find((tally) => tally.reason === object.reason);
|
|
396
|
+
const tally = existing ?? {
|
|
397
|
+
reason: object.reason,
|
|
398
|
+
count: 0,
|
|
399
|
+
bytes: 0,
|
|
400
|
+
oldest_disk_age_ms: 0,
|
|
401
|
+
};
|
|
402
|
+
tally.count += 1;
|
|
403
|
+
tally.bytes += object.byte_size;
|
|
404
|
+
tally.oldest_disk_age_ms = Math.max(tally.oldest_disk_age_ms, object.disk_age_ms || object.age_ms);
|
|
405
|
+
if (!existing)
|
|
406
|
+
inventory.uncommitted_reasons.push(tally);
|
|
407
|
+
}
|
|
408
|
+
/**
|
|
409
|
+
* Rank the reasons by BYTES, then name the biggest one.
|
|
410
|
+
*
|
|
411
|
+
* Bytes, not object count, because the question the ranking answers is "what is
|
|
412
|
+
* holding this laptop over its cap" — 234 objects at 1,836 MB and 17 at 81 MB
|
|
413
|
+
* are not the same finding, and counting rows would have called them close.
|
|
414
|
+
*/
|
|
415
|
+
function rankUncommittedReasons(inventory) {
|
|
416
|
+
inventory.uncommitted_reasons.sort((left, right) => right.bytes - left.bytes || right.count - left.count);
|
|
417
|
+
inventory.dominant_uncommitted_reason =
|
|
418
|
+
inventory.uncommitted_reasons[0] ?? null;
|
|
419
|
+
}
|
|
369
420
|
function emptyStagingInventory() {
|
|
370
421
|
return {
|
|
371
422
|
pack_count: 0,
|
|
@@ -376,6 +427,8 @@ function emptyStagingInventory() {
|
|
|
376
427
|
uncommitted_count: 0,
|
|
377
428
|
uncommitted_bytes: 0,
|
|
378
429
|
oldest_uncommitted_at: null,
|
|
430
|
+
uncommitted_reasons: [],
|
|
431
|
+
dominant_uncommitted_reason: null,
|
|
379
432
|
unknown_count: 0,
|
|
380
433
|
unknown_bytes: 0,
|
|
381
434
|
manifest_bytes: 0,
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Rebuilding an upload offer for evidence that is already on this disk.
|
|
3
|
+
*
|
|
4
|
+
* BLI-3797. The live sync only ever offers what THIS tick collected: it reads
|
|
5
|
+
* a transcript, hashes it, stages a copy, uploads that copy, and moves on. If
|
|
6
|
+
* the upload never happens — the process is killed between staging and
|
|
7
|
+
* delivery, the envelope is refused, the source file grows so next tick mints a
|
|
8
|
+
* different hash into a different pack — the staged bytes are orphaned. Nothing
|
|
9
|
+
* offers them again, ever. `disk-retention.ts` will not delete them (correctly:
|
|
10
|
+
* the ledger cannot vouch for them), so they sit until the disk fills.
|
|
11
|
+
*
|
|
12
|
+
* Measured on the reference Mac 2026-09-06, not reasoned about: 251 objects and
|
|
13
|
+
* 1,916.9 MB uncommitted against a 2,048 MB cap, 234 of them (1,835.8 MB)
|
|
14
|
+
* carrying `reconciled_unknown_to_server` — the SERVER was asked and answered
|
|
15
|
+
* that it has never held those hashes. `raw-evidence-staging.json` held six
|
|
16
|
+
* delivery-attempt rows in total, so those objects had not failed delivery;
|
|
17
|
+
* they had never been offered. The oldest pack was written 2026-07-23, 45 days
|
|
18
|
+
* earlier, and `cockpit doctor`'s fix sentence said "run `cockpit sync` to
|
|
19
|
+
* deliver them" — a sentence that could not come true.
|
|
20
|
+
*
|
|
21
|
+
* The repair is possible because each pack's own `manifest.json` already
|
|
22
|
+
* carries everything an offer needs: `object_key`, `content_hash_sha256`,
|
|
23
|
+
* `byte_size`, `media_type` and `redacted_summary` per file, and the pack's
|
|
24
|
+
* `work_context_id` / `session_id` / `operator_id` / `repo_basename` / `branch`
|
|
25
|
+
* beside them. This module rebuilds a `RawEvidenceUploadFile` from those
|
|
26
|
+
* recorded facts and NEVER re-derives them: the bytes on disk were sanitized
|
|
27
|
+
* when they were staged, and re-reading the original source would both cost the
|
|
28
|
+
* IO and risk offering content the redactor of the day would have masked
|
|
29
|
+
* differently.
|
|
30
|
+
*
|
|
31
|
+
* Metadata only, as everywhere in this family: pack ids, pack-relative paths,
|
|
32
|
+
* hashes, byte counts, reason labels. Never content, never an operator path.
|
|
33
|
+
*/
|
|
34
|
+
import fs from "node:fs/promises";
|
|
35
|
+
import path from "node:path";
|
|
36
|
+
import { RAW_EVIDENCE_BUCKET, RAW_EVIDENCE_RETENTION_MODE, } from "./adapters/raw-evidence-manifest.js";
|
|
37
|
+
import { RAW_EVIDENCE_DIR } from "./disk-usage.js";
|
|
38
|
+
import { describeError } from "./health-detail.js";
|
|
39
|
+
import { LOCAL_COLLECTOR_VERSION } from "./local-state.js";
|
|
40
|
+
import { deliveryHold, } from "./raw-evidence-staging.js";
|
|
41
|
+
/**
|
|
42
|
+
* How much undelivered backlog one tick may re-offer.
|
|
43
|
+
*
|
|
44
|
+
* A tick is fifteen minutes and the backlog is measured in gigabytes, so this
|
|
45
|
+
* is a drain rate, not a ceiling on what will eventually be sent: 512 MB a tick
|
|
46
|
+
* clears the reference Mac's 1.9 GB in four ticks (one hour) without competing
|
|
47
|
+
* with the tick's own collection, which owns a separate 2 GB budget. The object
|
|
48
|
+
* count bounds the number of `begin` round-trips for the opposite case — a
|
|
49
|
+
* backlog of thousands of small objects.
|
|
50
|
+
*/
|
|
51
|
+
export const REDELIVERY_MAX_BYTES_PER_TICK = 512 * 1024 * 1024;
|
|
52
|
+
export const REDELIVERY_MAX_OBJECTS_PER_TICK = 64;
|
|
53
|
+
/**
|
|
54
|
+
* Which undelivered objects this tick will re-offer, oldest bytes first.
|
|
55
|
+
*
|
|
56
|
+
* Oldest first is deliberate: the objects that have been undelivered longest
|
|
57
|
+
* are the ones most likely to be the only remaining copy of a session whose
|
|
58
|
+
* source transcript has since been rotated away by its agent host, and they are
|
|
59
|
+
* also the ones the disk cap will start refusing to keep. A newest-first drain
|
|
60
|
+
* would leave a 45-day-old orphan permanently at the back of the queue.
|
|
61
|
+
*
|
|
62
|
+
* `unknown` objects are deliberately NOT planned. "The ledger cannot say" is
|
|
63
|
+
* `cockpit clean --reconcile`'s question, it already runs one bounded batch per
|
|
64
|
+
* tick ahead of this, and it turns each answer into `committed` or
|
|
65
|
+
* `uncommitted` — so an unknown object reaches this planner on a later tick,
|
|
66
|
+
* with the server's own answer behind it, rather than being uploaded again on a
|
|
67
|
+
* guess.
|
|
68
|
+
*/
|
|
69
|
+
export async function planEvidenceRedelivery(options) {
|
|
70
|
+
const plan = {
|
|
71
|
+
packs: [],
|
|
72
|
+
object_count: 0,
|
|
73
|
+
bytes: 0,
|
|
74
|
+
held_count: 0,
|
|
75
|
+
held_bytes: 0,
|
|
76
|
+
unplannable: {},
|
|
77
|
+
deferred_count: 0,
|
|
78
|
+
deferred_bytes: 0,
|
|
79
|
+
};
|
|
80
|
+
const maxBytes = options.maxBytes ?? REDELIVERY_MAX_BYTES_PER_TICK;
|
|
81
|
+
const maxObjects = options.maxObjects ?? REDELIVERY_MAX_OBJECTS_PER_TICK;
|
|
82
|
+
const root = path.join(options.stateDir, RAW_EVIDENCE_DIR);
|
|
83
|
+
const candidates = uncommittedOldestFirst(options.inventory);
|
|
84
|
+
const manifests = new Map();
|
|
85
|
+
const packPlans = new Map();
|
|
86
|
+
for (const object of candidates) {
|
|
87
|
+
if (!object.content_hash) {
|
|
88
|
+
countUnplannable(plan, "no_content_hash");
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
if (deliveryHold(options.staging, object.content_hash, options.now)) {
|
|
92
|
+
plan.held_count += 1;
|
|
93
|
+
plan.held_bytes += object.byte_size;
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
if (plan.object_count >= maxObjects ||
|
|
97
|
+
plan.bytes + object.byte_size > maxBytes) {
|
|
98
|
+
plan.deferred_count += 1;
|
|
99
|
+
plan.deferred_bytes += object.byte_size;
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
if (!manifests.has(object.pack_id)) {
|
|
103
|
+
manifests.set(object.pack_id, await readPackManifest(path.join(root, object.pack_id), object.pack_id));
|
|
104
|
+
}
|
|
105
|
+
const manifest = manifests.get(object.pack_id) ?? null;
|
|
106
|
+
if (!manifest) {
|
|
107
|
+
countUnplannable(plan, "manifest_unreadable");
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
const row = manifest.files.get(object.relative_path);
|
|
111
|
+
if (!row) {
|
|
112
|
+
countUnplannable(plan, "no_manifest_row");
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
let packPlan = packPlans.get(object.pack_id);
|
|
116
|
+
if (!packPlan) {
|
|
117
|
+
packPlan = {
|
|
118
|
+
pack_id: object.pack_id,
|
|
119
|
+
provenance: manifest.provenance,
|
|
120
|
+
files: [],
|
|
121
|
+
bytes: 0,
|
|
122
|
+
};
|
|
123
|
+
packPlans.set(object.pack_id, packPlan);
|
|
124
|
+
plan.packs.push(packPlan);
|
|
125
|
+
}
|
|
126
|
+
packPlan.files.push({
|
|
127
|
+
pointer: {
|
|
128
|
+
raw_evidence_pointer_id: row.object_key,
|
|
129
|
+
privacy_classification: "remote_durable_raw_evidence",
|
|
130
|
+
retention_policy: {
|
|
131
|
+
mode: RAW_EVIDENCE_RETENTION_MODE,
|
|
132
|
+
privacy_classification: "remote_durable_raw_evidence",
|
|
133
|
+
},
|
|
134
|
+
storage_scope: "remote_object",
|
|
135
|
+
storage_bucket: RAW_EVIDENCE_BUCKET,
|
|
136
|
+
object_key: row.object_key,
|
|
137
|
+
content_hash_sha256: row.content_hash_sha256,
|
|
138
|
+
byte_size: row.byte_size,
|
|
139
|
+
media_type: row.media_type,
|
|
140
|
+
redacted_summary: row.redacted_summary,
|
|
141
|
+
},
|
|
142
|
+
local_path: path.join(root, object.pack_id, ...object.relative_path.split("/").filter(Boolean)),
|
|
143
|
+
kind: row.kind,
|
|
144
|
+
codex_session_id: null,
|
|
145
|
+
});
|
|
146
|
+
packPlan.bytes += object.byte_size;
|
|
147
|
+
plan.object_count += 1;
|
|
148
|
+
plan.bytes += object.byte_size;
|
|
149
|
+
}
|
|
150
|
+
return plan;
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Every uncommitted object, oldest staged bytes first.
|
|
154
|
+
*
|
|
155
|
+
* Sorted on `disk_age_ms` (the file's own mtime) rather than `age_ms`, because
|
|
156
|
+
* `age_ms` for a reconciled object is the age of the reconcile ANSWER — the
|
|
157
|
+
* same confusion that had 45-day-old orphans reading as one day old.
|
|
158
|
+
*/
|
|
159
|
+
function uncommittedOldestFirst(inventory) {
|
|
160
|
+
return inventory.packs
|
|
161
|
+
.flatMap((pack) => pack.objects)
|
|
162
|
+
.filter((object) => object.state === "uncommitted")
|
|
163
|
+
.sort((left, right) => (right.disk_age_ms || right.age_ms) - (left.disk_age_ms || left.age_ms));
|
|
164
|
+
}
|
|
165
|
+
function countUnplannable(plan, reason) {
|
|
166
|
+
plan.unplannable[reason] = (plan.unplannable[reason] ?? 0) + 1;
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* A pack's recorded provenance and file rows.
|
|
170
|
+
*
|
|
171
|
+
* Every field is read back from what the collector WROTE when it staged the
|
|
172
|
+
* pack; nothing is re-derived from the machine's current state. A pack staged
|
|
173
|
+
* six weeks ago under a work context that no longer exists still knows which
|
|
174
|
+
* repo, branch, operator and work context it belonged to, and that is the only
|
|
175
|
+
* honest label for those bytes.
|
|
176
|
+
*/
|
|
177
|
+
async function readPackManifest(dir, packId) {
|
|
178
|
+
const raw = await fs
|
|
179
|
+
.readFile(path.join(dir, "manifest.json"), "utf8")
|
|
180
|
+
.catch(() => null);
|
|
181
|
+
if (!raw)
|
|
182
|
+
return null;
|
|
183
|
+
try {
|
|
184
|
+
const parsed = JSON.parse(raw);
|
|
185
|
+
const provenance = provenanceFromManifest(parsed, packId);
|
|
186
|
+
if (!provenance)
|
|
187
|
+
return null;
|
|
188
|
+
const files = new Map();
|
|
189
|
+
for (const entry of asArray(parsed["files"])) {
|
|
190
|
+
const row = manifestFileRow(entry);
|
|
191
|
+
if (row)
|
|
192
|
+
files.set(row.relative_path, row.value);
|
|
193
|
+
}
|
|
194
|
+
return { provenance, files };
|
|
195
|
+
}
|
|
196
|
+
catch (error) {
|
|
197
|
+
console.error("[evidence-redelivery] pack manifest unreadable; its objects cannot be re-offered", JSON.stringify({
|
|
198
|
+
reason: "manifest_unreadable",
|
|
199
|
+
pack_id: packId,
|
|
200
|
+
...describeError(error),
|
|
201
|
+
}));
|
|
202
|
+
return null;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
function provenanceFromManifest(manifest, packId) {
|
|
206
|
+
const workContextId = asText(manifest["work_context_id"]);
|
|
207
|
+
const sessionId = asText(manifest["session_id"]);
|
|
208
|
+
const operatorId = asText(manifest["operator_id"]);
|
|
209
|
+
const repo = asText(manifest["repo_basename"]) ?? asText(manifest["repo_label"]);
|
|
210
|
+
if (!workContextId || !sessionId || !operatorId || !repo) {
|
|
211
|
+
console.error("[evidence-redelivery] pack manifest names no work context; its objects cannot be re-offered", JSON.stringify({
|
|
212
|
+
reason: "manifest_provenance_incomplete",
|
|
213
|
+
pack_id: packId,
|
|
214
|
+
has_work_context: Boolean(workContextId),
|
|
215
|
+
has_session: Boolean(sessionId),
|
|
216
|
+
has_operator: Boolean(operatorId),
|
|
217
|
+
has_repo: Boolean(repo),
|
|
218
|
+
}));
|
|
219
|
+
return null;
|
|
220
|
+
}
|
|
221
|
+
const repoLabel = asText(manifest["repo_label"]);
|
|
222
|
+
const worktreeLabel = asText(manifest["worktree_label"]);
|
|
223
|
+
return {
|
|
224
|
+
capture_source: "collector_runtime",
|
|
225
|
+
// The version that is re-offering, not the one that staged: this IS a
|
|
226
|
+
// delivery by today's collector, and pretending otherwise would misreport
|
|
227
|
+
// which build put the object in the bucket.
|
|
228
|
+
capture_adapter_version: LOCAL_COLLECTOR_VERSION,
|
|
229
|
+
collector_version: LOCAL_COLLECTOR_VERSION,
|
|
230
|
+
repo,
|
|
231
|
+
// A pack staged on a detached HEAD or a deleted worktree can have no branch
|
|
232
|
+
// recorded. `unknown` is a label, not a guess at which branch it was.
|
|
233
|
+
branch: asText(manifest["branch"]) ?? "unknown",
|
|
234
|
+
...(repoLabel ? { repo_label: repoLabel } : {}),
|
|
235
|
+
...(worktreeLabel ? { worktree_label: worktreeLabel } : {}),
|
|
236
|
+
operator_id: operatorId,
|
|
237
|
+
session_id: sessionId,
|
|
238
|
+
work_context_id: workContextId,
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
function manifestFileRow(entry) {
|
|
242
|
+
if (!entry || typeof entry !== "object")
|
|
243
|
+
return null;
|
|
244
|
+
const row = entry;
|
|
245
|
+
const relativePath = asText(row["relative_path"]);
|
|
246
|
+
const objectKey = asText(row["object_key"]);
|
|
247
|
+
const hash = asText(row["content_hash_sha256"]);
|
|
248
|
+
const mediaType = asText(row["media_type"]);
|
|
249
|
+
const summary = asText(row["redacted_summary"]);
|
|
250
|
+
const byteSize = row["byte_size"];
|
|
251
|
+
if (!relativePath ||
|
|
252
|
+
!objectKey ||
|
|
253
|
+
!hash ||
|
|
254
|
+
hash.length !== 64 ||
|
|
255
|
+
!mediaType ||
|
|
256
|
+
!summary ||
|
|
257
|
+
typeof byteSize !== "number") {
|
|
258
|
+
return null;
|
|
259
|
+
}
|
|
260
|
+
return {
|
|
261
|
+
relative_path: relativePath,
|
|
262
|
+
value: {
|
|
263
|
+
object_key: objectKey,
|
|
264
|
+
content_hash_sha256: hash,
|
|
265
|
+
byte_size: byteSize,
|
|
266
|
+
media_type: mediaType,
|
|
267
|
+
redacted_summary: summary,
|
|
268
|
+
kind: asText(row["kind"]) ?? "unknown",
|
|
269
|
+
},
|
|
270
|
+
};
|
|
271
|
+
}
|
|
272
|
+
function asArray(value) {
|
|
273
|
+
return Array.isArray(value) ? value : [];
|
|
274
|
+
}
|
|
275
|
+
function asText(value) {
|
|
276
|
+
return typeof value === "string" && value.trim() ? value : null;
|
|
277
|
+
}
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Offering undelivered staged evidence again, on the next tick and every tick
|
|
3
|
+
* after, until it lands or names why it cannot.
|
|
4
|
+
*
|
|
5
|
+
* BLI-3797. `evidence-redelivery-plan.ts` says WHICH bytes and rebuilds the
|
|
6
|
+
* offer; this file spends it against the same chunked upload client the live
|
|
7
|
+
* sync uses, writes each success into the commit ledger and each failure into
|
|
8
|
+
* the delivery-attempt history (so the existing 15-min → 6-h backoff applies
|
|
9
|
+
* and one doomed object is not re-offered 96 times a day), and says what
|
|
10
|
+
* happened on every branch.
|
|
11
|
+
*
|
|
12
|
+
* Four properties, the same ones the reconcile and prune follow-ups keep:
|
|
13
|
+
*
|
|
14
|
+
* - **It never throws and never fails a tick.** A drain that cannot run is a
|
|
15
|
+
* log line and a receipt, never a reason collection did not happen.
|
|
16
|
+
* - **It never deletes.** Nothing here removes a staged byte; the prune that
|
|
17
|
+
* follows it decides that, and it only ever takes what the ledger vouches
|
|
18
|
+
* for — which now includes whatever this just delivered.
|
|
19
|
+
* - **It is bounded.** One drain rate per tick (see the plan module), so the
|
|
20
|
+
* backlog clears in hours without competing with the tick's own collection.
|
|
21
|
+
* - **It says so on both branches.** A tick with nothing undelivered is silent
|
|
22
|
+
* (the steady state, 96 times a day); every other outcome, including a clean
|
|
23
|
+
* drain, gets one `[evidence-redelivery]` line with counts and reasons.
|
|
24
|
+
*/
|
|
25
|
+
import { uploadRawEvidenceFilesChunked, } from "./evidence-upload-client.js";
|
|
26
|
+
import { planEvidenceRedelivery, } from "./evidence-redelivery-plan.js";
|
|
27
|
+
import { markObjectCommitted, readRawEvidenceCursor, writeRawEvidenceCursor, } from "./cursors/raw-evidence-cursor.js";
|
|
28
|
+
import { readStagingInventory } from "./disk-usage.js";
|
|
29
|
+
import { describeError } from "./health-detail.js";
|
|
30
|
+
import { getCollectorRuntimePaths, readLocalCollectorSessionFile, } from "./local-state.js";
|
|
31
|
+
import { clearDeliveryAttempt, readRawEvidenceStagingState, recordDeliveryFailure, writeRawEvidenceStagingState, } from "./raw-evidence-staging.js";
|
|
32
|
+
/**
|
|
33
|
+
* The drain the sync tick runs and `cockpit doctor`'s disk fix reuses.
|
|
34
|
+
* Never throws.
|
|
35
|
+
*/
|
|
36
|
+
export async function runEvidenceRedelivery(options) {
|
|
37
|
+
const env = options.env ?? process.env;
|
|
38
|
+
const now = options.now ?? new Date();
|
|
39
|
+
if (env["COCKPIT_DISABLE_EVIDENCE_REDELIVERY"] === "1") {
|
|
40
|
+
return emptyResult("disabled", "skipped");
|
|
41
|
+
}
|
|
42
|
+
const paths = getCollectorRuntimePaths(options.homeDir);
|
|
43
|
+
try {
|
|
44
|
+
const inventory = options.inventory ?? (await readStagingInventory(paths, now));
|
|
45
|
+
if (inventory.uncommitted_count === 0) {
|
|
46
|
+
// The steady state, 96 ticks a day. Saying so every time would bury the
|
|
47
|
+
// ticks that matter; `cockpit doctor`'s disk row still says it out loud
|
|
48
|
+
// to a person who asks.
|
|
49
|
+
return emptyResult("nothing_uncommitted", "ok");
|
|
50
|
+
}
|
|
51
|
+
const staging = await readRawEvidenceStagingState(paths.state_dir);
|
|
52
|
+
const plan = await planEvidenceRedelivery({
|
|
53
|
+
stateDir: paths.state_dir,
|
|
54
|
+
inventory,
|
|
55
|
+
staging,
|
|
56
|
+
now,
|
|
57
|
+
...(options.maxBytes != null ? { maxBytes: options.maxBytes } : {}),
|
|
58
|
+
...(options.maxObjects != null ? { maxObjects: options.maxObjects } : {}),
|
|
59
|
+
});
|
|
60
|
+
if (plan.object_count === 0) {
|
|
61
|
+
const result = emptyPlanResult(plan);
|
|
62
|
+
reportRedelivery(result, plan);
|
|
63
|
+
return result;
|
|
64
|
+
}
|
|
65
|
+
const session = await readLocalCollectorSessionFile(paths).catch(() => null);
|
|
66
|
+
if (!session ||
|
|
67
|
+
session.session_state !== "valid" ||
|
|
68
|
+
typeof session.device_token !== "string" ||
|
|
69
|
+
!session.device_token) {
|
|
70
|
+
// Not a failure of the backlog: this machine simply cannot speak to the
|
|
71
|
+
// dashboard right now. The bytes stay, the reason is named, and the next
|
|
72
|
+
// tick after a `cockpit login` picks them straight back up.
|
|
73
|
+
const result = {
|
|
74
|
+
...emptyPlanResult(plan),
|
|
75
|
+
status: "skipped",
|
|
76
|
+
reason: "no_device_session",
|
|
77
|
+
};
|
|
78
|
+
reportRedelivery(result, plan);
|
|
79
|
+
return result;
|
|
80
|
+
}
|
|
81
|
+
const outcome = await deliverPlan({
|
|
82
|
+
plan,
|
|
83
|
+
dashboardUrl: options.dashboardUrl,
|
|
84
|
+
deviceToken: session.device_token,
|
|
85
|
+
fetchImpl: options.fetch ?? fetch,
|
|
86
|
+
now,
|
|
87
|
+
});
|
|
88
|
+
await recordOutcomes(paths, staging, outcome.outcomes, now);
|
|
89
|
+
const result = {
|
|
90
|
+
status: outcome.failed > 0 && outcome.uploaded === 0 ? "fail" : "ok",
|
|
91
|
+
reason: "redelivered",
|
|
92
|
+
offered: plan.object_count,
|
|
93
|
+
offered_bytes: plan.bytes,
|
|
94
|
+
uploaded: outcome.uploaded,
|
|
95
|
+
uploaded_bytes: outcome.uploadedBytes,
|
|
96
|
+
reused: outcome.reused,
|
|
97
|
+
failed: outcome.failed,
|
|
98
|
+
held: plan.held_count,
|
|
99
|
+
deferred: plan.deferred_count,
|
|
100
|
+
deferred_bytes: plan.deferred_bytes,
|
|
101
|
+
packs: plan.packs.length,
|
|
102
|
+
failure_reasons: outcome.failureReasons,
|
|
103
|
+
};
|
|
104
|
+
reportRedelivery(result, plan);
|
|
105
|
+
return result;
|
|
106
|
+
}
|
|
107
|
+
catch (error) {
|
|
108
|
+
console.error("[evidence-redelivery] the drain could not run; nothing was re-offered", JSON.stringify({ reason: "redelivery_threw", ...describeError(error) }));
|
|
109
|
+
return { ...emptyResult("redelivery_threw", "fail") };
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* One `uploadRawEvidenceFilesChunked` call per pack, because provenance is a
|
|
114
|
+
* PACK fact: the repo, branch, operator, session and work context these bytes
|
|
115
|
+
* belonged to were recorded when they were staged, and sending one pack's
|
|
116
|
+
* objects under another pack's labels would file six-week-old evidence against
|
|
117
|
+
* today's ticket.
|
|
118
|
+
*/
|
|
119
|
+
async function deliverPlan(options) {
|
|
120
|
+
const result = {
|
|
121
|
+
outcomes: [],
|
|
122
|
+
uploaded: 0,
|
|
123
|
+
uploadedBytes: 0,
|
|
124
|
+
reused: 0,
|
|
125
|
+
failed: 0,
|
|
126
|
+
failureReasons: [],
|
|
127
|
+
};
|
|
128
|
+
for (const pack of options.plan.packs) {
|
|
129
|
+
const uploaded = await deliverOnePack(pack, options);
|
|
130
|
+
for (const outcome of uploaded) {
|
|
131
|
+
result.outcomes.push(outcome);
|
|
132
|
+
if (outcome.upload_state === "uploaded") {
|
|
133
|
+
result.uploaded += 1;
|
|
134
|
+
result.uploadedBytes += outcome.pointer.byte_size ?? 0;
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
if (outcome.upload_state === "reused_existing") {
|
|
138
|
+
result.reused += 1;
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
result.failed += 1;
|
|
142
|
+
const reason = outcome.reason ?? "unknown";
|
|
143
|
+
if (!result.failureReasons.includes(reason)) {
|
|
144
|
+
result.failureReasons.push(reason);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
return result;
|
|
149
|
+
}
|
|
150
|
+
async function deliverOnePack(pack, options) {
|
|
151
|
+
try {
|
|
152
|
+
const uploaded = await uploadRawEvidenceFilesChunked({
|
|
153
|
+
fetchImpl: options.fetchImpl,
|
|
154
|
+
dashboardUrl: options.dashboardUrl,
|
|
155
|
+
deviceToken: options.deviceToken,
|
|
156
|
+
provenance: pack.provenance,
|
|
157
|
+
generatedAt: options.now.toISOString(),
|
|
158
|
+
files: pack.files,
|
|
159
|
+
});
|
|
160
|
+
return uploaded.outcomes;
|
|
161
|
+
}
|
|
162
|
+
catch (error) {
|
|
163
|
+
// The upload client already reports its own transport failures; this is the
|
|
164
|
+
// net for anything it does not, and it must not abandon the packs behind
|
|
165
|
+
// this one. Each file is reported failed so the backoff history records the
|
|
166
|
+
// attempt rather than re-offering all of them again in fifteen minutes.
|
|
167
|
+
console.error("[evidence-redelivery] a pack's re-offer threw; the rest of the drain continues", JSON.stringify({
|
|
168
|
+
reason: "pack_redelivery_threw",
|
|
169
|
+
pack_id: pack.pack_id,
|
|
170
|
+
object_count: pack.files.length,
|
|
171
|
+
...describeError(error),
|
|
172
|
+
}));
|
|
173
|
+
return pack.files.map((file) => ({
|
|
174
|
+
pointer: file.pointer,
|
|
175
|
+
object_key: file.pointer.object_key ?? "",
|
|
176
|
+
codex_session_id: null,
|
|
177
|
+
kind: file.kind ?? "unknown",
|
|
178
|
+
upload_state: "upload_failed",
|
|
179
|
+
reason: "pack_redelivery_threw",
|
|
180
|
+
uploaded_chunk_count: 0,
|
|
181
|
+
}));
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Write what happened where the next tick will read it: the commit ledger for
|
|
186
|
+
* anything that landed, the delivery-attempt history for anything that did not.
|
|
187
|
+
*
|
|
188
|
+
* Both writes happen even when the run partly failed, and the ledger is written
|
|
189
|
+
* FIRST — a delivered object whose ledger row was lost would be re-offered
|
|
190
|
+
* forever, which is the failure this whole module exists to end.
|
|
191
|
+
*/
|
|
192
|
+
async function recordOutcomes(paths, staging, outcomes, now) {
|
|
193
|
+
const cursor = await readRawEvidenceCursor(paths);
|
|
194
|
+
let ledgerChanged = false;
|
|
195
|
+
let stagingChanged = false;
|
|
196
|
+
for (const outcome of outcomes) {
|
|
197
|
+
const hash = outcome.pointer.content_hash_sha256;
|
|
198
|
+
if (!hash)
|
|
199
|
+
continue;
|
|
200
|
+
if (outcome.upload_state === "upload_failed") {
|
|
201
|
+
recordDeliveryFailure(staging, hash, {
|
|
202
|
+
reason: outcome.reason ?? "unknown",
|
|
203
|
+
attemptedAt: now,
|
|
204
|
+
byteSize: outcome.pointer.byte_size ?? 0,
|
|
205
|
+
});
|
|
206
|
+
stagingChanged = true;
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
209
|
+
markObjectCommitted(cursor, hash, {
|
|
210
|
+
object_key: outcome.object_key,
|
|
211
|
+
byte_size: outcome.pointer.byte_size ?? 0,
|
|
212
|
+
committed_at: now.toISOString(),
|
|
213
|
+
});
|
|
214
|
+
ledgerChanged = true;
|
|
215
|
+
if (clearDeliveryAttempt(staging, hash))
|
|
216
|
+
stagingChanged = true;
|
|
217
|
+
}
|
|
218
|
+
if (ledgerChanged)
|
|
219
|
+
await writeRawEvidenceCursor(paths, cursor);
|
|
220
|
+
if (stagingChanged) {
|
|
221
|
+
await writeRawEvidenceStagingState(paths.state_dir, staging);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* One line per non-steady-state tick. Counts, byte totals and reason labels
|
|
226
|
+
* only — no pack id, no hash, no path, per the logging contract.
|
|
227
|
+
*/
|
|
228
|
+
function reportRedelivery(result, plan) {
|
|
229
|
+
const fields = {
|
|
230
|
+
reason: result.reason,
|
|
231
|
+
status: result.status,
|
|
232
|
+
offered: result.offered,
|
|
233
|
+
offered_bytes: result.offered_bytes,
|
|
234
|
+
uploaded: result.uploaded,
|
|
235
|
+
uploaded_bytes: result.uploaded_bytes,
|
|
236
|
+
reused: result.reused,
|
|
237
|
+
failed: result.failed,
|
|
238
|
+
held_by_backoff: result.held,
|
|
239
|
+
deferred_to_next_tick: result.deferred,
|
|
240
|
+
deferred_bytes: result.deferred_bytes,
|
|
241
|
+
packs: result.packs,
|
|
242
|
+
unplannable: plan.unplannable,
|
|
243
|
+
failure_reasons: result.failure_reasons,
|
|
244
|
+
};
|
|
245
|
+
if (result.status === "fail") {
|
|
246
|
+
console.error("[evidence-redelivery] every re-offered object was refused", JSON.stringify(fields));
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
console.error(result.uploaded > 0
|
|
250
|
+
? "[evidence-redelivery] undelivered staged evidence landed"
|
|
251
|
+
: "[evidence-redelivery] undelivered staged evidence was not re-offered this tick", JSON.stringify(fields));
|
|
252
|
+
}
|
|
253
|
+
function emptyPlanResult(plan) {
|
|
254
|
+
const nothingPlannable = plan.held_count > 0 && plan.deferred_count === 0
|
|
255
|
+
? "all_held_by_backoff"
|
|
256
|
+
: "nothing_plannable";
|
|
257
|
+
return {
|
|
258
|
+
...emptyResult(nothingPlannable, "ok"),
|
|
259
|
+
held: plan.held_count,
|
|
260
|
+
deferred: plan.deferred_count,
|
|
261
|
+
deferred_bytes: plan.deferred_bytes,
|
|
262
|
+
};
|
|
263
|
+
}
|
|
264
|
+
function emptyResult(reason, status) {
|
|
265
|
+
return {
|
|
266
|
+
status,
|
|
267
|
+
reason,
|
|
268
|
+
offered: 0,
|
|
269
|
+
offered_bytes: 0,
|
|
270
|
+
uploaded: 0,
|
|
271
|
+
uploaded_bytes: 0,
|
|
272
|
+
reused: 0,
|
|
273
|
+
failed: 0,
|
|
274
|
+
held: 0,
|
|
275
|
+
deferred: 0,
|
|
276
|
+
deferred_bytes: 0,
|
|
277
|
+
packs: 0,
|
|
278
|
+
failure_reasons: [],
|
|
279
|
+
};
|
|
280
|
+
}
|