@bli-cockpit/cli 0.2.173 → 0.2.179
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/backfill-lock-wait.js +4 -1
- package/dist/commands/doctor-backfill.js +160 -0
- package/dist/commands/doctor-budget.js +14 -17
- package/dist/commands/doctor-collection-lock.js +201 -0
- package/dist/commands/doctor-disk-words.js +34 -3
- package/dist/commands/doctor-pipeline-verdicts.js +13 -3
- package/dist/commands/doctor-pipeline.js +22 -149
- package/dist/commands/doctor-progress.js +54 -0
- package/dist/commands/doctor-report.js +2 -1
- package/dist/commands/doctor.js +11 -3
- package/dist/commands/local-args-tower-logs.js +30 -11
- package/dist/commands/local-help-commands-tower-ops.js +11 -3
- package/dist/commands/local-help-commands.js +1 -1
- package/dist/commands/logs-stored.js +56 -0
- package/dist/commands/logs.js +18 -2
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/sync-run.js +3 -0
- package/dist/commands/sync-yield.js +4 -0
- package/dist/commands/sync.js +30 -11
- package/dist/evidence-upload-transport.js +4 -0
- package/dist/state-dir-audit.js +32 -1
- package/dist/sync-lock-holder.js +82 -0
- package/dist/sync-lock.js +111 -13
- package/dist/sync-tick-timing.js +34 -0
- package/package.json +4 -4
- package/dist/commands/doctor-lock-wait.js +0 -55
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { doctorBackfillBudgetSeconds, doctorLockWaitSeconds } from "./doctor-budget.js";
|
|
3
|
-
import { inspectBackfillLock } from "../backfill-lock.js";
|
|
4
|
-
import { readBackfillCompletionMarker, readBackfillCursor, } from "../cursors/backfill-cursor.js";
|
|
1
|
+
import { withCollectionGate } from "./doctor-collection-lock.js";
|
|
5
2
|
import { savedDiscoveryLimitArgs } from "../discovery-limits.js";
|
|
6
3
|
import { runStagingPrune } from "../disk-prune.js";
|
|
7
4
|
import { retentionOptionsFromEnv } from "../disk-retention.js";
|
|
@@ -10,11 +7,11 @@ import { readDiskFootprint } from "../disk-usage.js";
|
|
|
10
7
|
import { runEvidenceRedelivery } from "../evidence-redelivery.js";
|
|
11
8
|
import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths } from "../local-state.js";
|
|
12
9
|
import { runRawEvidenceLocalGc, rawEvidenceGcSummary } from "../raw-evidence-gc.js";
|
|
13
|
-
import { runBackfillCommand } from "./backfill.js";
|
|
14
10
|
import { FOOTPRINT_NEEDS_FIX_BYTES, stagingPausedWords, diskRowMessage, footprintHolder, footprintTotalBytes, mib, redeliveryLine, } from "./doctor-disk-words.js";
|
|
15
11
|
import { doctorRoots } from "./doctor-access.js";
|
|
16
|
-
import {
|
|
17
|
-
import {
|
|
12
|
+
import { startProgress, waitingLine } from "./doctor-progress.js";
|
|
13
|
+
import { jsonField, parseDoctorSyncJson, syncFailureMessage, syncBacklogDrainingVerdict, syncStandAsideVerdict, } from "./doctor-pipeline-verdicts.js";
|
|
14
|
+
import { fail, needsFix, ok, skipped } from "./doctor-report.js";
|
|
18
15
|
/**
|
|
19
16
|
* The `backfill-complete`, `gc-checked`, `disk-bounded`, and `sync-fresh`
|
|
20
17
|
* check family: does the collection pipeline itself have everything it
|
|
@@ -29,108 +26,9 @@ import { asRecord, fail, needsFix, ok, skipped } from "./doctor-report.js";
|
|
|
29
26
|
* completion marker proves, what a stand-aside receipt means, when an
|
|
30
27
|
* unfinished backfill is busy rather than broken — are pure functions in
|
|
31
28
|
* `doctor-pipeline-verdicts.ts`, and every one of them is re-exported from
|
|
32
|
-
* here so callers keep one address.
|
|
29
|
+
* here so callers keep one address. The catch-up row itself moved to
|
|
30
|
+
* `doctor-backfill.ts` (BLI-12872) and is re-exported below as well.
|
|
33
31
|
*/
|
|
34
|
-
export async function checkBackfillState(context) {
|
|
35
|
-
const paths = getCollectorRuntimePaths(context.command.homeDir);
|
|
36
|
-
const roots = await doctorRoots(context);
|
|
37
|
-
const marker = await readBackfillCompletionMarker(paths);
|
|
38
|
-
const covered = backfillCompletionStepState(marker, roots);
|
|
39
|
-
if (covered)
|
|
40
|
-
return covered;
|
|
41
|
-
const lock = await inspectBackfillLock(paths);
|
|
42
|
-
if (lock.held) {
|
|
43
|
-
return needsFix("backfill-complete", "backfill_already_running", `another catch-up run is still going, and this one has not finished yet (running since ${lock.held_since ?? "unknown"})`);
|
|
44
|
-
}
|
|
45
|
-
const cursor = await readBackfillCursor(paths);
|
|
46
|
-
return needsFix("backfill-complete", cursor.updated_at ? "partial" : "never_run", "the catch-up over your old sessions has not finished");
|
|
47
|
-
}
|
|
48
|
-
export async function fixBackfillState(context) {
|
|
49
|
-
const started = Date.now();
|
|
50
|
-
const deadline = started + doctorBackfillBudgetSeconds(context, started) * 1000;
|
|
51
|
-
let caughtUp = 0;
|
|
52
|
-
let chunks = 0;
|
|
53
|
-
let live = null;
|
|
54
|
-
const elapsed = () => {
|
|
55
|
-
const seconds = Math.floor((Date.now() - started) / 1000);
|
|
56
|
-
return `${Math.floor(seconds / 60)}m${String(seconds % 60).padStart(2, "0")}s`;
|
|
57
|
-
};
|
|
58
|
-
const receipt = () => `caught up on ${caughtUp} old sessions in ${chunks} chunk${chunks === 1 ? "" : "s"} over ${elapsed()}`;
|
|
59
|
-
let remaining = null;
|
|
60
|
-
const unfinished = (code, reason) => ({
|
|
61
|
-
...needsFix("backfill-complete", code, `${receipt()}, ${remaining ?? "unknown"} still to go; ${reason}; run \`cockpit doctor\` again to continue`),
|
|
62
|
-
nextAction: "cockpit doctor",
|
|
63
|
-
});
|
|
64
|
-
const progress = setInterval(() => {
|
|
65
|
-
const seconds = Math.floor((Date.now() - started) / 1000);
|
|
66
|
-
const uploaded = live === null ? null : caughtUp + live.uploaded;
|
|
67
|
-
context.io.stderr.write(live === null
|
|
68
|
-
? `backfill: starting, ${elapsed()}\n`
|
|
69
|
-
: `backfill: ${uploaded} caught up, ${live.processed} processed in this chunk, ${live.remaining} to go, ${elapsed()}\n`);
|
|
70
|
-
console.error("[doctor] backfill-complete progress", JSON.stringify({
|
|
71
|
-
status: live === null ? "starting" : "running", processed: live?.processed ?? null,
|
|
72
|
-
uploaded, caught_up: uploaded, remaining: live?.remaining ?? null,
|
|
73
|
-
chunks, elapsed_seconds: seconds,
|
|
74
|
-
}));
|
|
75
|
-
}, 30_000);
|
|
76
|
-
try {
|
|
77
|
-
while (Date.now() < deadline) {
|
|
78
|
-
// Every chunk reacquires the collector's locks. Bound lock waiting by
|
|
79
|
-
// the overall budget too; never abandon an in-flight cursor write.
|
|
80
|
-
const chunkContext = { ...context, command: { ...context.command,
|
|
81
|
-
lockWaitSeconds: Math.min(doctorLockWaitSeconds(context), (deadline - Date.now()) / 1000),
|
|
82
|
-
} };
|
|
83
|
-
const row = await withDoctorLockWait(chunkContext, async () => {
|
|
84
|
-
if (Date.now() >= deadline)
|
|
85
|
-
return unfinished("backfill_budget_timeout", "backfill time budget reached");
|
|
86
|
-
const capture = capturedIo(context.io, false);
|
|
87
|
-
live = null;
|
|
88
|
-
chunks += 1;
|
|
89
|
-
const code = await runBackfillCommand({
|
|
90
|
-
homeDir: context.command.homeDir, repoRoot: context.command.repoRoot,
|
|
91
|
-
all: true, dryRun: false, yes: true, json: true,
|
|
92
|
-
deadlineMs: deadline,
|
|
93
|
-
// Doctor's own `withDoctorLockWait` owns the waiting (BLI-11091).
|
|
94
|
-
syncLockWaitMs: 0,
|
|
95
|
-
onProgress: (progress) => { live = progress; },
|
|
96
|
-
}, capture.io);
|
|
97
|
-
const parsed = parseDoctorBackfillJson(capture.stdout());
|
|
98
|
-
const counts = asRecord(parsed?.counts);
|
|
99
|
-
const count = (value) => typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null;
|
|
100
|
-
caughtUp += count(counts?.["backfilled"]) ?? 0;
|
|
101
|
-
remaining = count(counts?.["remaining"]) ?? remaining;
|
|
102
|
-
live = null;
|
|
103
|
-
if (code === 0 && parsed?.status === "complete") {
|
|
104
|
-
// Reporting totals include permanent caps. Preserve their marker
|
|
105
|
-
// note instead of retrying history the collector says is complete.
|
|
106
|
-
const checked = await checkBackfillState(context);
|
|
107
|
-
return ok("backfill-complete", checked.status === "ok" ? checked.code : "complete", receipt() + (checked.status === "ok" ? `; ${checked.message}` : ""));
|
|
108
|
-
}
|
|
109
|
-
const verdict = backfillFixVerdict(parsed, jsonField(capture.stderr(), "failure_reason"));
|
|
110
|
-
if (parsed?.status === "partial" &&
|
|
111
|
-
(parsed.failure_reason === "deferred_budget_exhausted" || !parsed.failure_reason)) {
|
|
112
|
-
return needsFix("backfill-complete", "backfill_chunk_pending", "more history remains");
|
|
113
|
-
}
|
|
114
|
-
return { ...verdict, message: typeof parsed?.failure_reason === "string" ? parsed.failure_reason : "backfill failed without a valid completion receipt" };
|
|
115
|
-
});
|
|
116
|
-
if (row.status === "ok") {
|
|
117
|
-
console.error("[doctor] backfill-complete finished", JSON.stringify({ caught_up: caughtUp, remaining, chunks, elapsed_seconds: Math.floor((Date.now() - started) / 1000) }));
|
|
118
|
-
return row;
|
|
119
|
-
}
|
|
120
|
-
if (row.code !== "backfill_chunk_pending") {
|
|
121
|
-
return unfinished(row.code === "lock_wait_timeout" && Date.now() >= deadline ? "backfill_budget_timeout" : row.code, row.message);
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
return unfinished("backfill_budget_timeout", "backfill time budget reached");
|
|
125
|
-
}
|
|
126
|
-
catch (error) {
|
|
127
|
-
console.error("[doctor] backfill-complete failed", JSON.stringify({ reason: "backfill_threw", error_name: error instanceof Error ? error.name : "unknown" }));
|
|
128
|
-
return unfinished("backfill_threw", "backfill failed");
|
|
129
|
-
}
|
|
130
|
-
finally {
|
|
131
|
-
clearInterval(progress);
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
32
|
export async function checkGcState(context) {
|
|
135
33
|
if (context.io.env["COCKPIT_DISABLE_GC"] === "1") {
|
|
136
34
|
return skipped("gc-checked", "skipped_disabled", "cleanup is switched off");
|
|
@@ -253,42 +151,6 @@ export async function fixDiskState(context) {
|
|
|
253
151
|
// Unaccepted evidence still occupies the cap. Keep the disk row actionable.
|
|
254
152
|
return needsFix("disk-bounded", "staging_cap_blocked_by_uncommitted", `freed ${mib(pruned.deleted_bytes)} MB; ${message}; ${pruned.cap_blocked_count} object(s) the upload ledger cannot vouch for are holding the rest; ${redelivered}; or \`cockpit clean --all-committed\` to drop every accepted copy now`);
|
|
255
153
|
}
|
|
256
|
-
function capturedIo(io, forward) {
|
|
257
|
-
const stdoutChunks = [];
|
|
258
|
-
const stderrChunks = [];
|
|
259
|
-
return {
|
|
260
|
-
io: {
|
|
261
|
-
...io,
|
|
262
|
-
stdout: captureStream(io.stdout, stdoutChunks, forward),
|
|
263
|
-
stderr: captureStream(io.stderr, stderrChunks, forward),
|
|
264
|
-
},
|
|
265
|
-
stdout: () => stdoutChunks.join(""),
|
|
266
|
-
stderr: () => stderrChunks.join(""),
|
|
267
|
-
};
|
|
268
|
-
}
|
|
269
|
-
function captureStream(target, chunks, forward) {
|
|
270
|
-
return {
|
|
271
|
-
write(chunk, encoding, callback) {
|
|
272
|
-
const text = Buffer.isBuffer(chunk) ? chunk.toString("utf8") : String(chunk);
|
|
273
|
-
chunks.push(text);
|
|
274
|
-
if (forward) {
|
|
275
|
-
if (typeof encoding === "function") {
|
|
276
|
-
target.write(chunk, encoding);
|
|
277
|
-
}
|
|
278
|
-
else {
|
|
279
|
-
target.write(chunk, encoding, callback);
|
|
280
|
-
}
|
|
281
|
-
}
|
|
282
|
-
else if (typeof encoding === "function") {
|
|
283
|
-
encoding();
|
|
284
|
-
}
|
|
285
|
-
else {
|
|
286
|
-
callback?.();
|
|
287
|
-
}
|
|
288
|
-
return true;
|
|
289
|
-
},
|
|
290
|
-
};
|
|
291
|
-
}
|
|
292
154
|
export async function checkSyncState(context) {
|
|
293
155
|
const roots = await doctorRoots(context);
|
|
294
156
|
if (roots.length === 0) {
|
|
@@ -299,8 +161,13 @@ export async function checkSyncState(context) {
|
|
|
299
161
|
// and only turns green from those command receipts.
|
|
300
162
|
return needsFix("sync-fresh", "per_root_verification_required", `fresh upload proof is required for ${roots.length} saved root${roots.length === 1 ? "" : "s"}`);
|
|
301
163
|
}
|
|
164
|
+
/**
|
|
165
|
+
* The `sync-fresh` repair. Looks at the collection lock first and never waits
|
|
166
|
+
* for it (BLI-12872): a running background sync is reported as exactly that,
|
|
167
|
+
* and a stuck one is ended before doctor's own sync runs.
|
|
168
|
+
*/
|
|
302
169
|
export async function fixSyncState(context) {
|
|
303
|
-
return
|
|
170
|
+
return withCollectionGate(context, "sync-fresh", () => runSyncRepair(context));
|
|
304
171
|
}
|
|
305
172
|
async function runSyncRepair(context) {
|
|
306
173
|
const exec = context.io.exec;
|
|
@@ -321,7 +188,10 @@ async function runSyncRepair(context) {
|
|
|
321
188
|
if (context.command.dashboardUrl !== DEFAULT_DASHBOARD_URL) {
|
|
322
189
|
args.push("--dashboard-url", context.command.dashboardUrl);
|
|
323
190
|
}
|
|
324
|
-
|
|
191
|
+
// One line every 30 s while doctor's own sync runs; the exec has nothing
|
|
192
|
+
// to count, so the line says it is still working and for how long.
|
|
193
|
+
const stopProgress = startProgress((text) => context.io.stderr.write(text), (elapsedMs) => waitingLine("sync", elapsedMs));
|
|
194
|
+
const result = await Promise.resolve(exec("cockpit", args)).finally(stopProgress);
|
|
325
195
|
const parsed = parseDoctorSyncJson(result.stdout);
|
|
326
196
|
const output = `${result.stdout}\n${result.stderr}`;
|
|
327
197
|
const status = jsonField(output, "status");
|
|
@@ -335,10 +205,12 @@ async function runSyncRepair(context) {
|
|
|
335
205
|
owner_alive: standAside.ownerAlive,
|
|
336
206
|
held_since: standAside.heldSince,
|
|
337
207
|
}));
|
|
338
|
-
|
|
208
|
+
// The gate (`withCollectionGate`) reads the lock again and turns this
|
|
209
|
+
// into "background sync is running"; the code only travels to it.
|
|
210
|
+
return needsFix("sync-fresh", standAside.code, "the sync stood aside for the collection lock");
|
|
339
211
|
}
|
|
340
212
|
if (status === "live_sync_paused_during_backfill") {
|
|
341
|
-
return needsFix("sync-fresh", status, "
|
|
213
|
+
return needsFix("sync-fresh", status, "the sync paused for the historical catch-up");
|
|
342
214
|
}
|
|
343
215
|
if (result.code !== 0) {
|
|
344
216
|
const draining = syncBacklogDrainingVerdict(parsed);
|
|
@@ -347,7 +219,7 @@ async function runSyncRepair(context) {
|
|
|
347
219
|
`file${draining.remainingObjects === 1 ? "" : "s"} left for later this run); ` +
|
|
348
220
|
"rerun `cockpit sync` to continue");
|
|
349
221
|
}
|
|
350
|
-
return fail("sync-fresh", status ?? "sync_failed", `${repoRoot}: ${syncFailureMessage(parsed)}`);
|
|
222
|
+
return fail("sync-fresh", status ?? "sync_failed", `${repoRoot}: ${syncFailureMessage(parsed, result.code)}`);
|
|
351
223
|
}
|
|
352
224
|
if (parsed?.collection_complete === false && typeof parsed.discovery_notice === "string") {
|
|
353
225
|
console.error("[doctor] partial discovery", JSON.stringify({ reason: "discovery_partial" }));
|
|
@@ -364,4 +236,5 @@ async function runSyncRepair(context) {
|
|
|
364
236
|
// Re-exported so `doctor.ts`, the doctor tests and every other caller keep
|
|
365
237
|
// importing these from `./doctor-pipeline.js` regardless of which sibling
|
|
366
238
|
// computes them.
|
|
239
|
+
export { checkBackfillState, fixBackfillState } from "./doctor-backfill.js";
|
|
367
240
|
export { backfillCompletionStepState, backfillFixVerdict, isCollectionBusyReason, syncBacklogDrainingVerdict, syncStandAsideVerdict, } from "./doctor-pipeline-verdicts.js";
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The one progress line `cockpit doctor` prints while a long repair runs
|
|
3
|
+
* (BLI-12872).
|
|
4
|
+
*
|
|
5
|
+
* Until now two writers shared stderr: the lock wait's "waiting for the
|
|
6
|
+
* background sync to finish" and the catch-up's "backfill: starting", and a
|
|
7
|
+
* 13-minute run printed that pair alternately about 40 times, which says
|
|
8
|
+
* nothing. Doctor no longer waits on a lock, so one writer is left, and its
|
|
9
|
+
* rule is: at most one line every 30 seconds, and each line says what CHANGED
|
|
10
|
+
* since the last one, or that nothing did.
|
|
11
|
+
*
|
|
12
|
+
* Counts and elapsed time only; never a path, an id or a session name.
|
|
13
|
+
*/
|
|
14
|
+
export const PROGRESS_EVERY_MS = 30_000;
|
|
15
|
+
/** "5m00s" for a progress line: seconds matter here, unlike a lock's age. */
|
|
16
|
+
export function clock(ms) {
|
|
17
|
+
const seconds = Math.floor(ms / 1000);
|
|
18
|
+
return `${Math.floor(seconds / 60)}m${String(seconds % 60).padStart(2, "0")}s`;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* The line for this tick. `last` is what the previous line reported (null for
|
|
22
|
+
* the first line), so the sentence is always a difference, never a repeat.
|
|
23
|
+
*/
|
|
24
|
+
export function backfillProgressLine(now, last, elapsedMs) {
|
|
25
|
+
if (now.remaining === null) {
|
|
26
|
+
return `backfill: reading your old sessions, ${clock(elapsedMs)}`;
|
|
27
|
+
}
|
|
28
|
+
const counts = `${now.caughtUp} caught up, ${now.remaining} to go`;
|
|
29
|
+
if (last === null || last.remaining === null) {
|
|
30
|
+
return `backfill: ${counts}, ${clock(elapsedMs)}`;
|
|
31
|
+
}
|
|
32
|
+
const gained = now.caughtUp - last.caughtUp;
|
|
33
|
+
if (gained <= 0 && now.remaining === last.remaining) {
|
|
34
|
+
return `backfill: no change in the last 30s (${counts}), ${clock(elapsedMs)}`;
|
|
35
|
+
}
|
|
36
|
+
return `backfill: +${Math.max(0, gained)} caught up in the last 30s (${counts}), ${clock(elapsedMs)}`;
|
|
37
|
+
}
|
|
38
|
+
/** The line for a repair that is one long command with nothing to count (`sync`). */
|
|
39
|
+
export function waitingLine(what, elapsedMs) {
|
|
40
|
+
return `${what}: still working, ${clock(elapsedMs)} so far`;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Print `describe(elapsedMs)` every 30 seconds until stopped. The timer never
|
|
44
|
+
* keeps the process alive. Returns the stop function.
|
|
45
|
+
*/
|
|
46
|
+
export function startProgress(write, describe, options = {}) {
|
|
47
|
+
const now = options.now ?? (() => Date.now());
|
|
48
|
+
const started = now();
|
|
49
|
+
const timer = (options.setIntervalImpl ?? setInterval)(() => {
|
|
50
|
+
write(`${describe(now() - started)}\n`);
|
|
51
|
+
}, PROGRESS_EVERY_MS);
|
|
52
|
+
timer.unref?.();
|
|
53
|
+
return () => (options.clearIntervalImpl ?? clearInterval)(timer);
|
|
54
|
+
}
|
|
@@ -157,7 +157,8 @@ export function repairReceiptLines(repairs) {
|
|
|
157
157
|
return [];
|
|
158
158
|
return [
|
|
159
159
|
"Repairs (before -> after):",
|
|
160
|
-
|
|
160
|
+
// BLI-12872: a row left "skipped" is a background sync doing the work; it is in progress, not waiting on a person.
|
|
161
|
+
...ran.map((repair) => ` ${repair.after?.startsWith("ok:") ? "fixed" : repair.after?.startsWith("skipped:") ? "in progress" : "still needs you"} ${repair.step}: ${repair.before_message ?? ""} -> ${repair.after_message ?? ""}`.replaceAll("—", ";")),
|
|
161
162
|
];
|
|
162
163
|
}
|
|
163
164
|
export function doctorNeedsPerson(rows, receipt) {
|
package/dist/commands/doctor.js
CHANGED
|
@@ -4,6 +4,7 @@ import { describeError } from "../health-detail.js";
|
|
|
4
4
|
import { withStage } from "../crash-guard.js";
|
|
5
5
|
import { checkSingleInstallState, readRootsCoverWork, fixAuthState, fixRootState, readAuthState, readRootState, } from "./doctor-access.js";
|
|
6
6
|
import { backfillCompletionStepState, backfillFixVerdict, checkBackfillState, checkDiskState, checkGcState, checkSyncState, fixBackfillState, fixDiskState, fixGcState, fixSyncState, syncBacklogDrainingVerdict, syncStandAsideVerdict, } from "./doctor-pipeline.js";
|
|
7
|
+
import { gateRow, readCollectionGate } from "./doctor-collection-lock.js";
|
|
7
8
|
import { checkMcpAnswersState, fixMcpAnswersState } from "./doctor-mcp.js";
|
|
8
9
|
import { budgetSpentRow, doctorRunDeadline, DOCTOR_JSON_BUDGET_SECONDS } from "./doctor-budget.js";
|
|
9
10
|
import { checkMemoryDaemonState } from "./doctor-memory-daemon.js";
|
|
@@ -49,15 +50,22 @@ export async function runDoctorWithDeps(command, io, deps) {
|
|
|
49
50
|
const prerequisitesReady = !needsCollection || rows.filter((prior) => prior.id === "authed" || prior.id === "roots-ok").every((prior) => prior.status === "ok");
|
|
50
51
|
const canFix = !diagnoseOnly && prerequisitesReady && invariant.fix &&
|
|
51
52
|
(!invariant.requiresInteractiveFix || isInteractiveDoctorFix(context) || Boolean(command.collectionRoots?.length));
|
|
52
|
-
|
|
53
|
+
let spent = row.status !== "ok" && row.status !== "skipped" && canFix ? budgetSpentRow(context, row) : null;
|
|
54
|
+
if (spent && needsCollection) {
|
|
55
|
+
// BLI-12872: a running sync is the answer even when the budget is spent; looking costs nothing.
|
|
56
|
+
const gate = await readCollectionGate(context, undefined, { mayEnd: false });
|
|
57
|
+
if (gate.kind === "running")
|
|
58
|
+
spent = gateRow(invariant.id, gate);
|
|
59
|
+
}
|
|
53
60
|
if (spent) {
|
|
54
61
|
row = spent;
|
|
55
|
-
repairs.push({ step: invariant.id, outcome: "needs_person", reason: row.code });
|
|
62
|
+
repairs.push({ step: invariant.id, outcome: row.status === "skipped" ? "not_needed" : "needs_person", reason: row.code });
|
|
56
63
|
}
|
|
57
64
|
else if (row.status !== "ok" && row.status !== "skipped" && canFix && invariant.fix) {
|
|
58
65
|
attempted.add(invariant.id);
|
|
59
66
|
row = await withStage(`doctor:fix:${invariant.id}`, () => invariant.fix(context, row));
|
|
60
|
-
|
|
67
|
+
// BLI-12872: a skipped row after a fix is a sync already doing the work, not a person's job.
|
|
68
|
+
repairs.push({ step: invariant.id, outcome: row.status === "ok" ? "repaired" : row.status === "skipped" ? "not_needed" : "needs_person", reason: row.code });
|
|
61
69
|
// The replacement process owns the one final receipt, including JSON.
|
|
62
70
|
if (row.reexecExitCode !== undefined)
|
|
63
71
|
return row.reexecExitCode;
|
|
@@ -3,28 +3,44 @@
|
|
|
3
3
|
*
|
|
4
4
|
* cockpit logs <source> [--since 15m|1h|2d|<iso>] [--raw <template-id>] [--limit <n>]
|
|
5
5
|
* [--all-levels] [--table <file>] [--json]
|
|
6
|
+
* cockpit logs <project> [--since 24h] [--json] (BLI-12855: stored digests)
|
|
7
|
+
* cockpit logs dashboard|railway --stored [--since 1h] (Tower's stored digests)
|
|
8
|
+
* cockpit logs --list (the projects Tower digests)
|
|
6
9
|
*
|
|
7
10
|
* <source> is `collector`, `dashboard`, `railway`, `railway:<service>` or `-`
|
|
8
|
-
* (lines on stdin)
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
+
* (lines on stdin): read-only, computed on this machine from the source's own CLI
|
|
12
|
+
* or files, nothing sent to Tower. Or <source> is a registry project id (`kolm`,
|
|
13
|
+
* `tower`...): then the digest is the one Tower's own cron stored, read through
|
|
14
|
+
* the dashboard's `/api/logs` door with this machine's device token, so it works
|
|
15
|
+
* with no `vercel` or `railway` login at all.
|
|
11
16
|
*/
|
|
12
|
-
import {
|
|
17
|
+
import { LOG_PROJECT_PATTERN } from "@bli-cockpit/telemetry-core/log-digest";
|
|
18
|
+
import { optionalNonEmpty, optionalPositiveInteger, optionalUrl, parseNamedArgs } from "./local-arg-values.js";
|
|
13
19
|
/** The source spellings, for the usage line and the refusal. `railway:<service>` takes any service name. */
|
|
14
|
-
export const LOGS_SOURCES = ["collector", "dashboard", "railway", "railway:<service>", "-"];
|
|
15
|
-
|
|
20
|
+
export const LOGS_SOURCES = ["collector", "dashboard", "railway", "railway:<service>", "-", "<project>"];
|
|
21
|
+
/** The spellings that read a local log; every other name is a registry project (stored digests). */
|
|
22
|
+
const LOCAL_SOURCE_PATTERN = /^(collector|dashboard|railway|railway:[A-Za-z0-9][A-Za-z0-9._-]{0,63}|-)$/;
|
|
16
23
|
const SINCE_PATTERN = /^(\d{1,4}[smhd]|\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}(:\d{2}(\.\d{1,9})?)?(Z|[+-]\d{2}:\d{2})?)?)$/;
|
|
17
24
|
export function parseLogsArgs(args) {
|
|
18
25
|
const values = parseNamedArgs(args, {
|
|
19
|
-
allowedFlags: ["--home", "--since", "--raw", "--limit", "--table", "--all-levels", "--json"],
|
|
20
|
-
valueFlags: ["--home", "--since", "--raw", "--limit", "--table"],
|
|
26
|
+
allowedFlags: ["--home", "--since", "--raw", "--limit", "--table", "--all-levels", "--stored", "--list", "--dashboard-url", "--json"],
|
|
27
|
+
valueFlags: ["--home", "--since", "--raw", "--limit", "--table", "--dashboard-url"],
|
|
21
28
|
});
|
|
22
|
-
const
|
|
23
|
-
|
|
24
|
-
|
|
29
|
+
const list = values.booleans.has("--list");
|
|
30
|
+
const source = values.positionals[0] ?? (list ? "tower" : undefined);
|
|
31
|
+
if (!source || (!LOCAL_SOURCE_PATTERN.test(source) && !LOG_PROJECT_PATTERN.test(source))) {
|
|
32
|
+
throw new Error(`logs needs a source: ${LOGS_SOURCES.join(", ")}. Example: \`cockpit logs dashboard --since 1h\` or \`cockpit logs kolm --since 24h\`.`);
|
|
25
33
|
}
|
|
26
34
|
if (values.positionals.length > 1)
|
|
27
35
|
throw new Error("logs takes one source; put the window in --since.");
|
|
36
|
+
const isProject = !LOCAL_SOURCE_PATTERN.test(source);
|
|
37
|
+
const stored = isProject || values.booleans.has("--stored");
|
|
38
|
+
if (values.booleans.has("--stored") && !["dashboard", "railway"].includes(source) && !isProject) {
|
|
39
|
+
throw new Error("--stored reads Tower's stored digests: use it with dashboard or railway, or name a project (`cockpit logs kolm`).");
|
|
40
|
+
}
|
|
41
|
+
if (stored && (values.flags.get("--raw") || values.flags.get("--table") || values.booleans.has("--all-levels"))) {
|
|
42
|
+
throw new Error("a stored digest keeps no raw line, and its classes were set when it was stored: --raw, --table and --all-levels read a local log.");
|
|
43
|
+
}
|
|
28
44
|
const since = optionalNonEmpty(values.flags.get("--since"));
|
|
29
45
|
if (since && !SINCE_PATTERN.test(since)) {
|
|
30
46
|
throw new Error(`--since wants 15m, 1h, 2d or an ISO time like 2026-10-07T03:00Z; got ${JSON.stringify(since.slice(0, 40))}.`);
|
|
@@ -47,6 +63,9 @@ export function parseLogsArgs(args) {
|
|
|
47
63
|
...(rawLimit === undefined ? {} : { rawLimit }),
|
|
48
64
|
...(values.flags.get("--table") ? { tableFile: values.flags.get("--table") } : {}),
|
|
49
65
|
allLevels: values.booleans.has("--all-levels"),
|
|
66
|
+
stored,
|
|
67
|
+
list,
|
|
68
|
+
dashboardUrl: optionalUrl(values.flags.get("--dashboard-url")),
|
|
50
69
|
homeDir: optionalNonEmpty(values.flags.get("--home")),
|
|
51
70
|
json: values.booleans.has("--json"),
|
|
52
71
|
};
|
|
@@ -13,9 +13,9 @@ export const TOWER_OPS_COMMAND_HELP = [
|
|
|
13
13
|
[
|
|
14
14
|
"logs",
|
|
15
15
|
[
|
|
16
|
-
"Usage: cockpit logs <collector|dashboard|railway|railway:<service
|
|
16
|
+
"Usage: cockpit logs <collector|dashboard|railway|railway:<service>|-|project> [--since 15m|1h|2d|<iso>] [--raw <template-id> [--limit <n>]] [--all-levels] [--table <file>] [--stored] [--list] [--json]",
|
|
17
17
|
"",
|
|
18
|
-
"A token-cheap digest of Tower's own logs, read on THIS machine (nothing is uploaded). Variable parts of",
|
|
18
|
+
"A token-cheap digest of Tower's own logs, read on THIS machine (nothing is uploaded) unless a project is named. Variable parts of",
|
|
19
19
|
"each line are masked into a template, equal templates are counted, routine runs become counts, and every",
|
|
20
20
|
"template is classified: our_bug and unknown first, then capacity, config_drift, rate_pushback, hiccup,",
|
|
21
21
|
"and one COUNTED line for dead_source and benign (counted, never alerted). About 2 KB.",
|
|
@@ -27,10 +27,18 @@ export const TOWER_OPS_COMMAND_HELP = [
|
|
|
27
27
|
" railway:<service> one service (ladder, runner, linear-sync, ...). Project and environment are always passed",
|
|
28
28
|
" explicitly: this repo's `railway link` points at the runner service.",
|
|
29
29
|
" - lines on stdin: plain text, `journalctl -o json`, `railway logs --json` or `vercel logs --json`.",
|
|
30
|
+
" <project> a BLI project (tower, kolm, portpal, pearl-portal, triage-crm, ...; `--list` names them): the digest Tower's",
|
|
31
|
+
" own 15-minute cron STORED from that project's Vercel, Railway and Supabase logs, merged over --since",
|
|
32
|
+
" (default 24h, at most 30d), real bugs first. Needs only this machine's Tower login: no vercel or railway",
|
|
33
|
+
" login. It keeps no raw line, so --raw, --table and --all-levels do not apply.",
|
|
34
|
+
"--stored dashboard or railway read Tower's stored digest of project `tower` (this also happens by itself, and says so,",
|
|
35
|
+
" when the local vercel or railway CLI is missing or logged out).",
|
|
36
|
+
"--list every project, every source, and which are wired or not_wired (with the reason).",
|
|
30
37
|
"--raw <template-id> the original lines behind one template id from a digest (redacted, newest last; --limit n, default 20).",
|
|
31
38
|
"--table <file> a class table JSON (same shape as the packaged one) to try a new pattern before committing it.",
|
|
32
39
|
"A source that cannot be read is a named refusal, never a stack trace: vercel_cli_missing, vercel_not_logged_in,",
|
|
33
|
-
"railway_cli_missing, railway_not_logged_in, railway_service_unknown, collector_logs_missing, stdin_is_terminal,
|
|
40
|
+
"railway_cli_missing, railway_not_logged_in, railway_service_unknown, collector_logs_missing, stdin_is_terminal, and for a",
|
|
41
|
+
"project unknown_project, since_exceeds_retention, stored_read_failed, not_paired, ...",
|
|
34
42
|
"--json writes one object to stdout: { ok, kind, source, window, lines, templates[], routine, notes, text, measure };",
|
|
35
43
|
"a refusal is { ok: false, error, detail } and exits 1. Run list and teaching: docs/runbooks/tower-logs.md.",
|
|
36
44
|
"The bli-tower MCP server has the twin: logs_digest.",
|
|
@@ -94,7 +94,7 @@ export function localSubcommandHelp(command) {
|
|
|
94
94
|
"roots-cover-work is informational: it names how many sessions are inside vs. outside saved roots (all already uploaded, outside ones labelled) and gives cockpit roots add commands for widening a root's redacted transcript content. It never needs a fix.",
|
|
95
95
|
"--check or --no-repair diagnoses without repairs. --dry-run previews repairs.",
|
|
96
96
|
"--backfill-budget defaults to 900 seconds; catch-up repeats chunks until complete, with progress every 30 seconds.",
|
|
97
|
-
"--lock-wait
|
|
97
|
+
"Doctor never waits for the background sync: a running sync is reported as running (with how long ago it started) and doctor finishes; a sync that stopped responding or made no progress for 15 minutes is ended and doctor carries on. --lock-wait is accepted and changes nothing. Progress prints at most every 30 seconds and says what changed.",
|
|
98
98
|
"--json without either flag answers within 45 seconds; a repair that does not fit reads doctor_budget_spent.",
|
|
99
99
|
"--json includes steps, repairs, and needs_person. Exit 0 means nothing needs you.",
|
|
100
100
|
"Maintainers: --update-tag next keeps self-update on the prerelease candidate.",
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { askAgentDoor, emitAgentDoor, failAgentDoor, openAgentDoor } from "./agent-door.js";
|
|
2
|
+
import { writeLine } from "./cli-io.js";
|
|
3
|
+
const TAG = "[logs cli]";
|
|
4
|
+
const READ_DEADLINE_MS = 55_000;
|
|
5
|
+
/** The project a stored read is for: the named project, or `tower` for `dashboard` and `railway`. */
|
|
6
|
+
export function storedProject(command) {
|
|
7
|
+
if (!command.stored)
|
|
8
|
+
return null;
|
|
9
|
+
return command.source === "dashboard" || command.source === "railway" ? "tower" : command.source;
|
|
10
|
+
}
|
|
11
|
+
export async function runStoredLogs(command, io, options) {
|
|
12
|
+
let door;
|
|
13
|
+
try {
|
|
14
|
+
door = await openAgentDoor("logs", command, io);
|
|
15
|
+
}
|
|
16
|
+
catch (error) {
|
|
17
|
+
const detail = error instanceof Error ? error.message : "this machine is not paired with Tower";
|
|
18
|
+
if (options.onNotPaired)
|
|
19
|
+
return options.onNotPaired(detail);
|
|
20
|
+
writeLine(io.stderr, `${TAG} refused ${JSON.stringify({ reason: "not_paired" })}`);
|
|
21
|
+
if (command.json)
|
|
22
|
+
writeLine(io.stdout, JSON.stringify({ ok: false, error: "not_paired", detail }));
|
|
23
|
+
else
|
|
24
|
+
writeLine(io.stderr, detail);
|
|
25
|
+
return 1;
|
|
26
|
+
}
|
|
27
|
+
if (command.list) {
|
|
28
|
+
const answer = await askAgentDoor(door, { path: "/api/logs?list=1", method: "GET", label: "logs list", timeoutMs: READ_DEADLINE_MS });
|
|
29
|
+
if (!answer.ok)
|
|
30
|
+
return failAgentDoor(door, TAG, answer.reason, answer.detail);
|
|
31
|
+
const body = answer.body;
|
|
32
|
+
if (command.json)
|
|
33
|
+
return emitAgentDoor(door, body);
|
|
34
|
+
for (const project of body.projects) {
|
|
35
|
+
writeLine(io.stdout, `${project.id} ${project.name}`);
|
|
36
|
+
for (const source of project.sources) {
|
|
37
|
+
writeLine(io.stdout, ` ${source.status === "wired" ? "wired " : "not_wired"} ${source.source}${source.reason ? `: ${source.reason}` : ""}`);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return 0;
|
|
41
|
+
}
|
|
42
|
+
const query = new URLSearchParams({ project: options.project });
|
|
43
|
+
if (command.since)
|
|
44
|
+
query.set("since", command.since);
|
|
45
|
+
const answer = await askAgentDoor(door, { path: `/api/logs?${query.toString()}`, method: "GET", label: "logs", timeoutMs: READ_DEADLINE_MS });
|
|
46
|
+
if (!answer.ok)
|
|
47
|
+
return failAgentDoor(door, TAG, answer.reason, answer.detail);
|
|
48
|
+
const body = answer.body;
|
|
49
|
+
writeLine(io.stderr, `${TAG} stored digest read ${JSON.stringify({ project: body.project, windows: body.windows, windows_ok: body.windows_ok, refusals: body.refusals.length, still_refusing: body.still_refusing?.length ?? null, not_wired: body.not_wired?.length ?? null, fallback: Boolean(options.fallbackReason) })}`);
|
|
50
|
+
if (command.json)
|
|
51
|
+
return emitAgentDoor(door, options.fallbackReason ? { ...body, fallback_from: options.fallbackReason } : body);
|
|
52
|
+
if (options.fallbackReason)
|
|
53
|
+
writeLine(io.stdout, `(no local ${command.source} access: ${options.fallbackReason}; this is the digest Tower stored)`);
|
|
54
|
+
writeLine(io.stdout, body.text);
|
|
55
|
+
return 0;
|
|
56
|
+
}
|
package/dist/commands/logs.js
CHANGED
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
* Read-only and local: the source's own CLI or files are read on THIS machine,
|
|
5
5
|
* masked into templates, counted and classified by `@bli-cockpit/telemetry-core/log-digest`,
|
|
6
6
|
* and printed in about 2 KB with real bugs (`our_bug`, `unknown`) first. Nothing
|
|
7
|
-
* is uploaded and no Tower door is called
|
|
7
|
+
* is uploaded and no Tower door is called (the stored half, `logs-stored.ts`,
|
|
8
|
+
* is the one exception: a project name reads what Tower's cron stored). The MCP server's `logs_digest` tool
|
|
8
9
|
* calls the same `digestSource`, so the two print the same thing.
|
|
9
10
|
*
|
|
10
11
|
* A source that cannot be read is a NAMED refusal (`vercel_cli_missing`,
|
|
@@ -12,9 +13,10 @@
|
|
|
12
13
|
* detail }` on stdout under `--json`, exit 1. Never a stack trace.
|
|
13
14
|
*/
|
|
14
15
|
import { readFile } from "node:fs/promises";
|
|
15
|
-
import { ClassTable, digestSource } from "@bli-cockpit/telemetry-core/log-digest";
|
|
16
|
+
import { ClassTable, LOCAL_ACCESS_REFUSALS, digestSource } from "@bli-cockpit/telemetry-core/log-digest";
|
|
16
17
|
import { getCollectorRuntimePaths } from "../local-state-paths.js";
|
|
17
18
|
import { isInteractiveStdin, readPipedText, writeLine } from "./cli-io.js";
|
|
19
|
+
import { runStoredLogs, storedProject } from "./logs-stored.js";
|
|
18
20
|
const TAG = "[logs cli]";
|
|
19
21
|
const MAX_STDIN_CHARS = 64 * 1024 * 1024;
|
|
20
22
|
function refuse(io, json, reason, detail) {
|
|
@@ -26,6 +28,10 @@ function refuse(io, json, reason, detail) {
|
|
|
26
28
|
return 1;
|
|
27
29
|
}
|
|
28
30
|
export async function runLogs(command, io, deps = {}) {
|
|
31
|
+
// BLI-12855: a project name (or --stored, or --list) reads the digests Tower's cron stored, not a local log.
|
|
32
|
+
const stored = storedProject(command);
|
|
33
|
+
if (command.list || stored)
|
|
34
|
+
return runStoredLogs(command, io, { project: stored ?? "tower" });
|
|
29
35
|
const paths = getCollectorRuntimePaths(command.homeDir);
|
|
30
36
|
let table;
|
|
31
37
|
if (command.tableFile) {
|
|
@@ -61,6 +67,16 @@ export async function runLogs(command, io, deps = {}) {
|
|
|
61
67
|
nowMs: deps.nowMs,
|
|
62
68
|
log: (message, fields) => writeLine(io.stderr, `${message} ${JSON.stringify(fields)}`),
|
|
63
69
|
});
|
|
70
|
+
if (!outcome.ok && LOCAL_ACCESS_REFUSALS.has(outcome.reason) && (command.source === "dashboard" || command.source === "railway") && !command.raw) {
|
|
71
|
+
// No local access to the platform: say so, then answer from what the cron stored (a refusal of its own if this machine is not paired).
|
|
72
|
+
writeLine(io.stderr, `${TAG} local source unreadable, reading the stored digest ${JSON.stringify({ reason: outcome.reason })}`);
|
|
73
|
+
return runStoredLogs(command, io, {
|
|
74
|
+
project: "tower",
|
|
75
|
+
fallbackReason: outcome.reason,
|
|
76
|
+
// The person asked for a local read: if the stored digest is out of reach too, the local refusal is the answer.
|
|
77
|
+
onNotPaired: () => refuse(io, command.json, outcome.reason, `${outcome.detail} This machine is not paired with Tower (cockpit login), so the stored digest could not be read either.`),
|
|
78
|
+
});
|
|
79
|
+
}
|
|
64
80
|
if (!outcome.ok)
|
|
65
81
|
return refuse(io, command.json, outcome.reason, outcome.detail);
|
|
66
82
|
writeLine(io.stdout, command.json ? JSON.stringify(outcome) : outcome.text);
|
|
@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
24
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
24
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.179");
|
|
25
25
|
return 0;
|
|
26
26
|
}
|
|
27
27
|
|
|
@@ -7,6 +7,7 @@ import { healStubsDuringTick } from "./sync-stub-heal.js";
|
|
|
7
7
|
import { backfillWaitingCheck } from "./sync-yield.js";
|
|
8
8
|
import { getCollectorRuntimePaths } from "../local-state.js";
|
|
9
9
|
import { rawEvidenceDedupSummary, sweepDuplicateStagedRawEvidence, } from "../raw-evidence-gc.js";
|
|
10
|
+
import { noteSyncProgress } from "../sync-lock.js";
|
|
10
11
|
/**
|
|
11
12
|
* Collect everything this tick may collect, then report it.
|
|
12
13
|
*
|
|
@@ -25,6 +26,7 @@ export async function runSyncLocked(command, io) {
|
|
|
25
26
|
if (!dedup.skipped && dedup.removed_dirs > 0) {
|
|
26
27
|
writeLine(io.stdout, rawEvidenceDedupSummary(dedup));
|
|
27
28
|
}
|
|
29
|
+
noteSyncProgress(); // BLI-12872: the sweep finished; the lock's progress mark
|
|
28
30
|
let discoveryNotice = null;
|
|
29
31
|
const worktrees = await discoverCommandWorktrees(collectionRoots, {
|
|
30
32
|
maxDepth: command.maxDepth,
|
|
@@ -33,6 +35,7 @@ export async function runSyncLocked(command, io) {
|
|
|
33
35
|
allowEmpty: true,
|
|
34
36
|
onPartial: (notice) => { discoveryNotice = notice; },
|
|
35
37
|
}, io);
|
|
38
|
+
noteSyncProgress(); // BLI-12872: discovery finished
|
|
36
39
|
const run = await runAttributedWorktreeSync({
|
|
37
40
|
homeDir: command.homeDir,
|
|
38
41
|
dashboardUrl: command.dashboardUrl,
|
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
* worktree's upload, which is why each one logs its own elapsed time.
|
|
16
16
|
*/
|
|
17
17
|
import { inspectBackfillLock } from "../backfill-lock.js";
|
|
18
|
+
import { noteSyncProgress } from "../sync-lock.js";
|
|
18
19
|
/** The waiter's flag: the backfill lock is held while its owner waits for ours. */
|
|
19
20
|
export function backfillWaitingCheck(paths, inspect = inspectBackfillLock) {
|
|
20
21
|
return async () => {
|
|
@@ -50,6 +51,9 @@ export async function runYieldingAtBoundaries(options) {
|
|
|
50
51
|
const itemStarted = now();
|
|
51
52
|
const result = await options.runOne(item);
|
|
52
53
|
results.push(result);
|
|
54
|
+
// BLI-12872: a finished worktree is the lock's progress mark, what keeps a
|
|
55
|
+
// long tick from being called stuck while it keeps working.
|
|
56
|
+
noteSyncProgress();
|
|
53
57
|
console.error("[session-sync] worktree synced", JSON.stringify({
|
|
54
58
|
index: index + 1,
|
|
55
59
|
total: options.items.length,
|