@bli-cockpit/cli 0.2.174 → 0.2.180
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/backfill-lock-wait.js +4 -1
- package/dist/commands/brain.js +35 -0
- package/dist/commands/doctor-backfill.js +160 -0
- package/dist/commands/doctor-budget.js +14 -17
- package/dist/commands/doctor-collection-lock.js +201 -0
- package/dist/commands/doctor-disk-words.js +34 -3
- package/dist/commands/doctor-pipeline-verdicts.js +13 -3
- package/dist/commands/doctor-pipeline.js +22 -149
- package/dist/commands/doctor-progress.js +54 -0
- package/dist/commands/doctor-report.js +2 -1
- package/dist/commands/doctor.js +11 -3
- package/dist/commands/local-args-tower-brain.js +38 -0
- package/dist/commands/local-args-tower.js +1 -0
- package/dist/commands/local-args.js +3 -1
- package/dist/commands/local-command-shapes-brain.js +5 -0
- package/dist/commands/local-help-commands-tower-ops.js +17 -0
- package/dist/commands/local-help-commands.js +1 -1
- package/dist/commands/local-help.js +2 -0
- package/dist/commands/local.js +3 -0
- package/dist/commands/logs-stored.js +1 -1
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/status.js +1 -1
- package/dist/commands/sync-run.js +3 -0
- package/dist/commands/sync-yield.js +4 -0
- package/dist/commands/sync.js +30 -11
- package/dist/evidence-upload-transport.js +4 -0
- package/dist/local-state-status.js +5 -1
- package/dist/raw-evidence-stuck-summary.js +21 -1
- package/dist/state-dir-audit.js +32 -1
- package/dist/sync-lock-holder.js +82 -0
- package/dist/sync-lock.js +111 -13
- package/dist/sync-tick-timing.js +34 -0
- package/package.json +4 -4
- package/dist/commands/doctor-lock-wait.js +0 -55
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { doctorBackfillBudgetSeconds, doctorLockWaitSeconds } from "./doctor-budget.js";
|
|
3
|
-
import { inspectBackfillLock } from "../backfill-lock.js";
|
|
4
|
-
import { readBackfillCompletionMarker, readBackfillCursor, } from "../cursors/backfill-cursor.js";
|
|
1
|
+
import { withCollectionGate } from "./doctor-collection-lock.js";
|
|
5
2
|
import { savedDiscoveryLimitArgs } from "../discovery-limits.js";
|
|
6
3
|
import { runStagingPrune } from "../disk-prune.js";
|
|
7
4
|
import { retentionOptionsFromEnv } from "../disk-retention.js";
|
|
@@ -10,11 +7,11 @@ import { readDiskFootprint } from "../disk-usage.js";
|
|
|
10
7
|
import { runEvidenceRedelivery } from "../evidence-redelivery.js";
|
|
11
8
|
import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths } from "../local-state.js";
|
|
12
9
|
import { runRawEvidenceLocalGc, rawEvidenceGcSummary } from "../raw-evidence-gc.js";
|
|
13
|
-
import { runBackfillCommand } from "./backfill.js";
|
|
14
10
|
import { FOOTPRINT_NEEDS_FIX_BYTES, stagingPausedWords, diskRowMessage, footprintHolder, footprintTotalBytes, mib, redeliveryLine, } from "./doctor-disk-words.js";
|
|
15
11
|
import { doctorRoots } from "./doctor-access.js";
|
|
16
|
-
import {
|
|
17
|
-
import {
|
|
12
|
+
import { startProgress, waitingLine } from "./doctor-progress.js";
|
|
13
|
+
import { jsonField, parseDoctorSyncJson, syncFailureMessage, syncBacklogDrainingVerdict, syncStandAsideVerdict, } from "./doctor-pipeline-verdicts.js";
|
|
14
|
+
import { fail, needsFix, ok, skipped } from "./doctor-report.js";
|
|
18
15
|
/**
|
|
19
16
|
* The `backfill-complete`, `gc-checked`, `disk-bounded`, and `sync-fresh`
|
|
20
17
|
* check family: does the collection pipeline itself have everything it
|
|
@@ -29,108 +26,9 @@ import { asRecord, fail, needsFix, ok, skipped } from "./doctor-report.js";
|
|
|
29
26
|
* completion marker proves, what a stand-aside receipt means, when an
|
|
30
27
|
* unfinished backfill is busy rather than broken — are pure functions in
|
|
31
28
|
* `doctor-pipeline-verdicts.ts`, and every one of them is re-exported from
|
|
32
|
-
* here so callers keep one address.
|
|
29
|
+
* here so callers keep one address. The catch-up row itself moved to
|
|
30
|
+
* `doctor-backfill.ts` (BLI-12872) and is re-exported below as well.
|
|
33
31
|
*/
|
|
34
|
-
export async function checkBackfillState(context) {
|
|
35
|
-
const paths = getCollectorRuntimePaths(context.command.homeDir);
|
|
36
|
-
const roots = await doctorRoots(context);
|
|
37
|
-
const marker = await readBackfillCompletionMarker(paths);
|
|
38
|
-
const covered = backfillCompletionStepState(marker, roots);
|
|
39
|
-
if (covered)
|
|
40
|
-
return covered;
|
|
41
|
-
const lock = await inspectBackfillLock(paths);
|
|
42
|
-
if (lock.held) {
|
|
43
|
-
return needsFix("backfill-complete", "backfill_already_running", `another catch-up run is still going, and this one has not finished yet (running since ${lock.held_since ?? "unknown"})`);
|
|
44
|
-
}
|
|
45
|
-
const cursor = await readBackfillCursor(paths);
|
|
46
|
-
return needsFix("backfill-complete", cursor.updated_at ? "partial" : "never_run", "the catch-up over your old sessions has not finished");
|
|
47
|
-
}
|
|
48
|
-
export async function fixBackfillState(context) {
|
|
49
|
-
const started = Date.now();
|
|
50
|
-
const deadline = started + doctorBackfillBudgetSeconds(context, started) * 1000;
|
|
51
|
-
let caughtUp = 0;
|
|
52
|
-
let chunks = 0;
|
|
53
|
-
let live = null;
|
|
54
|
-
const elapsed = () => {
|
|
55
|
-
const seconds = Math.floor((Date.now() - started) / 1000);
|
|
56
|
-
return `${Math.floor(seconds / 60)}m${String(seconds % 60).padStart(2, "0")}s`;
|
|
57
|
-
};
|
|
58
|
-
const receipt = () => `caught up on ${caughtUp} old sessions in ${chunks} chunk${chunks === 1 ? "" : "s"} over ${elapsed()}`;
|
|
59
|
-
let remaining = null;
|
|
60
|
-
const unfinished = (code, reason) => ({
|
|
61
|
-
...needsFix("backfill-complete", code, `${receipt()}, ${remaining ?? "unknown"} still to go; ${reason}; run \`cockpit doctor\` again to continue`),
|
|
62
|
-
nextAction: "cockpit doctor",
|
|
63
|
-
});
|
|
64
|
-
const progress = setInterval(() => {
|
|
65
|
-
const seconds = Math.floor((Date.now() - started) / 1000);
|
|
66
|
-
const uploaded = live === null ? null : caughtUp + live.uploaded;
|
|
67
|
-
context.io.stderr.write(live === null
|
|
68
|
-
? `backfill: starting, ${elapsed()}\n`
|
|
69
|
-
: `backfill: ${uploaded} caught up, ${live.processed} processed in this chunk, ${live.remaining} to go, ${elapsed()}\n`);
|
|
70
|
-
console.error("[doctor] backfill-complete progress", JSON.stringify({
|
|
71
|
-
status: live === null ? "starting" : "running", processed: live?.processed ?? null,
|
|
72
|
-
uploaded, caught_up: uploaded, remaining: live?.remaining ?? null,
|
|
73
|
-
chunks, elapsed_seconds: seconds,
|
|
74
|
-
}));
|
|
75
|
-
}, 30_000);
|
|
76
|
-
try {
|
|
77
|
-
while (Date.now() < deadline) {
|
|
78
|
-
// Every chunk reacquires the collector's locks. Bound lock waiting by
|
|
79
|
-
// the overall budget too; never abandon an in-flight cursor write.
|
|
80
|
-
const chunkContext = { ...context, command: { ...context.command,
|
|
81
|
-
lockWaitSeconds: Math.min(doctorLockWaitSeconds(context), (deadline - Date.now()) / 1000),
|
|
82
|
-
} };
|
|
83
|
-
const row = await withDoctorLockWait(chunkContext, async () => {
|
|
84
|
-
if (Date.now() >= deadline)
|
|
85
|
-
return unfinished("backfill_budget_timeout", "backfill time budget reached");
|
|
86
|
-
const capture = capturedIo(context.io, false);
|
|
87
|
-
live = null;
|
|
88
|
-
chunks += 1;
|
|
89
|
-
const code = await runBackfillCommand({
|
|
90
|
-
homeDir: context.command.homeDir, repoRoot: context.command.repoRoot,
|
|
91
|
-
all: true, dryRun: false, yes: true, json: true,
|
|
92
|
-
deadlineMs: deadline,
|
|
93
|
-
// Doctor's own `withDoctorLockWait` owns the waiting (BLI-11091).
|
|
94
|
-
syncLockWaitMs: 0,
|
|
95
|
-
onProgress: (progress) => { live = progress; },
|
|
96
|
-
}, capture.io);
|
|
97
|
-
const parsed = parseDoctorBackfillJson(capture.stdout());
|
|
98
|
-
const counts = asRecord(parsed?.counts);
|
|
99
|
-
const count = (value) => typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null;
|
|
100
|
-
caughtUp += count(counts?.["backfilled"]) ?? 0;
|
|
101
|
-
remaining = count(counts?.["remaining"]) ?? remaining;
|
|
102
|
-
live = null;
|
|
103
|
-
if (code === 0 && parsed?.status === "complete") {
|
|
104
|
-
// Reporting totals include permanent caps. Preserve their marker
|
|
105
|
-
// note instead of retrying history the collector says is complete.
|
|
106
|
-
const checked = await checkBackfillState(context);
|
|
107
|
-
return ok("backfill-complete", checked.status === "ok" ? checked.code : "complete", receipt() + (checked.status === "ok" ? `; ${checked.message}` : ""));
|
|
108
|
-
}
|
|
109
|
-
const verdict = backfillFixVerdict(parsed, jsonField(capture.stderr(), "failure_reason"));
|
|
110
|
-
if (parsed?.status === "partial" &&
|
|
111
|
-
(parsed.failure_reason === "deferred_budget_exhausted" || !parsed.failure_reason)) {
|
|
112
|
-
return needsFix("backfill-complete", "backfill_chunk_pending", "more history remains");
|
|
113
|
-
}
|
|
114
|
-
return { ...verdict, message: typeof parsed?.failure_reason === "string" ? parsed.failure_reason : "backfill failed without a valid completion receipt" };
|
|
115
|
-
});
|
|
116
|
-
if (row.status === "ok") {
|
|
117
|
-
console.error("[doctor] backfill-complete finished", JSON.stringify({ caught_up: caughtUp, remaining, chunks, elapsed_seconds: Math.floor((Date.now() - started) / 1000) }));
|
|
118
|
-
return row;
|
|
119
|
-
}
|
|
120
|
-
if (row.code !== "backfill_chunk_pending") {
|
|
121
|
-
return unfinished(row.code === "lock_wait_timeout" && Date.now() >= deadline ? "backfill_budget_timeout" : row.code, row.message);
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
return unfinished("backfill_budget_timeout", "backfill time budget reached");
|
|
125
|
-
}
|
|
126
|
-
catch (error) {
|
|
127
|
-
console.error("[doctor] backfill-complete failed", JSON.stringify({ reason: "backfill_threw", error_name: error instanceof Error ? error.name : "unknown" }));
|
|
128
|
-
return unfinished("backfill_threw", "backfill failed");
|
|
129
|
-
}
|
|
130
|
-
finally {
|
|
131
|
-
clearInterval(progress);
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
32
|
export async function checkGcState(context) {
|
|
135
33
|
if (context.io.env["COCKPIT_DISABLE_GC"] === "1") {
|
|
136
34
|
return skipped("gc-checked", "skipped_disabled", "cleanup is switched off");
|
|
@@ -253,42 +151,6 @@ export async function fixDiskState(context) {
|
|
|
253
151
|
// Unaccepted evidence still occupies the cap. Keep the disk row actionable.
|
|
254
152
|
return needsFix("disk-bounded", "staging_cap_blocked_by_uncommitted", `freed ${mib(pruned.deleted_bytes)} MB; ${message}; ${pruned.cap_blocked_count} object(s) the upload ledger cannot vouch for are holding the rest; ${redelivered}; or \`cockpit clean --all-committed\` to drop every accepted copy now`);
|
|
255
153
|
}
|
|
256
|
-
function capturedIo(io, forward) {
|
|
257
|
-
const stdoutChunks = [];
|
|
258
|
-
const stderrChunks = [];
|
|
259
|
-
return {
|
|
260
|
-
io: {
|
|
261
|
-
...io,
|
|
262
|
-
stdout: captureStream(io.stdout, stdoutChunks, forward),
|
|
263
|
-
stderr: captureStream(io.stderr, stderrChunks, forward),
|
|
264
|
-
},
|
|
265
|
-
stdout: () => stdoutChunks.join(""),
|
|
266
|
-
stderr: () => stderrChunks.join(""),
|
|
267
|
-
};
|
|
268
|
-
}
|
|
269
|
-
function captureStream(target, chunks, forward) {
|
|
270
|
-
return {
|
|
271
|
-
write(chunk, encoding, callback) {
|
|
272
|
-
const text = Buffer.isBuffer(chunk) ? chunk.toString("utf8") : String(chunk);
|
|
273
|
-
chunks.push(text);
|
|
274
|
-
if (forward) {
|
|
275
|
-
if (typeof encoding === "function") {
|
|
276
|
-
target.write(chunk, encoding);
|
|
277
|
-
}
|
|
278
|
-
else {
|
|
279
|
-
target.write(chunk, encoding, callback);
|
|
280
|
-
}
|
|
281
|
-
}
|
|
282
|
-
else if (typeof encoding === "function") {
|
|
283
|
-
encoding();
|
|
284
|
-
}
|
|
285
|
-
else {
|
|
286
|
-
callback?.();
|
|
287
|
-
}
|
|
288
|
-
return true;
|
|
289
|
-
},
|
|
290
|
-
};
|
|
291
|
-
}
|
|
292
154
|
export async function checkSyncState(context) {
|
|
293
155
|
const roots = await doctorRoots(context);
|
|
294
156
|
if (roots.length === 0) {
|
|
@@ -299,8 +161,13 @@ export async function checkSyncState(context) {
|
|
|
299
161
|
// and only turns green from those command receipts.
|
|
300
162
|
return needsFix("sync-fresh", "per_root_verification_required", `fresh upload proof is required for ${roots.length} saved root${roots.length === 1 ? "" : "s"}`);
|
|
301
163
|
}
|
|
164
|
+
/**
|
|
165
|
+
* The `sync-fresh` repair. Looks at the collection lock first and never waits
|
|
166
|
+
* for it (BLI-12872): a running background sync is reported as exactly that,
|
|
167
|
+
* and a stuck one is ended before doctor's own sync runs.
|
|
168
|
+
*/
|
|
302
169
|
export async function fixSyncState(context) {
|
|
303
|
-
return
|
|
170
|
+
return withCollectionGate(context, "sync-fresh", () => runSyncRepair(context));
|
|
304
171
|
}
|
|
305
172
|
async function runSyncRepair(context) {
|
|
306
173
|
const exec = context.io.exec;
|
|
@@ -321,7 +188,10 @@ async function runSyncRepair(context) {
|
|
|
321
188
|
if (context.command.dashboardUrl !== DEFAULT_DASHBOARD_URL) {
|
|
322
189
|
args.push("--dashboard-url", context.command.dashboardUrl);
|
|
323
190
|
}
|
|
324
|
-
|
|
191
|
+
// One line every 30 s while doctor's own sync runs; the exec has nothing
|
|
192
|
+
// to count, so the line says it is still working and for how long.
|
|
193
|
+
const stopProgress = startProgress((text) => context.io.stderr.write(text), (elapsedMs) => waitingLine("sync", elapsedMs));
|
|
194
|
+
const result = await Promise.resolve(exec("cockpit", args)).finally(stopProgress);
|
|
325
195
|
const parsed = parseDoctorSyncJson(result.stdout);
|
|
326
196
|
const output = `${result.stdout}\n${result.stderr}`;
|
|
327
197
|
const status = jsonField(output, "status");
|
|
@@ -335,10 +205,12 @@ async function runSyncRepair(context) {
|
|
|
335
205
|
owner_alive: standAside.ownerAlive,
|
|
336
206
|
held_since: standAside.heldSince,
|
|
337
207
|
}));
|
|
338
|
-
|
|
208
|
+
// The gate (`withCollectionGate`) reads the lock again and turns this
|
|
209
|
+
// into "background sync is running"; the code only travels to it.
|
|
210
|
+
return needsFix("sync-fresh", standAside.code, "the sync stood aside for the collection lock");
|
|
339
211
|
}
|
|
340
212
|
if (status === "live_sync_paused_during_backfill") {
|
|
341
|
-
return needsFix("sync-fresh", status, "
|
|
213
|
+
return needsFix("sync-fresh", status, "the sync paused for the historical catch-up");
|
|
342
214
|
}
|
|
343
215
|
if (result.code !== 0) {
|
|
344
216
|
const draining = syncBacklogDrainingVerdict(parsed);
|
|
@@ -347,7 +219,7 @@ async function runSyncRepair(context) {
|
|
|
347
219
|
`file${draining.remainingObjects === 1 ? "" : "s"} left for later this run); ` +
|
|
348
220
|
"rerun `cockpit sync` to continue");
|
|
349
221
|
}
|
|
350
|
-
return fail("sync-fresh", status ?? "sync_failed", `${repoRoot}: ${syncFailureMessage(parsed)}`);
|
|
222
|
+
return fail("sync-fresh", status ?? "sync_failed", `${repoRoot}: ${syncFailureMessage(parsed, result.code)}`);
|
|
351
223
|
}
|
|
352
224
|
if (parsed?.collection_complete === false && typeof parsed.discovery_notice === "string") {
|
|
353
225
|
console.error("[doctor] partial discovery", JSON.stringify({ reason: "discovery_partial" }));
|
|
@@ -364,4 +236,5 @@ async function runSyncRepair(context) {
|
|
|
364
236
|
// Re-exported so `doctor.ts`, the doctor tests and every other caller keep
|
|
365
237
|
// importing these from `./doctor-pipeline.js` regardless of which sibling
|
|
366
238
|
// computes them.
|
|
239
|
+
export { checkBackfillState, fixBackfillState } from "./doctor-backfill.js";
|
|
367
240
|
export { backfillCompletionStepState, backfillFixVerdict, isCollectionBusyReason, syncBacklogDrainingVerdict, syncStandAsideVerdict, } from "./doctor-pipeline-verdicts.js";
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The one progress line `cockpit doctor` prints while a long repair runs
|
|
3
|
+
* (BLI-12872).
|
|
4
|
+
*
|
|
5
|
+
* Until now two writers shared stderr: the lock wait's "waiting for the
|
|
6
|
+
* background sync to finish" and the catch-up's "backfill: starting", and a
|
|
7
|
+
* 13-minute run printed that pair alternately about 40 times, which says
|
|
8
|
+
* nothing. Doctor no longer waits on a lock, so one writer is left, and its
|
|
9
|
+
* rule is: at most one line every 30 seconds, and each line says what CHANGED
|
|
10
|
+
* since the last one, or that nothing did.
|
|
11
|
+
*
|
|
12
|
+
* Counts and elapsed time only; never a path, an id or a session name.
|
|
13
|
+
*/
|
|
14
|
+
export const PROGRESS_EVERY_MS = 30_000;
|
|
15
|
+
/** "5m00s" for a progress line: seconds matter here, unlike a lock's age. */
|
|
16
|
+
export function clock(ms) {
|
|
17
|
+
const seconds = Math.floor(ms / 1000);
|
|
18
|
+
return `${Math.floor(seconds / 60)}m${String(seconds % 60).padStart(2, "0")}s`;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* The line for this tick. `last` is what the previous line reported (null for
|
|
22
|
+
* the first line), so the sentence is always a difference, never a repeat.
|
|
23
|
+
*/
|
|
24
|
+
export function backfillProgressLine(now, last, elapsedMs) {
|
|
25
|
+
if (now.remaining === null) {
|
|
26
|
+
return `backfill: reading your old sessions, ${clock(elapsedMs)}`;
|
|
27
|
+
}
|
|
28
|
+
const counts = `${now.caughtUp} caught up, ${now.remaining} to go`;
|
|
29
|
+
if (last === null || last.remaining === null) {
|
|
30
|
+
return `backfill: ${counts}, ${clock(elapsedMs)}`;
|
|
31
|
+
}
|
|
32
|
+
const gained = now.caughtUp - last.caughtUp;
|
|
33
|
+
if (gained <= 0 && now.remaining === last.remaining) {
|
|
34
|
+
return `backfill: no change in the last 30s (${counts}), ${clock(elapsedMs)}`;
|
|
35
|
+
}
|
|
36
|
+
return `backfill: +${Math.max(0, gained)} caught up in the last 30s (${counts}), ${clock(elapsedMs)}`;
|
|
37
|
+
}
|
|
38
|
+
/** The line for a repair that is one long command with nothing to count (`sync`). */
|
|
39
|
+
export function waitingLine(what, elapsedMs) {
|
|
40
|
+
return `${what}: still working, ${clock(elapsedMs)} so far`;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Print `describe(elapsedMs)` every 30 seconds until stopped. The timer never
|
|
44
|
+
* keeps the process alive. Returns the stop function.
|
|
45
|
+
*/
|
|
46
|
+
export function startProgress(write, describe, options = {}) {
|
|
47
|
+
const now = options.now ?? (() => Date.now());
|
|
48
|
+
const started = now();
|
|
49
|
+
const timer = (options.setIntervalImpl ?? setInterval)(() => {
|
|
50
|
+
write(`${describe(now() - started)}\n`);
|
|
51
|
+
}, PROGRESS_EVERY_MS);
|
|
52
|
+
timer.unref?.();
|
|
53
|
+
return () => (options.clearIntervalImpl ?? clearInterval)(timer);
|
|
54
|
+
}
|
|
@@ -157,7 +157,8 @@ export function repairReceiptLines(repairs) {
|
|
|
157
157
|
return [];
|
|
158
158
|
return [
|
|
159
159
|
"Repairs (before -> after):",
|
|
160
|
-
|
|
160
|
+
// BLI-12872: a row left "skipped" is a background sync doing the work; it is in progress, not waiting on a person.
|
|
161
|
+
...ran.map((repair) => ` ${repair.after?.startsWith("ok:") ? "fixed" : repair.after?.startsWith("skipped:") ? "in progress" : "still needs you"} ${repair.step}: ${repair.before_message ?? ""} -> ${repair.after_message ?? ""}`.replaceAll("—", ";")),
|
|
161
162
|
];
|
|
162
163
|
}
|
|
163
164
|
export function doctorNeedsPerson(rows, receipt) {
|
package/dist/commands/doctor.js
CHANGED
|
@@ -4,6 +4,7 @@ import { describeError } from "../health-detail.js";
|
|
|
4
4
|
import { withStage } from "../crash-guard.js";
|
|
5
5
|
import { checkSingleInstallState, readRootsCoverWork, fixAuthState, fixRootState, readAuthState, readRootState, } from "./doctor-access.js";
|
|
6
6
|
import { backfillCompletionStepState, backfillFixVerdict, checkBackfillState, checkDiskState, checkGcState, checkSyncState, fixBackfillState, fixDiskState, fixGcState, fixSyncState, syncBacklogDrainingVerdict, syncStandAsideVerdict, } from "./doctor-pipeline.js";
|
|
7
|
+
import { gateRow, readCollectionGate } from "./doctor-collection-lock.js";
|
|
7
8
|
import { checkMcpAnswersState, fixMcpAnswersState } from "./doctor-mcp.js";
|
|
8
9
|
import { budgetSpentRow, doctorRunDeadline, DOCTOR_JSON_BUDGET_SECONDS } from "./doctor-budget.js";
|
|
9
10
|
import { checkMemoryDaemonState } from "./doctor-memory-daemon.js";
|
|
@@ -49,15 +50,22 @@ export async function runDoctorWithDeps(command, io, deps) {
|
|
|
49
50
|
const prerequisitesReady = !needsCollection || rows.filter((prior) => prior.id === "authed" || prior.id === "roots-ok").every((prior) => prior.status === "ok");
|
|
50
51
|
const canFix = !diagnoseOnly && prerequisitesReady && invariant.fix &&
|
|
51
52
|
(!invariant.requiresInteractiveFix || isInteractiveDoctorFix(context) || Boolean(command.collectionRoots?.length));
|
|
52
|
-
|
|
53
|
+
let spent = row.status !== "ok" && row.status !== "skipped" && canFix ? budgetSpentRow(context, row) : null;
|
|
54
|
+
if (spent && needsCollection) {
|
|
55
|
+
// BLI-12872: a running sync is the answer even when the budget is spent; looking costs nothing.
|
|
56
|
+
const gate = await readCollectionGate(context, undefined, { mayEnd: false });
|
|
57
|
+
if (gate.kind === "running")
|
|
58
|
+
spent = gateRow(invariant.id, gate);
|
|
59
|
+
}
|
|
53
60
|
if (spent) {
|
|
54
61
|
row = spent;
|
|
55
|
-
repairs.push({ step: invariant.id, outcome: "needs_person", reason: row.code });
|
|
62
|
+
repairs.push({ step: invariant.id, outcome: row.status === "skipped" ? "not_needed" : "needs_person", reason: row.code });
|
|
56
63
|
}
|
|
57
64
|
else if (row.status !== "ok" && row.status !== "skipped" && canFix && invariant.fix) {
|
|
58
65
|
attempted.add(invariant.id);
|
|
59
66
|
row = await withStage(`doctor:fix:${invariant.id}`, () => invariant.fix(context, row));
|
|
60
|
-
|
|
67
|
+
// BLI-12872: a skipped row after a fix is a sync already doing the work, not a person's job.
|
|
68
|
+
repairs.push({ step: invariant.id, outcome: row.status === "ok" ? "repaired" : row.status === "skipped" ? "not_needed" : "needs_person", reason: row.code });
|
|
61
69
|
// The replacement process owns the one final receipt, including JSON.
|
|
62
70
|
if (row.reexecExitCode !== undefined)
|
|
63
71
|
return row.reexecExitCode;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `cockpit brain` argument parsing (BLI-12859).
|
|
3
|
+
*
|
|
4
|
+
* cockpit brain status [--project <id>] [--show open|running|needs_human|fixed|not_actionable|cleared] [--json]
|
|
5
|
+
*
|
|
6
|
+
* Tower's incident queue: per project, how many incidents are open, being worked, waiting on a
|
|
7
|
+
* person, fixed, not actionable or cleared, the newest ones with their Tower issue, and the last
|
|
8
|
+
* 7 days of verdicts. It reads `GET /api/brain/incidents` with this machine's device token.
|
|
9
|
+
*/
|
|
10
|
+
import { LOG_PROJECT_PATTERN } from "@bli-cockpit/telemetry-core/log-digest";
|
|
11
|
+
import { optionalNonEmpty, optionalUrl, parseNamedArgs } from "./local-arg-values.js";
|
|
12
|
+
/** The statuses the queue's door accepts for `--show`; the migration's CHECK constraint is the other copy. */
|
|
13
|
+
export const BRAIN_SHOW_STATUSES = ["open", "running", "needs_human", "fixed", "not_actionable", "cleared"];
|
|
14
|
+
export function parseBrainArgs(args) {
|
|
15
|
+
const values = parseNamedArgs(args, {
|
|
16
|
+
allowedFlags: ["--project", "--show", "--home", "--dashboard-url", "--json"],
|
|
17
|
+
valueFlags: ["--project", "--show", "--home", "--dashboard-url"],
|
|
18
|
+
});
|
|
19
|
+
const action = values.positionals[0];
|
|
20
|
+
if (action !== "status" || values.positionals.length > 1) {
|
|
21
|
+
throw new Error("brain takes one verb: `status`. Example: `cockpit brain status` or `cockpit brain status --project pearl-portal`.");
|
|
22
|
+
}
|
|
23
|
+
const project = optionalNonEmpty(values.flags.get("--project"));
|
|
24
|
+
if (project && !LOG_PROJECT_PATTERN.test(project))
|
|
25
|
+
throw new Error("--project wants a registry project id like pearl-portal (`cockpit logs --list` names them).");
|
|
26
|
+
const show = optionalNonEmpty(values.flags.get("--show"));
|
|
27
|
+
if (show && !BRAIN_SHOW_STATUSES.includes(show))
|
|
28
|
+
throw new Error(`--show wants one of ${BRAIN_SHOW_STATUSES.join(", ")}.`);
|
|
29
|
+
return {
|
|
30
|
+
kind: "brain",
|
|
31
|
+
action: "status",
|
|
32
|
+
...(project ? { project } : {}),
|
|
33
|
+
...(show ? { show } : {}),
|
|
34
|
+
dashboardUrl: optionalUrl(values.flags.get("--dashboard-url")),
|
|
35
|
+
homeDir: optionalNonEmpty(values.flags.get("--home")),
|
|
36
|
+
json: values.booleans.has("--json"),
|
|
37
|
+
};
|
|
38
|
+
}
|
|
@@ -39,5 +39,6 @@ export { parseMailArgs } from "./local-args-tower-mail.js";
|
|
|
39
39
|
export { parseCalArgs } from "./local-args-tower-cal.js";
|
|
40
40
|
export { parseUsageArgs } from "./local-args-tower-usage.js";
|
|
41
41
|
export { parseLogsArgs } from "./local-args-tower-logs.js";
|
|
42
|
+
export { parseBrainArgs } from "./local-args-tower-brain.js";
|
|
42
43
|
export { parseCareersArgs } from './local-args-tower-careers.js';
|
|
43
44
|
export { parseRenameArgs } from './local-args-tower-rename.js';
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
import { parseRootsArgs } from "./roots.js";
|
|
19
19
|
import { SURFACE_ALIASES } from "./local-help.js";
|
|
20
20
|
import { parseAgentRulesArgs, parseAnalyzeArgs, parseAutostartArgs, parseBackfillArgs, parseCleanArgs, parseDoctorArgs, parseInstallArgs, parseLoginArgs, parseMemoryArgs, parseLogoutArgs, parseOnboardArgs, parseReleaseArgs, parseServeArgs, parseSessionsArgs, parseSourcesArgs, parseStartArgs, parseStatusArgs, parseSyncArgs, parseUpdateArgs, } from "./local-args-collector.js";
|
|
21
|
-
import { parseBriefArgs, parseCorrectArgs, parseDocsArgs, parseIssueArgs, parseJarvisArgs, parseMailArgs, parseCalArgs, parseModelArgs, parseMsgArgs, parseNotesArgs, parseOpsArgs, parseProjectArgs, parseModelsArgs, parseScoutArgs, parseGithubArgs, parseSearchArgs, parseSettingsArgs, parseSlackArgs, parseTeamArgs, parseWorkbookArgs, parseUsageArgs, parseLogsArgs, parseCareersArgs, parseRenameArgs, } from "./local-args-tower.js";
|
|
21
|
+
import { parseBriefArgs, parseCorrectArgs, parseDocsArgs, parseIssueArgs, parseJarvisArgs, parseMailArgs, parseCalArgs, parseModelArgs, parseMsgArgs, parseNotesArgs, parseOpsArgs, parseProjectArgs, parseModelsArgs, parseScoutArgs, parseGithubArgs, parseSearchArgs, parseSettingsArgs, parseSlackArgs, parseTeamArgs, parseWorkbookArgs, parseUsageArgs, parseLogsArgs, parseBrainArgs, parseCareersArgs, parseRenameArgs, } from "./local-args-tower.js";
|
|
22
22
|
// `normalizeUrl` has always been part of this module's surface — `local.ts` and
|
|
23
23
|
// `local-auth.ts` import it from here — so it stays exported from this address
|
|
24
24
|
// even though it now lives next door. The same goes for the four names the
|
|
@@ -135,6 +135,8 @@ export function parseLocalArgs(argv) {
|
|
|
135
135
|
return parseUsageArgs(argv.slice(1));
|
|
136
136
|
case "logs":
|
|
137
137
|
return parseLogsArgs(argv.slice(1));
|
|
138
|
+
case "brain":
|
|
139
|
+
return parseBrainArgs(argv.slice(1));
|
|
138
140
|
default:
|
|
139
141
|
throw new Error(`Unknown local command: ${command ?? ""}`);
|
|
140
142
|
}
|
|
@@ -44,6 +44,23 @@ export const TOWER_OPS_COMMAND_HELP = [
|
|
|
44
44
|
"The bli-tower MCP server has the twin: logs_digest.",
|
|
45
45
|
],
|
|
46
46
|
],
|
|
47
|
+
[
|
|
48
|
+
"brain",
|
|
49
|
+
[
|
|
50
|
+
"Usage: cockpit brain status [--project <id>] [--show open|running|needs_human|fixed|not_actionable|cleared] [--dashboard-url <url>] [--json]",
|
|
51
|
+
"",
|
|
52
|
+
"Tower's incident queue. Every 15 minutes the log-digest cron turns each recurring problem in a project's logs (classes",
|
|
53
|
+
"our_bug, config_drift, capacity, unknown) into ONE incident with ONE Tower issue; a repeat adds to its count, never a twin.",
|
|
54
|
+
"A template quiet for 24 hours is marked cleared. This prints, per project, how many incidents are open, being worked,",
|
|
55
|
+
"waiting on a person, fixed, not actionable or cleared, the newest ones in the --show status (default open) with their Tower",
|
|
56
|
+
"issue, how often each was seen and what it said, and the last 7 days of verdicts. Needs only this machine's Tower login.",
|
|
57
|
+
"--project <id> one project (`cockpit logs --list` names them).",
|
|
58
|
+
"--show <status> list the newest incidents in that status instead of open.",
|
|
59
|
+
"--json writes one object to stdout: { ok, generated_at, projects[{ project, counts, incidents[], verdicts_7d }], totals, coverage, text };",
|
|
60
|
+
"a refusal is { ok: false, error, detail } and exits 1 (unknown_project, invalid_status, stored_read_failed, not_paired, ...).",
|
|
61
|
+
"Teaching: docs/architecture/tower-brain.md. The bli-tower MCP server has the twin: brain_status.",
|
|
62
|
+
],
|
|
63
|
+
],
|
|
47
64
|
[
|
|
48
65
|
"github",
|
|
49
66
|
[
|
|
@@ -94,7 +94,7 @@ export function localSubcommandHelp(command) {
|
|
|
94
94
|
"roots-cover-work is informational: it names how many sessions are inside vs. outside saved roots (all already uploaded, outside ones labelled) and gives cockpit roots add commands for widening a root's redacted transcript content. It never needs a fix.",
|
|
95
95
|
"--check or --no-repair diagnoses without repairs. --dry-run previews repairs.",
|
|
96
96
|
"--backfill-budget defaults to 900 seconds; catch-up repeats chunks until complete, with progress every 30 seconds.",
|
|
97
|
-
"--lock-wait
|
|
97
|
+
"Doctor never waits for the background sync: a running sync is reported as running (with how long ago it started) and doctor finishes; a sync that stopped responding or made no progress for 15 minutes is ended and doctor carries on. --lock-wait is accepted and changes nothing. Progress prints at most every 30 seconds and says what changed.",
|
|
98
98
|
"--json without either flag answers within 45 seconds; a repair that does not fit reads doctor_budget_spent.",
|
|
99
99
|
"--json includes steps, repairs, and needs_person. Exit 0 means nothing needs you.",
|
|
100
100
|
"Maintainers: --update-tag next keeps self-update on the prerelease candidate.",
|
|
@@ -79,6 +79,7 @@ export const rootCommandNames = new Set([
|
|
|
79
79
|
"rename",
|
|
80
80
|
"usage",
|
|
81
81
|
"logs",
|
|
82
|
+
"brain",
|
|
82
83
|
...SURFACE_ALIASES.keys(),
|
|
83
84
|
]);
|
|
84
85
|
/** Public names omit compatibility spellings while the router keeps accepting them. */
|
|
@@ -134,6 +135,7 @@ export function localCommandHelp(command) {
|
|
|
134
135
|
" cockpit careers [list|show <id>|rescreen <id>|send-takehome <id>|decide <id> --decision <passed_on|archived|reopen>|followup <id> [--record-sent-at <iso>]|interview <id> <--at <iso>|--clear>|grade <id> [--now]|resume <id> [--out <path>]|takehome show <role>|takehome set <role> --body-file <path>|settings show|settings set --enabled <true|false> --followup-enabled <true|false> --followup-after-days <n> --takehome-window-hours <n>] [--role <slug>] [--min-score <n>] [--since <date>] [--stage <s>] [--force] [--json]",
|
|
135
136
|
" cockpit rename [list|resolve <id> --scope <page|everywhere|undo>] [--dashboard-url <url>] [--json]",
|
|
136
137
|
" cockpit logs <collector|dashboard|railway|railway:<service>|-> [--since 15m|1h|2d|<iso>] [--raw <template-id> [--limit <n>]] [--all-levels] [--table <file>] [--json]",
|
|
138
|
+
" cockpit brain status [--project <id>] [--show open|running|needs_human|fixed|not_actionable|cleared] [--dashboard-url <url>] [--json]",
|
|
137
139
|
" cockpit usage sessions [--task <BLI-NNNN> | --subject <name>] [--repo <label>] [--since <window>] [--json]",
|
|
138
140
|
" cockpit usage sessions --session <id> [--from-window <n>] [--json]",
|
|
139
141
|
" --by-topic groups work types; --by-subject groups open subjects from session summaries.",
|
package/dist/commands/local.js
CHANGED
|
@@ -50,6 +50,7 @@ import { runCareers } from "./careers.js";
|
|
|
50
50
|
import { runRename } from "./rename-scope.js";
|
|
51
51
|
import { runUsage } from "./usage.js";
|
|
52
52
|
import { runLogs } from "./logs.js";
|
|
53
|
+
import { runBrain } from "./brain.js";
|
|
53
54
|
import { parseLocalArgs } from "./local-args.js";
|
|
54
55
|
// `./local.js` is the published entry point for this command surface: the
|
|
55
56
|
// public CLI's generated root, commands/root.ts, doctor.ts and the test suite
|
|
@@ -245,6 +246,8 @@ export async function runLocalCockpitCli(argv, io = defaultIo()) {
|
|
|
245
246
|
return await runUsage(command, io);
|
|
246
247
|
case "logs":
|
|
247
248
|
return await runLogs(command, io);
|
|
249
|
+
case "brain":
|
|
250
|
+
return await runBrain(command, io);
|
|
248
251
|
}
|
|
249
252
|
}
|
|
250
253
|
catch (error) {
|
|
@@ -46,7 +46,7 @@ export async function runStoredLogs(command, io, options) {
|
|
|
46
46
|
if (!answer.ok)
|
|
47
47
|
return failAgentDoor(door, TAG, answer.reason, answer.detail);
|
|
48
48
|
const body = answer.body;
|
|
49
|
-
writeLine(io.stderr, `${TAG} stored digest read ${JSON.stringify({ project: body.project, windows: body.windows, windows_ok: body.windows_ok, refusals: body.refusals.length, fallback: Boolean(options.fallbackReason) })}`);
|
|
49
|
+
writeLine(io.stderr, `${TAG} stored digest read ${JSON.stringify({ project: body.project, windows: body.windows, windows_ok: body.windows_ok, refusals: body.refusals.length, still_refusing: body.still_refusing?.length ?? null, not_wired: body.not_wired?.length ?? null, fallback: Boolean(options.fallbackReason) })}`);
|
|
50
50
|
if (command.json)
|
|
51
51
|
return emitAgentDoor(door, options.fallbackReason ? { ...body, fallback_from: options.fallbackReason } : body);
|
|
52
52
|
if (options.fallbackReason)
|
|
@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
24
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
24
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.180");
|
|
25
25
|
return 0;
|
|
26
26
|
}
|
|
27
27
|
|
package/dist/commands/status.js
CHANGED
|
@@ -80,7 +80,7 @@ async function writeMultiRepoStatus(command, io, backfillCursor, setupReceipt, w
|
|
|
80
80
|
writeLine(io.stdout, `Old sessions: ${backfillCursorLine(backfillCursor)}`);
|
|
81
81
|
writeLine(io.stdout, await statusDiskLine(command.homeDir));
|
|
82
82
|
for (const status of statuses) {
|
|
83
|
-
writeLine(io.stdout, `- ${status.repo_label ?? status.repo}/${status.worktree_label ?? "worktree"} · ${status.branch} · head:${shortSha(status.head_sha)} · ${status.upload_state}`);
|
|
83
|
+
writeLine(io.stdout, `- ${status.repo_label ?? status.repo}/${status.worktree_label ?? "worktree"} · ${status.branch} · head:${shortSha(status.head_sha)} · ${uploadsLine(status.upload_state)}`);
|
|
84
84
|
}
|
|
85
85
|
writeConnectedBlock(io, setupReceipt);
|
|
86
86
|
return 0;
|
|
@@ -7,6 +7,7 @@ import { healStubsDuringTick } from "./sync-stub-heal.js";
|
|
|
7
7
|
import { backfillWaitingCheck } from "./sync-yield.js";
|
|
8
8
|
import { getCollectorRuntimePaths } from "../local-state.js";
|
|
9
9
|
import { rawEvidenceDedupSummary, sweepDuplicateStagedRawEvidence, } from "../raw-evidence-gc.js";
|
|
10
|
+
import { noteSyncProgress } from "../sync-lock.js";
|
|
10
11
|
/**
|
|
11
12
|
* Collect everything this tick may collect, then report it.
|
|
12
13
|
*
|
|
@@ -25,6 +26,7 @@ export async function runSyncLocked(command, io) {
|
|
|
25
26
|
if (!dedup.skipped && dedup.removed_dirs > 0) {
|
|
26
27
|
writeLine(io.stdout, rawEvidenceDedupSummary(dedup));
|
|
27
28
|
}
|
|
29
|
+
noteSyncProgress(); // BLI-12872: the sweep finished; the lock's progress mark
|
|
28
30
|
let discoveryNotice = null;
|
|
29
31
|
const worktrees = await discoverCommandWorktrees(collectionRoots, {
|
|
30
32
|
maxDepth: command.maxDepth,
|
|
@@ -33,6 +35,7 @@ export async function runSyncLocked(command, io) {
|
|
|
33
35
|
allowEmpty: true,
|
|
34
36
|
onPartial: (notice) => { discoveryNotice = notice; },
|
|
35
37
|
}, io);
|
|
38
|
+
noteSyncProgress(); // BLI-12872: discovery finished
|
|
36
39
|
const run = await runAttributedWorktreeSync({
|
|
37
40
|
homeDir: command.homeDir,
|
|
38
41
|
dashboardUrl: command.dashboardUrl,
|
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
* worktree's upload, which is why each one logs its own elapsed time.
|
|
16
16
|
*/
|
|
17
17
|
import { inspectBackfillLock } from "../backfill-lock.js";
|
|
18
|
+
import { noteSyncProgress } from "../sync-lock.js";
|
|
18
19
|
/** The waiter's flag: the backfill lock is held while its owner waits for ours. */
|
|
19
20
|
export function backfillWaitingCheck(paths, inspect = inspectBackfillLock) {
|
|
20
21
|
return async () => {
|
|
@@ -50,6 +51,9 @@ export async function runYieldingAtBoundaries(options) {
|
|
|
50
51
|
const itemStarted = now();
|
|
51
52
|
const result = await options.runOne(item);
|
|
52
53
|
results.push(result);
|
|
54
|
+
// BLI-12872: a finished worktree is the lock's progress mark, what keeps a
|
|
55
|
+
// long tick from being called stuck while it keeps working.
|
|
56
|
+
noteSyncProgress();
|
|
53
57
|
console.error("[session-sync] worktree synced", JSON.stringify({
|
|
54
58
|
index: index + 1,
|
|
55
59
|
total: options.items.length,
|