@bli-cockpit/cli 0.2.99 → 0.2.101
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-rules.js +2 -1
- package/dist/backfill-lock.js +1 -1
- package/dist/commands/backfill-checkpoint.js +3 -1
- package/dist/commands/backfill-issues.js +8 -55
- package/dist/commands/backfill-report.js +22 -6
- package/dist/commands/backfill-scan.js +2 -1
- package/dist/commands/backfill-skip-policy.js +134 -0
- package/dist/commands/careers.js +16 -0
- package/dist/commands/doctor-access.js +34 -10
- package/dist/commands/doctor-lock-wait.js +46 -0
- package/dist/commands/doctor-pipeline-verdicts.js +238 -0
- package/dist/commands/doctor-pipeline.js +49 -111
- package/dist/commands/doctor-registration.js +23 -2
- package/dist/commands/doctor-report.js +48 -9
- package/dist/commands/doctor-update.js +16 -5
- package/dist/commands/doctor.js +98 -58
- package/dist/commands/local-args-collector-setup.js +6 -0
- package/dist/commands/local-args-tower-careers.js +20 -0
- package/dist/commands/local-args-tower-pages.js +17 -2
- package/dist/commands/local-args-tower-usage.js +2 -2
- package/dist/commands/local-args-tower.js +2 -1
- package/dist/commands/local-args.js +3 -1
- package/dist/commands/local-help-commands-tower.js +4 -2
- package/dist/commands/local-help-commands.js +28 -12
- package/dist/commands/local-help.js +4 -2
- package/dist/commands/local.js +4 -0
- package/dist/commands/notes-file.js +8 -1
- package/dist/commands/notes-folders.js +35 -0
- package/dist/commands/notes-writes.js +37 -4
- package/dist/commands/notes.js +6 -0
- package/dist/commands/public-root.js +4 -4
- package/dist/commands/usage-format.js +18 -0
- package/dist/commands/usage.js +13 -3
- package/dist/cursors/backfill-completion-marker.js +135 -0
- package/dist/cursors/backfill-cursor.js +18 -99
- package/dist/scheduled-self-update.js +1 -1
- package/dist/sync-lock.js +15 -1
- package/package.json +2 -2
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The pure verdicts the pipeline rows compute, and nothing else.
|
|
3
|
+
*
|
|
4
|
+
* `doctor-pipeline.ts` is the four checks that TOUCH the machine (run the
|
|
5
|
+
* catch-up, prune the disk, exec a sync). Everything here is the opposite: a
|
|
6
|
+
* function that takes a receipt or a marker someone else read and answers
|
|
7
|
+
* "what does that mean for the row". They live apart so each verdict can be
|
|
8
|
+
* unit-tested against a payload recorded off a real machine, with no exec, no
|
|
9
|
+
* filesystem and no home directory.
|
|
10
|
+
*
|
|
11
|
+
* Read it as four answers:
|
|
12
|
+
*
|
|
13
|
+
* isCollectionBusyReason another run owns the lock; nothing is broken
|
|
14
|
+
* backfillCompletionStepState what an all-history marker proves
|
|
15
|
+
* backfillFixVerdict busy vs progress vs actually broken
|
|
16
|
+
* syncBacklogDrainingVerdict a backlog draining is not a failed tick
|
|
17
|
+
* syncStandAsideVerdict is the lock's owner still alive
|
|
18
|
+
*
|
|
19
|
+
* Every name here is re-exported from `./doctor-pipeline.js`, the address its
|
|
20
|
+
* callers already know.
|
|
21
|
+
*/
|
|
22
|
+
import { backfillCompletionCovers } from "../cursors/backfill-cursor.js";
|
|
23
|
+
import { describeError } from "../health-detail.js";
|
|
24
|
+
import { SYNC_LOCK_STALE_TAKEOVER_MS } from "../sync-lock.js";
|
|
25
|
+
import { asRecord, fail, needsFix, ok } from "./doctor-report.js";
|
|
26
|
+
/**
|
|
27
|
+
* BLI-4303: the reasons a collection command stood aside because ANOTHER
|
|
28
|
+
* collection run already owned the machine's single collection lock.
|
|
29
|
+
*
|
|
30
|
+
* `backfill` reports them as `failure_reason`, `sync` as its top-level
|
|
31
|
+
* `status`; both exit 0 and both say `retryable: true`. On a machine whose
|
|
32
|
+
* launchd job also watches `~/.claude/projects` and `~/.codex/sessions`, a tick
|
|
33
|
+
* is in flight most of the time an agent is working — the reference Mac held
|
|
34
|
+
* the lock in 76 of 89 samples over three minutes — so doctor's own repair
|
|
35
|
+
* almost always met a running sync and called it a failure. A busy machine is
|
|
36
|
+
* the collector working, never a machine to repair: these produce a named
|
|
37
|
+
* non-red row and the next run proves the receipt.
|
|
38
|
+
*/
|
|
39
|
+
const COLLECTION_BUSY_REASONS = new Set([
|
|
40
|
+
// `cockpit sync` stood aside for another sync.
|
|
41
|
+
"sync_already_running",
|
|
42
|
+
// `cockpit sync` stood aside for a running backfill.
|
|
43
|
+
"live_sync_paused_during_backfill",
|
|
44
|
+
// `cockpit backfill` stood aside for another backfill.
|
|
45
|
+
"backfill_already_running",
|
|
46
|
+
]);
|
|
47
|
+
export function isCollectionBusyReason(reason) {
|
|
48
|
+
return typeof reason === "string" && COLLECTION_BUSY_REASONS.has(reason);
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Pure so it can be unit-tested without touching the real machine's home
|
|
52
|
+
* directory (`getCollectorRuntimePaths()` defaults to `os.homedir()` and
|
|
53
|
+
* doctor never threads `--home` through the backfill steps). Returns `null`
|
|
54
|
+
* when the marker does not cover the roots/sources — the caller falls
|
|
55
|
+
* through to the lock/never-run diagnosis in that case.
|
|
56
|
+
*
|
|
57
|
+
* BLI-2727/BLI-4303: a marker whose only outstanding entries are deterministic
|
|
58
|
+
* skips (files over the upload cap, helper transcripts over the per-session
|
|
59
|
+
* sidecar cap) is still a completed backfill. It reads green with a named
|
|
60
|
+
* note, never a red `needs_fix`/`fail`, so a cap nobody can lift from here
|
|
61
|
+
* never reads as "backfill never completed" on repeat doctor runs.
|
|
62
|
+
*/
|
|
63
|
+
export function backfillCompletionStepState(marker, roots) {
|
|
64
|
+
if (!backfillCompletionCovers(marker, roots, ["codex", "claude_code"])) {
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
const caughtUp = "caught up on old Codex and Claude sessions in every saved folder";
|
|
68
|
+
const oversized = marker?.oversized_skips;
|
|
69
|
+
const sidecarCaps = marker?.sidecar_cap_skips;
|
|
70
|
+
const notes = [];
|
|
71
|
+
if (oversized && oversized.count > 0) {
|
|
72
|
+
notes.push(`${oversized.count} file${oversized.count === 1 ? "" : "s"} too big to upload`);
|
|
73
|
+
}
|
|
74
|
+
// BLI-4303: a cap is named, never silent. It no longer blocks completion, so
|
|
75
|
+
// the green row is the only place a person would ever learn it fired.
|
|
76
|
+
if (sidecarCaps && sidecarCaps.count > 0) {
|
|
77
|
+
notes.push(`${sidecarCaps.count} helper transcript${sidecarCaps.count === 1 ? "" : "s"} past the per-session cap`);
|
|
78
|
+
}
|
|
79
|
+
if (notes.length === 0)
|
|
80
|
+
return ok("backfill-complete", "complete", caughtUp);
|
|
81
|
+
const code = oversized && sidecarCaps
|
|
82
|
+
? "complete_with_capped_skips"
|
|
83
|
+
: oversized
|
|
84
|
+
? "complete_with_oversized_skips"
|
|
85
|
+
: "complete_with_sidecar_caps";
|
|
86
|
+
return ok("backfill-complete", code, `${caughtUp} (${code} · ${notes.join(", ")})`);
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* `cockpit backfill --json` prints exactly one JSON document to stdout. Parsed
|
|
90
|
+
* rather than field-scraped for the same reason sync is (BLI-2728): `counts`
|
|
91
|
+
* and `batches` both carry a `failed`, and a regex cannot tell them apart.
|
|
92
|
+
*/
|
|
93
|
+
export function parseDoctorBackfillJson(stdout) {
|
|
94
|
+
try {
|
|
95
|
+
const parsed = JSON.parse(stdout.trim());
|
|
96
|
+
return parsed && typeof parsed === "object" ? parsed : null;
|
|
97
|
+
}
|
|
98
|
+
catch (error) {
|
|
99
|
+
console.error("[cockpit-doctor] backfill --json stdout was not one JSON document", JSON.stringify({
|
|
100
|
+
reason: "backfill_json_unparseable",
|
|
101
|
+
byte_size: stdout.length,
|
|
102
|
+
...describeError(error),
|
|
103
|
+
}));
|
|
104
|
+
return null;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* BLI-4303: what a backfill that did not reach `complete` means for the row.
|
|
109
|
+
*
|
|
110
|
+
* Three different machines used to read `❌ backfill did not complete`, and
|
|
111
|
+
* only one of them was broken:
|
|
112
|
+
*
|
|
113
|
+
* busy another collection run owned the lock, so this one never started
|
|
114
|
+
* progress it uploaded history and left more of it for the next run
|
|
115
|
+
* broken it could not do the work — auth, the server, a read failure
|
|
116
|
+
*
|
|
117
|
+
* Only the third is a `fail`. The first two are `needs_fix`, which is the same
|
|
118
|
+
* word `checkBackfillState` already uses for an unfinished catch-up: the fix
|
|
119
|
+
* side had been the only place that called an unfinished backfill a failure.
|
|
120
|
+
* Pure so the three verdicts are testable off a recorded `--json` payload.
|
|
121
|
+
*/
|
|
122
|
+
export function backfillFixVerdict(parsed, scrapedReason) {
|
|
123
|
+
const reason = (typeof parsed?.failure_reason === "string" ? parsed.failure_reason : null) ??
|
|
124
|
+
scrapedReason;
|
|
125
|
+
if (isCollectionBusyReason(reason)) {
|
|
126
|
+
return needsFix("backfill-complete", reason ?? "collection_busy", "another collection run is using your sessions right now, so the catch-up " +
|
|
127
|
+
"stood aside; it runs on the next `cockpit doctor` or `cockpit backfill --all --yes`");
|
|
128
|
+
}
|
|
129
|
+
const counts = asRecord(parsed?.counts);
|
|
130
|
+
const batches = asRecord(parsed?.batches);
|
|
131
|
+
const backfilled = positiveNumberOrZero(counts?.["backfilled"]);
|
|
132
|
+
const remaining = positiveNumberOrZero(counts?.["remaining"]);
|
|
133
|
+
const completedBatches = positiveNumberOrZero(batches?.["completed"]);
|
|
134
|
+
if (backfilled > 0 || completedBatches > 0) {
|
|
135
|
+
return needsFix("backfill-complete", reason ?? "backfill_incomplete", `caught up on ${backfilled} old session${backfilled === 1 ? "" : "s"} this run` +
|
|
136
|
+
(remaining > 0 ? `, ${remaining} still to go` : "") +
|
|
137
|
+
"; run `cockpit backfill --all --yes` again to continue");
|
|
138
|
+
}
|
|
139
|
+
return fail("backfill-complete", reason ?? "backfill_failed", "backfill did not complete");
|
|
140
|
+
}
|
|
141
|
+
export function jsonField(output, field) {
|
|
142
|
+
const escaped = field.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
|
143
|
+
const match = output.match(new RegExp(`"${escaped}"\\s*:\\s*"([^"]+)"`, "u"));
|
|
144
|
+
return match?.[1] ?? null;
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* `cockpit sync --json` prints exactly one JSON document to stdout (stderr is
|
|
148
|
+
* for human text; see AGENTS.md logging conventions), so this is a real parse
|
|
149
|
+
* rather than the doctor module's usual regex field-scrape — which cannot
|
|
150
|
+
* disambiguate same-named fields nested under `codex_sessions.codex` vs
|
|
151
|
+
* `codex_sessions.claude` (BLI-2728).
|
|
152
|
+
*/
|
|
153
|
+
export function parseDoctorSyncJson(stdout) {
|
|
154
|
+
try {
|
|
155
|
+
const parsed = JSON.parse(stdout.trim());
|
|
156
|
+
return parsed && typeof parsed === "object" ? parsed : null;
|
|
157
|
+
}
|
|
158
|
+
catch (error) {
|
|
159
|
+
// `null` sends doctor back to its regex field-scrape, quietly losing the
|
|
160
|
+
// BLI-2728 disambiguation. Something wrote to stdout that was not the one
|
|
161
|
+
// JSON document the contract promises — a stray console.log in the
|
|
162
|
+
// collector would do exactly this and look like nothing at all.
|
|
163
|
+
console.error("[cockpit-doctor] sync --json stdout was not one JSON document", JSON.stringify({
|
|
164
|
+
reason: "sync_json_unparseable",
|
|
165
|
+
byte_size: stdout.length,
|
|
166
|
+
...describeError(error),
|
|
167
|
+
}));
|
|
168
|
+
return null;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* BLI-2728: a tick that only deferred objects past the per-tick raw-evidence
|
|
173
|
+
* object budget (`RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET`, adapters/raw-evidence.ts)
|
|
174
|
+
* is a backlog that is draining, not a failure — `attributedSyncRunStatus`
|
|
175
|
+
* marks the run not-fully-`ok` (so `cockpit sync` exits non-zero and doctor's
|
|
176
|
+
* exec sees `code !== 0`) purely because objects remain queued, with the
|
|
177
|
+
* per-repo upload itself still having succeeded. A genuine failure (auth,
|
|
178
|
+
* server rejection, network, a real upload_failed outcome, an unposted
|
|
179
|
+
* session report) must still read red — this only fires when NOTHING else in
|
|
180
|
+
* the tick's own summary looks wrong. Pure so it is unit-testable without a
|
|
181
|
+
* live exec/fs harness; the remaining-object count is read straight from the
|
|
182
|
+
* tick's own summary, never recomputed.
|
|
183
|
+
*/
|
|
184
|
+
export function syncBacklogDrainingVerdict(parsed) {
|
|
185
|
+
if (!parsed)
|
|
186
|
+
return null;
|
|
187
|
+
const deferredObjects = positiveNumberOrZero(parsed.raw_evidence_deferred_object_budget);
|
|
188
|
+
if (deferredObjects <= 0)
|
|
189
|
+
return null;
|
|
190
|
+
const deferredBytes = positiveNumberOrZero(parsed.raw_evidence_deferred_byte_budget);
|
|
191
|
+
const failedCount = positiveNumberOrZero(parsed.raw_evidence_failed_count);
|
|
192
|
+
const retryReasons = Array.isArray(parsed.raw_evidence_retry_reasons)
|
|
193
|
+
? parsed.raw_evidence_retry_reasons.length
|
|
194
|
+
: 0;
|
|
195
|
+
const sessions = asRecord(parsed.codex_sessions);
|
|
196
|
+
const reportPosted = sessions?.["report_posted"];
|
|
197
|
+
const codexReadFailures = positiveNumberOrZero(asRecord(sessions?.["codex"])?.["read_failures"]);
|
|
198
|
+
const claudeSessions = asRecord(sessions?.["claude"]);
|
|
199
|
+
const claudeReadFailures = positiveNumberOrZero(claudeSessions?.["read_failures"]);
|
|
200
|
+
const claudeSidecarsFailed = positiveNumberOrZero(claudeSessions?.["sidecars_failed"]);
|
|
201
|
+
const onlyDeferredObjectBudget = deferredBytes === 0 &&
|
|
202
|
+
failedCount === 0 &&
|
|
203
|
+
retryReasons === 0 &&
|
|
204
|
+
reportPosted === true &&
|
|
205
|
+
codexReadFailures === 0 &&
|
|
206
|
+
claudeReadFailures === 0 &&
|
|
207
|
+
claudeSidecarsFailed === 0;
|
|
208
|
+
return onlyDeferredObjectBudget ? { remainingObjects: deferredObjects } : null;
|
|
209
|
+
}
|
|
210
|
+
/**
|
|
211
|
+
* BLI-4303: a sync that stood aside for whoever already owns the collection
|
|
212
|
+
* lock, and whether that owner is still alive.
|
|
213
|
+
*
|
|
214
|
+
* The receipt's `held_since` is the holder's last heartbeat, refreshed every
|
|
215
|
+
* 30 s by the running process itself, and a lock whose heartbeat goes older
|
|
216
|
+
* than `SYNC_LOCK_STALE_TAKEOVER_MS` is taken over by the next sync. So a
|
|
217
|
+
* heartbeat inside that window is a live collection run and the honest answer
|
|
218
|
+
* for the row is "this machine is collecting"; a heartbeat outside it (or none
|
|
219
|
+
* at all, which is what a lock file this machine could not create looks like —
|
|
220
|
+
* see `sync-lock.ts`) is a machine that owes a person a look.
|
|
221
|
+
*
|
|
222
|
+
* Pure so both verdicts are testable off a recorded receipt.
|
|
223
|
+
*/
|
|
224
|
+
export function syncStandAsideVerdict(parsed, status, now = new Date()) {
|
|
225
|
+
const code = (typeof parsed?.status === "string" ? parsed.status : null) ?? status;
|
|
226
|
+
if (!isCollectionBusyReason(code))
|
|
227
|
+
return null;
|
|
228
|
+
const heldSince = typeof parsed?.held_since === "string" ? parsed.held_since : null;
|
|
229
|
+
const heartbeatMs = heldSince ? Date.parse(heldSince) : Number.NaN;
|
|
230
|
+
const ownerAlive = Number.isFinite(heartbeatMs) &&
|
|
231
|
+
now.getTime() - heartbeatMs <= SYNC_LOCK_STALE_TAKEOVER_MS;
|
|
232
|
+
return { code: code, heldSince, ownerAlive };
|
|
233
|
+
}
|
|
234
|
+
function positiveNumberOrZero(value) {
|
|
235
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0
|
|
236
|
+
? value
|
|
237
|
+
: 0;
|
|
238
|
+
}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
+
import { withDoctorLockWait } from "./doctor-lock-wait.js";
|
|
1
2
|
import fs from "node:fs/promises";
|
|
2
3
|
import path from "node:path";
|
|
3
4
|
import { inspectBackfillLock } from "../backfill-lock.js";
|
|
4
|
-
import {
|
|
5
|
+
import { readBackfillCompletionMarker, readBackfillCursor, } from "../cursors/backfill-cursor.js";
|
|
5
6
|
import { savedDiscoveryLimitArgs } from "../discovery-limits.js";
|
|
6
|
-
import { describeError } from "../health-detail.js";
|
|
7
7
|
import { runStagingPrune } from "../disk-prune.js";
|
|
8
8
|
import { retentionOptionsFromEnv } from "../disk-retention.js";
|
|
9
9
|
import { readDiskFootprint } from "../disk-usage.js";
|
|
@@ -13,7 +13,8 @@ import { runRawEvidenceLocalGc, rawEvidenceGcSummary } from "../raw-evidence-gc.
|
|
|
13
13
|
import { runBackfillCommand } from "./backfill.js";
|
|
14
14
|
import { diskRowMessage, mib, redeliveryLine } from "./doctor-disk-words.js";
|
|
15
15
|
import { doctorRoots } from "./doctor-access.js";
|
|
16
|
-
import {
|
|
16
|
+
import { backfillCompletionStepState, backfillFixVerdict, jsonField, parseDoctorBackfillJson, parseDoctorSyncJson, syncBacklogDrainingVerdict, syncStandAsideVerdict, } from "./doctor-pipeline-verdicts.js";
|
|
17
|
+
import { fail, needsFix, ok, skipped } from "./doctor-report.js";
|
|
17
18
|
/**
|
|
18
19
|
* The `backfill-complete`, `gc-checked`, `disk-bounded`, and `sync-fresh`
|
|
19
20
|
* check family: does the collection pipeline itself have everything it
|
|
@@ -22,33 +23,17 @@ import { asRecord, fail, needsFix, ok, skipped } from "./doctor-report.js";
|
|
|
22
23
|
* is committed, disk-bounded reads per-file against the upload ledger
|
|
23
24
|
* (BLI-3619), and sync-fresh is the one check whose "fix" is a live proof
|
|
24
25
|
* (`cockpit sync`) rather than a local computation.
|
|
25
|
-
*/
|
|
26
|
-
const GC_MIN_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
|
27
|
-
/**
|
|
28
|
-
* Pure so it can be unit-tested without touching the real machine's home
|
|
29
|
-
* directory (`getCollectorRuntimePaths()` defaults to `os.homedir()` and
|
|
30
|
-
* doctor never threads `--home` through the backfill steps). Returns `null`
|
|
31
|
-
* when the marker does not cover the roots/sources — the caller falls
|
|
32
|
-
* through to the lock/never-run diagnosis in that case.
|
|
33
26
|
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
27
|
+
* This module is the half that TOUCHES the machine: it runs the catch-up,
|
|
28
|
+
* prunes the disk, execs a sync. The verdicts those steps reach — what a
|
|
29
|
+
* completion marker proves, what a stand-aside receipt means, when an
|
|
30
|
+
* unfinished backfill is busy rather than broken — are pure functions in
|
|
31
|
+
* `doctor-pipeline-verdicts.ts`, and every one of them is re-exported from
|
|
32
|
+
* here so callers keep one address.
|
|
38
33
|
*/
|
|
39
|
-
|
|
40
|
-
if (!backfillCompletionCovers(marker, roots, ["codex", "claude_code"])) {
|
|
41
|
-
return null;
|
|
42
|
-
}
|
|
43
|
-
const oversized = marker?.oversized_skips;
|
|
44
|
-
if (oversized && oversized.count > 0) {
|
|
45
|
-
return ok("backfill-complete", "complete_with_oversized_skips", `caught up on old Codex and Claude sessions in every saved folder ` +
|
|
46
|
-
`(complete_with_oversized_skips · ${oversized.count} file${oversized.count === 1 ? "" : "s"} too big to upload)`);
|
|
47
|
-
}
|
|
48
|
-
return ok("backfill-complete", "complete", "caught up on old Codex and Claude sessions in every saved folder");
|
|
49
|
-
}
|
|
34
|
+
const GC_MIN_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
|
50
35
|
export async function checkBackfillState(context) {
|
|
51
|
-
const paths = getCollectorRuntimePaths();
|
|
36
|
+
const paths = getCollectorRuntimePaths(context.command.homeDir);
|
|
52
37
|
const roots = await doctorRoots(context);
|
|
53
38
|
const marker = await readBackfillCompletionMarker(paths);
|
|
54
39
|
const covered = backfillCompletionStepState(marker, roots);
|
|
@@ -62,15 +47,20 @@ export async function checkBackfillState(context) {
|
|
|
62
47
|
return needsFix("backfill-complete", cursor.updated_at ? "partial" : "never_run", "the catch-up over your old sessions has not finished");
|
|
63
48
|
}
|
|
64
49
|
export async function fixBackfillState(context) {
|
|
50
|
+
return withDoctorLockWait(context, () => runBackfillRepair(context));
|
|
51
|
+
}
|
|
52
|
+
async function runBackfillRepair(context) {
|
|
65
53
|
const capture = capturedIo(context.io, !context.command.json);
|
|
66
54
|
const code = await runBackfillCommand({
|
|
55
|
+
homeDir: context.command.homeDir,
|
|
67
56
|
repoRoot: context.command.repoRoot,
|
|
68
57
|
all: true,
|
|
69
58
|
dryRun: false,
|
|
70
59
|
yes: true,
|
|
71
60
|
json: true,
|
|
72
61
|
}, capture.io);
|
|
73
|
-
const
|
|
62
|
+
const stdout = capture.stdout();
|
|
63
|
+
const output = `${stdout}\n${capture.stderr()}`;
|
|
74
64
|
if (code === 0) {
|
|
75
65
|
// Re-read the marker this run just wrote instead of hand-rolling a second
|
|
76
66
|
// message: `checkBackfillState`'s pure core already knows how to say
|
|
@@ -79,19 +69,17 @@ export async function fixBackfillState(context) {
|
|
|
79
69
|
const recheck = await checkBackfillState(context);
|
|
80
70
|
if (recheck.status === "ok")
|
|
81
71
|
return recheck;
|
|
82
|
-
return
|
|
72
|
+
return recheck;
|
|
83
73
|
}
|
|
84
|
-
const
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
}
|
|
88
|
-
return fail("backfill-complete", reason ?? "backfill_failed", "backfill did not complete");
|
|
74
|
+
const verdict = backfillFixVerdict(parseDoctorBackfillJson(stdout), jsonField(output, "failure_reason"));
|
|
75
|
+
console.error("[cockpit-doctor] catch-up run did not finish", JSON.stringify({ reason: verdict.code, row_status: verdict.status, exit_code: code }));
|
|
76
|
+
return verdict;
|
|
89
77
|
}
|
|
90
78
|
export async function checkGcState(context) {
|
|
91
79
|
if (context.io.env["COCKPIT_DISABLE_GC"] === "1") {
|
|
92
80
|
return skipped("gc-checked", "skipped_disabled", "cleanup is switched off");
|
|
93
81
|
}
|
|
94
|
-
const paths = getCollectorRuntimePaths();
|
|
82
|
+
const paths = getCollectorRuntimePaths(context.command.homeDir);
|
|
95
83
|
const marker = path.join(paths.state_dir, ".last-raw-evidence-gc");
|
|
96
84
|
const info = await fs.stat(marker).catch(() => null);
|
|
97
85
|
if (info && Date.now() - info.mtimeMs < GC_MIN_INTERVAL_MS) {
|
|
@@ -100,7 +88,7 @@ export async function checkGcState(context) {
|
|
|
100
88
|
return needsFix("gc-checked", "due", "cleanup is due");
|
|
101
89
|
}
|
|
102
90
|
export async function fixGcState(context) {
|
|
103
|
-
const result = await runRawEvidenceLocalGc(getCollectorRuntimePaths(), context.io.env);
|
|
91
|
+
const result = await runRawEvidenceLocalGc(getCollectorRuntimePaths(context.command.homeDir), context.io.env);
|
|
104
92
|
if (result.skipped) {
|
|
105
93
|
return skipped("gc-checked", "skipped_throttled", "cleanup already ran today");
|
|
106
94
|
}
|
|
@@ -196,11 +184,6 @@ function captureStream(target, chunks, forward) {
|
|
|
196
184
|
},
|
|
197
185
|
};
|
|
198
186
|
}
|
|
199
|
-
function jsonField(output, field) {
|
|
200
|
-
const escaped = field.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
|
201
|
-
const match = output.match(new RegExp(`"${escaped}"\\s*:\\s*"([^"]+)"`, "u"));
|
|
202
|
-
return match?.[1] ?? null;
|
|
203
|
-
}
|
|
204
187
|
export async function checkSyncState(context) {
|
|
205
188
|
const roots = await doctorRoots(context);
|
|
206
189
|
if (roots.length === 0) {
|
|
@@ -211,76 +194,10 @@ export async function checkSyncState(context) {
|
|
|
211
194
|
// and only turns green from those command receipts.
|
|
212
195
|
return needsFix("sync-fresh", "per_root_verification_required", `fresh upload proof is required for ${roots.length} saved root${roots.length === 1 ? "" : "s"}`);
|
|
213
196
|
}
|
|
214
|
-
/**
|
|
215
|
-
* `cockpit sync --json` prints exactly one JSON document to stdout (stderr is
|
|
216
|
-
* for human text; see AGENTS.md logging conventions), so this is a real parse
|
|
217
|
-
* rather than the doctor module's usual regex field-scrape — which cannot
|
|
218
|
-
* disambiguate same-named fields nested under `codex_sessions.codex` vs
|
|
219
|
-
* `codex_sessions.claude` (BLI-2728).
|
|
220
|
-
*/
|
|
221
|
-
function parseDoctorSyncJson(stdout) {
|
|
222
|
-
try {
|
|
223
|
-
const parsed = JSON.parse(stdout.trim());
|
|
224
|
-
return parsed && typeof parsed === "object" ? parsed : null;
|
|
225
|
-
}
|
|
226
|
-
catch (error) {
|
|
227
|
-
// `null` sends doctor back to its regex field-scrape, quietly losing the
|
|
228
|
-
// BLI-2728 disambiguation. Something wrote to stdout that was not the one
|
|
229
|
-
// JSON document the contract promises — a stray console.log in the
|
|
230
|
-
// collector would do exactly this and look like nothing at all.
|
|
231
|
-
console.error("[cockpit-doctor] sync --json stdout was not one JSON document", JSON.stringify({
|
|
232
|
-
reason: "sync_json_unparseable",
|
|
233
|
-
byte_size: stdout.length,
|
|
234
|
-
...describeError(error),
|
|
235
|
-
}));
|
|
236
|
-
return null;
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
/**
|
|
240
|
-
* BLI-2728: a tick that only deferred objects past the per-tick raw-evidence
|
|
241
|
-
* object budget (`RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET`, adapters/raw-evidence.ts)
|
|
242
|
-
* is a backlog that is draining, not a failure — `attributedSyncRunStatus`
|
|
243
|
-
* marks the run not-fully-`ok` (so `cockpit sync` exits non-zero and doctor's
|
|
244
|
-
* exec sees `code !== 0`) purely because objects remain queued, with the
|
|
245
|
-
* per-repo upload itself still having succeeded. A genuine failure (auth,
|
|
246
|
-
* server rejection, network, a real upload_failed outcome, an unposted
|
|
247
|
-
* session report) must still read red — this only fires when NOTHING else in
|
|
248
|
-
* the tick's own summary looks wrong. Pure so it is unit-testable without a
|
|
249
|
-
* live exec/fs harness; the remaining-object count is read straight from the
|
|
250
|
-
* tick's own summary, never recomputed.
|
|
251
|
-
*/
|
|
252
|
-
export function syncBacklogDrainingVerdict(parsed) {
|
|
253
|
-
if (!parsed)
|
|
254
|
-
return null;
|
|
255
|
-
const deferredObjects = positiveNumberOrZero(parsed.raw_evidence_deferred_object_budget);
|
|
256
|
-
if (deferredObjects <= 0)
|
|
257
|
-
return null;
|
|
258
|
-
const deferredBytes = positiveNumberOrZero(parsed.raw_evidence_deferred_byte_budget);
|
|
259
|
-
const failedCount = positiveNumberOrZero(parsed.raw_evidence_failed_count);
|
|
260
|
-
const retryReasons = Array.isArray(parsed.raw_evidence_retry_reasons)
|
|
261
|
-
? parsed.raw_evidence_retry_reasons.length
|
|
262
|
-
: 0;
|
|
263
|
-
const sessions = asRecord(parsed.codex_sessions);
|
|
264
|
-
const reportPosted = sessions?.["report_posted"];
|
|
265
|
-
const codexReadFailures = positiveNumberOrZero(asRecord(sessions?.["codex"])?.["read_failures"]);
|
|
266
|
-
const claudeSessions = asRecord(sessions?.["claude"]);
|
|
267
|
-
const claudeReadFailures = positiveNumberOrZero(claudeSessions?.["read_failures"]);
|
|
268
|
-
const claudeSidecarsFailed = positiveNumberOrZero(claudeSessions?.["sidecars_failed"]);
|
|
269
|
-
const onlyDeferredObjectBudget = deferredBytes === 0 &&
|
|
270
|
-
failedCount === 0 &&
|
|
271
|
-
retryReasons === 0 &&
|
|
272
|
-
reportPosted === true &&
|
|
273
|
-
codexReadFailures === 0 &&
|
|
274
|
-
claudeReadFailures === 0 &&
|
|
275
|
-
claudeSidecarsFailed === 0;
|
|
276
|
-
return onlyDeferredObjectBudget ? { remainingObjects: deferredObjects } : null;
|
|
277
|
-
}
|
|
278
|
-
function positiveNumberOrZero(value) {
|
|
279
|
-
return typeof value === "number" && Number.isFinite(value) && value > 0
|
|
280
|
-
? value
|
|
281
|
-
: 0;
|
|
282
|
-
}
|
|
283
197
|
export async function fixSyncState(context) {
|
|
198
|
+
return withDoctorLockWait(context, () => runSyncRepair(context));
|
|
199
|
+
}
|
|
200
|
+
async function runSyncRepair(context) {
|
|
284
201
|
const exec = context.io.exec;
|
|
285
202
|
if (!exec)
|
|
286
203
|
return fail("sync-fresh", "runner_unavailable", "sync runner unavailable");
|
|
@@ -294,6 +211,8 @@ export async function fixSyncState(context) {
|
|
|
294
211
|
const discoveryArgs = await savedDiscoveryLimitArgs(context.command.homeDir);
|
|
295
212
|
for (const repoRoot of roots) {
|
|
296
213
|
const args = ["sync", "--json", "--workspace", repoRoot, ...discoveryArgs];
|
|
214
|
+
if (context.command.homeDir)
|
|
215
|
+
args.push("--home", context.command.homeDir);
|
|
297
216
|
if (context.command.dashboardUrl !== DEFAULT_DASHBOARD_URL) {
|
|
298
217
|
args.push("--dashboard-url", context.command.dashboardUrl);
|
|
299
218
|
}
|
|
@@ -301,6 +220,21 @@ export async function fixSyncState(context) {
|
|
|
301
220
|
const parsed = parseDoctorSyncJson(result.stdout);
|
|
302
221
|
const output = `${result.stdout}\n${result.stderr}`;
|
|
303
222
|
const status = jsonField(output, "status");
|
|
223
|
+
// BLI-4303, before any verdict: a sync that stood aside for the collection
|
|
224
|
+
// lock never ran, so neither its exit code nor its missing `uploaded`
|
|
225
|
+
// receipt says anything about this machine's health.
|
|
226
|
+
const standAside = syncStandAsideVerdict(parsed, status);
|
|
227
|
+
if (standAside) {
|
|
228
|
+
console.error("[cockpit-doctor] sync stood aside for a running collection run", JSON.stringify({
|
|
229
|
+
reason: standAside.code,
|
|
230
|
+
owner_alive: standAside.ownerAlive,
|
|
231
|
+
held_since: standAside.heldSince,
|
|
232
|
+
}));
|
|
233
|
+
return needsFix("sync-fresh", standAside.code, "waiting for the collection lock");
|
|
234
|
+
}
|
|
235
|
+
if (status === "live_sync_paused_during_backfill") {
|
|
236
|
+
return needsFix("sync-fresh", status, "waiting for historical catch-up");
|
|
237
|
+
}
|
|
304
238
|
if (result.code !== 0) {
|
|
305
239
|
const draining = syncBacklogDrainingVerdict(parsed);
|
|
306
240
|
if (draining) {
|
|
@@ -317,4 +251,8 @@ export async function fixSyncState(context) {
|
|
|
317
251
|
return ok("sync-fresh", "synced", roots.length === 1
|
|
318
252
|
? "ran `cockpit sync` and received an uploaded receipt"
|
|
319
253
|
: `received uploaded receipts for ${roots.length} saved roots`);
|
|
320
|
-
}
|
|
254
|
+
}
|
|
255
|
+
// Re-exported so `doctor.ts`, the doctor tests and every other caller keep
|
|
256
|
+
// importing these from `./doctor-pipeline.js` regardless of which sibling
|
|
257
|
+
// computes them.
|
|
258
|
+
export { backfillCompletionStepState, backfillFixVerdict, isCollectionBusyReason, syncBacklogDrainingVerdict, syncStandAsideVerdict, } from "./doctor-pipeline-verdicts.js";
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { inspectAgentRules, installAgentRules } from "../agent-rules.js";
|
|
2
|
+
import { doctorRoots } from "./doctor-access.js";
|
|
1
3
|
import { installAutostartAgent } from "../autostart.js";
|
|
2
4
|
import { fail, needsFix, ok, skipped } from "./doctor-report.js";
|
|
3
5
|
import { autostartDoctorStatus, autostartRegistrationRoots, readAutostartRegistration, } from "./autostart-reading.js";
|
|
@@ -38,6 +40,8 @@ export async function fixAutostartState(context) {
|
|
|
38
40
|
// fix that registers a different boundary than the check inspected would
|
|
39
41
|
// leave the row red forever (BLI-3793).
|
|
40
42
|
const roots = await autostartRegistrationRoots(context.command.homeDir, context.command.repoRoot);
|
|
43
|
+
if (roots.length === 0)
|
|
44
|
+
return needsFix("autostart-alive", "no_roots", "Run `cockpit doctor` to choose approved collection roots first.");
|
|
41
45
|
const result = await installAutostartAgent({
|
|
42
46
|
...(roots[0] ? { repoRoot: roots[0] } : {}),
|
|
43
47
|
repoRoots: roots,
|
|
@@ -67,7 +71,13 @@ export async function checkMemoryState(context) {
|
|
|
67
71
|
return memoryStepState(outcome, "check");
|
|
68
72
|
}
|
|
69
73
|
export async function fixMemoryState(context) {
|
|
70
|
-
|
|
74
|
+
let outcome = await installMemoryIntegration(memoryCommandFor(context), context.io);
|
|
75
|
+
if (!outcome.bin_found) {
|
|
76
|
+
// The public CLI declares the memory and Tower MCP packages as dependencies.
|
|
77
|
+
// Reinstall through the existing updater to restore missing package files.
|
|
78
|
+
await context.deps.selfUpdate(context.io, { json: true, tag: context.command.updateTag });
|
|
79
|
+
outcome = await installMemoryIntegration(memoryCommandFor(context), context.io);
|
|
80
|
+
}
|
|
71
81
|
return memoryStepState(outcome, "fix");
|
|
72
82
|
}
|
|
73
83
|
function memoryCommandFor(context) {
|
|
@@ -95,7 +105,18 @@ function memoryStepState(outcome, phase) {
|
|
|
95
105
|
return needsFix("memory-registered", "registration_incomplete", `BLI Memory is not registered with ${pending.map((target) => target.target).join(", ")}`);
|
|
96
106
|
}
|
|
97
107
|
if (!outcome.bin_found) {
|
|
98
|
-
return
|
|
108
|
+
return needsFix("memory-registered", "bin_missing", "bli-memory-mcp is not on this machine yet; nothing was written and the next run will try again");
|
|
99
109
|
}
|
|
100
110
|
return ok("memory-registered", phase === "fix" ? "installed" : "already_installed", "BLI Memory is registered with both agent hosts");
|
|
111
|
+
}
|
|
112
|
+
export async function checkDoctorAgentRules(context) {
|
|
113
|
+
const result = await inspectAgentRules({ homeDir: context.command.homeDir, scopePaths: await doctorRoots(context) });
|
|
114
|
+
return result.installed ? ok("agent-rules", "installed", "agent rules are installed for both hosts") : needsFix("agent-rules", "missing", "run `cockpit agent-rules install`");
|
|
115
|
+
}
|
|
116
|
+
export async function fixDoctorAgentRules(context) {
|
|
117
|
+
const roots = await doctorRoots(context);
|
|
118
|
+
if (roots.length === 0)
|
|
119
|
+
return needsFix("agent-rules", "no_roots", "Run `cockpit doctor` to choose approved roots first.");
|
|
120
|
+
await installAgentRules({ homeDir: context.command.homeDir, scopePaths: roots });
|
|
121
|
+
return checkDoctorAgentRules(context);
|
|
101
122
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { setupReceiptPieces } from "@bli-cockpit/telemetry-core";
|
|
1
2
|
import { redactedHealthDetail } from "../health-detail.js";
|
|
2
3
|
import { setupReceiptBlock } from "./setup-receipt-lines.js";
|
|
3
4
|
export function ok(id, code, message) {
|
|
@@ -28,9 +29,9 @@ export function isInteractiveDoctorFix(context) {
|
|
|
28
29
|
return !context.command.json && Boolean(context.io.stdin.isTTY);
|
|
29
30
|
}
|
|
30
31
|
export async function maybeReportDoctorEvents(context, rows) {
|
|
31
|
-
if (context.command.dryRun)
|
|
32
|
-
return;
|
|
33
|
-
|
|
32
|
+
if (context.command.dryRun || context.command.checkOnly)
|
|
33
|
+
return null;
|
|
34
|
+
return context.deps.reportInstallEvents({
|
|
34
35
|
dashboardUrl: context.command.dashboardUrl,
|
|
35
36
|
command: "doctor",
|
|
36
37
|
events: rows.map(doctorEvent),
|
|
@@ -40,14 +41,17 @@ export async function maybeReportDoctorEvents(context, rows) {
|
|
|
40
41
|
}
|
|
41
42
|
export function writeDoctorOutput(command, io, rows,
|
|
42
43
|
/** BLI-3731. Absent means it could not be read; doctor says so rather than nothing. */
|
|
43
|
-
setupReceipt) {
|
|
44
|
+
setupReceipt, repairs = []) {
|
|
45
|
+
const needsPerson = doctorNeedsPerson(rows, setupReceipt);
|
|
44
46
|
if (command.json) {
|
|
45
47
|
writeLine(io.stdout, JSON.stringify({
|
|
46
|
-
status:
|
|
48
|
+
status: needsPerson.length > 0
|
|
47
49
|
? "blocked"
|
|
48
50
|
: "pass",
|
|
49
51
|
dry_run: command.dryRun,
|
|
50
52
|
steps: rows,
|
|
53
|
+
repairs,
|
|
54
|
+
needs_person: needsPerson,
|
|
51
55
|
setup_receipt: setupReceipt?.receipt ?? null,
|
|
52
56
|
memory_install: setupReceipt?.memory ?? null,
|
|
53
57
|
}, null, 2));
|
|
@@ -58,13 +62,13 @@ setupReceipt) {
|
|
|
58
62
|
// sentence, not the label (BLI-3194).
|
|
59
63
|
writeLine(io.stdout, "state step result");
|
|
60
64
|
for (const row of rows) {
|
|
61
|
-
writeLine(io.stdout, `${doctorMark(row)} ${row.id.padEnd(20)} ${oneLine(row.message)}`);
|
|
65
|
+
writeLine(io.stdout, `${doctorMark(row)} ${row.id.padEnd(20)} ${oneLine(row.message).replaceAll("—", ";")}`);
|
|
62
66
|
}
|
|
63
67
|
const explanations = rows.filter((row) => (row.hardStop || row.status === "fail") && row.message.includes("\n"));
|
|
64
68
|
for (const row of explanations) {
|
|
65
69
|
writeLine(io.stderr, "");
|
|
66
70
|
writeLine(io.stderr, `${row.id}:`);
|
|
67
|
-
writeLine(io.stderr, row.message);
|
|
71
|
+
writeLine(io.stderr, row.message.replaceAll("—", ";"));
|
|
68
72
|
}
|
|
69
73
|
// BLI-3731. The invariant table says whether collection is healthy; this
|
|
70
74
|
// says whether the machine is CONNECTED — the browser sign-in, the device
|
|
@@ -75,11 +79,14 @@ setupReceipt) {
|
|
|
75
79
|
for (const line of setupReceipt
|
|
76
80
|
? setupReceiptBlock(setupReceipt, { indent: " " })
|
|
77
81
|
: [" unknown — this machine could not be read this run."]) {
|
|
78
|
-
writeLine(io.stdout, line);
|
|
82
|
+
writeLine(io.stdout, line.replaceAll("—", ";"));
|
|
79
83
|
}
|
|
84
|
+
writeLine(io.stdout, needsPerson.length === 0 ? "Everything is fixed." : `${needsPerson.length} things still need you:`);
|
|
85
|
+
for (const item of needsPerson)
|
|
86
|
+
writeLine(io.stdout, ` ${item.step}: ${item.action}`);
|
|
80
87
|
}
|
|
81
88
|
function doctorEvent(row) {
|
|
82
|
-
const status = row.status === "fail" || row.hardStop
|
|
89
|
+
const status = row.status === "fail" || row.status === "needs_fix" || row.hardStop
|
|
83
90
|
? "fail"
|
|
84
91
|
: row.status === "skipped"
|
|
85
92
|
? "skipped"
|
|
@@ -124,4 +131,36 @@ export function asRecord(value) {
|
|
|
124
131
|
}
|
|
125
132
|
export function writeLine(stream, text) {
|
|
126
133
|
stream.write(`${text}\n`);
|
|
134
|
+
}
|
|
135
|
+
export function doctorNeedsPerson(rows, receipt) {
|
|
136
|
+
const defaults = {
|
|
137
|
+
"cli-latest": "npm install -g @bli-cockpit/cli@latest",
|
|
138
|
+
"authed": "cockpit login",
|
|
139
|
+
"roots-ok": "cockpit doctor",
|
|
140
|
+
"single-install": "Run npm uninstall -g @bli-cockpit/cli with the Node installation that owns the extra CLI listed above.",
|
|
141
|
+
"autostart-alive": "cockpit autostart install",
|
|
142
|
+
"memory-registered": "cockpit memory install",
|
|
143
|
+
"agent-rules": "cockpit agent-rules install",
|
|
144
|
+
"mcp-answers": "cockpit doctor",
|
|
145
|
+
"memory-daemon": "cockpit memory install",
|
|
146
|
+
"backfill-complete": "cockpit backfill --all --yes",
|
|
147
|
+
"sync-fresh": "cockpit sync --json",
|
|
148
|
+
"gc-checked": "cockpit doctor",
|
|
149
|
+
"disk-bounded": "cockpit clean --all-committed",
|
|
150
|
+
};
|
|
151
|
+
const items = rows.filter((row) => row.status === "fail" || row.status === "needs_fix" || row.hardStop).map((row) => ({
|
|
152
|
+
step: row.id, reason: row.code,
|
|
153
|
+
action: row.nextAction ?? row.message.match(/`([^`]+)`/u)?.[1] ?? defaults[row.id],
|
|
154
|
+
}));
|
|
155
|
+
if (receipt) {
|
|
156
|
+
for (const { key, piece } of setupReceiptPieces(receipt.receipt, receipt.memory)) {
|
|
157
|
+
if (piece.status === "ok" || piece.status === "skipped" || piece.status === "unsupported")
|
|
158
|
+
continue;
|
|
159
|
+
const step = key === "device" || key === "browser" ? "authed" : key === "collector.autostart" ? "autostart-alive" : "memory-registered";
|
|
160
|
+
if (piece.status !== "needs_trust" && items.some((item) => item.step === step))
|
|
161
|
+
continue;
|
|
162
|
+
items.push({ step: key, reason: piece.reason ?? piece.status, action: piece.status === "needs_trust" ? "Open Codex and run /hooks to trust the installed hooks." : defaults[step] });
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
return items;
|
|
127
166
|
}
|