@bli-cockpit/cli 0.2.99 → 0.2.101

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/dist/agent-rules.js +2 -1
  2. package/dist/backfill-lock.js +1 -1
  3. package/dist/commands/backfill-checkpoint.js +3 -1
  4. package/dist/commands/backfill-issues.js +8 -55
  5. package/dist/commands/backfill-report.js +22 -6
  6. package/dist/commands/backfill-scan.js +2 -1
  7. package/dist/commands/backfill-skip-policy.js +134 -0
  8. package/dist/commands/careers.js +16 -0
  9. package/dist/commands/doctor-access.js +34 -10
  10. package/dist/commands/doctor-lock-wait.js +46 -0
  11. package/dist/commands/doctor-pipeline-verdicts.js +238 -0
  12. package/dist/commands/doctor-pipeline.js +49 -111
  13. package/dist/commands/doctor-registration.js +23 -2
  14. package/dist/commands/doctor-report.js +48 -9
  15. package/dist/commands/doctor-update.js +16 -5
  16. package/dist/commands/doctor.js +98 -58
  17. package/dist/commands/local-args-collector-setup.js +6 -0
  18. package/dist/commands/local-args-tower-careers.js +20 -0
  19. package/dist/commands/local-args-tower-pages.js +17 -2
  20. package/dist/commands/local-args-tower-usage.js +2 -2
  21. package/dist/commands/local-args-tower.js +2 -1
  22. package/dist/commands/local-args.js +3 -1
  23. package/dist/commands/local-help-commands-tower.js +4 -2
  24. package/dist/commands/local-help-commands.js +28 -12
  25. package/dist/commands/local-help.js +4 -2
  26. package/dist/commands/local.js +4 -0
  27. package/dist/commands/notes-file.js +8 -1
  28. package/dist/commands/notes-folders.js +35 -0
  29. package/dist/commands/notes-writes.js +37 -4
  30. package/dist/commands/notes.js +6 -0
  31. package/dist/commands/public-root.js +4 -4
  32. package/dist/commands/usage-format.js +18 -0
  33. package/dist/commands/usage.js +13 -3
  34. package/dist/cursors/backfill-completion-marker.js +135 -0
  35. package/dist/cursors/backfill-cursor.js +18 -99
  36. package/dist/scheduled-self-update.js +1 -1
  37. package/dist/sync-lock.js +15 -1
  38. package/package.json +2 -2
@@ -0,0 +1,238 @@
1
+ /**
2
+ * The pure verdicts the pipeline rows compute, and nothing else.
3
+ *
4
+ * `doctor-pipeline.ts` is the four checks that TOUCH the machine (run the
5
+ * catch-up, prune the disk, exec a sync). Everything here is the opposite: a
6
+ * function that takes a receipt or a marker someone else read and answers
7
+ * "what does that mean for the row". They live apart so each verdict can be
8
+ * unit-tested against a payload recorded off a real machine, with no exec, no
9
+ * filesystem and no home directory.
10
+ *
11
+ * Read it as four answers:
12
+ *
13
+ * isCollectionBusyReason another run owns the lock; nothing is broken
14
+ * backfillCompletionStepState what an all-history marker proves
15
+ * backfillFixVerdict busy vs progress vs actually broken
16
+ * syncBacklogDrainingVerdict a backlog draining is not a failed tick
17
+ * syncStandAsideVerdict is the lock's owner still alive
18
+ *
19
+ * Every name here is re-exported from `./doctor-pipeline.js`, the address its
20
+ * callers already know.
21
+ */
22
+ import { backfillCompletionCovers } from "../cursors/backfill-cursor.js";
23
+ import { describeError } from "../health-detail.js";
24
+ import { SYNC_LOCK_STALE_TAKEOVER_MS } from "../sync-lock.js";
25
+ import { asRecord, fail, needsFix, ok } from "./doctor-report.js";
26
+ /**
27
+ * BLI-4303: the reasons a collection command stood aside because ANOTHER
28
+ * collection run already owned the machine's single collection lock.
29
+ *
30
+ * `backfill` reports them as `failure_reason`, `sync` as its top-level
31
+ * `status`; both exit 0 and both say `retryable: true`. On a machine whose
32
+ * launchd job also watches `~/.claude/projects` and `~/.codex/sessions`, a tick
33
+ * is in flight most of the time an agent is working — the reference Mac held
34
+ * the lock in 76 of 89 samples over three minutes — so doctor's own repair
35
+ * almost always met a running sync and called it a failure. A busy machine is
36
+ * the collector working, never a machine to repair: these produce a named
37
+ * non-red row and the next run proves the receipt.
38
+ */
39
+ const COLLECTION_BUSY_REASONS = new Set([
40
+ // `cockpit sync` stood aside for another sync.
41
+ "sync_already_running",
42
+ // `cockpit sync` stood aside for a running backfill.
43
+ "live_sync_paused_during_backfill",
44
+ // `cockpit backfill` stood aside for another backfill.
45
+ "backfill_already_running",
46
+ ]);
47
+ export function isCollectionBusyReason(reason) {
48
+ return typeof reason === "string" && COLLECTION_BUSY_REASONS.has(reason);
49
+ }
50
+ /**
51
+ * Pure so it can be unit-tested without touching the real machine's home
52
+ * directory (`getCollectorRuntimePaths()` defaults to `os.homedir()` and
53
+ * doctor never threads `--home` through the backfill steps). Returns `null`
54
+ * when the marker does not cover the roots/sources — the caller falls
55
+ * through to the lock/never-run diagnosis in that case.
56
+ *
57
+ * BLI-2727/BLI-4303: a marker whose only outstanding entries are deterministic
58
+ * skips (files over the upload cap, helper transcripts over the per-session
59
+ * sidecar cap) is still a completed backfill. It reads green with a named
60
+ * note, never a red `needs_fix`/`fail`, so a cap nobody can lift from here
61
+ * never reads as "backfill never completed" on repeat doctor runs.
62
+ */
63
+ export function backfillCompletionStepState(marker, roots) {
64
+ if (!backfillCompletionCovers(marker, roots, ["codex", "claude_code"])) {
65
+ return null;
66
+ }
67
+ const caughtUp = "caught up on old Codex and Claude sessions in every saved folder";
68
+ const oversized = marker?.oversized_skips;
69
+ const sidecarCaps = marker?.sidecar_cap_skips;
70
+ const notes = [];
71
+ if (oversized && oversized.count > 0) {
72
+ notes.push(`${oversized.count} file${oversized.count === 1 ? "" : "s"} too big to upload`);
73
+ }
74
+ // BLI-4303: a cap is named, never silent. It no longer blocks completion, so
75
+ // the green row is the only place a person would ever learn it fired.
76
+ if (sidecarCaps && sidecarCaps.count > 0) {
77
+ notes.push(`${sidecarCaps.count} helper transcript${sidecarCaps.count === 1 ? "" : "s"} past the per-session cap`);
78
+ }
79
+ if (notes.length === 0)
80
+ return ok("backfill-complete", "complete", caughtUp);
81
+ const code = oversized && sidecarCaps
82
+ ? "complete_with_capped_skips"
83
+ : oversized
84
+ ? "complete_with_oversized_skips"
85
+ : "complete_with_sidecar_caps";
86
+ return ok("backfill-complete", code, `${caughtUp} (${code} · ${notes.join(", ")})`);
87
+ }
88
+ /**
89
+ * `cockpit backfill --json` prints exactly one JSON document to stdout. Parsed
90
+ * rather than field-scraped for the same reason sync is (BLI-2728): `counts`
91
+ * and `batches` both carry a `failed`, and a regex cannot tell them apart.
92
+ */
93
+ export function parseDoctorBackfillJson(stdout) {
94
+ try {
95
+ const parsed = JSON.parse(stdout.trim());
96
+ return parsed && typeof parsed === "object" ? parsed : null;
97
+ }
98
+ catch (error) {
99
+ console.error("[cockpit-doctor] backfill --json stdout was not one JSON document", JSON.stringify({
100
+ reason: "backfill_json_unparseable",
101
+ byte_size: stdout.length,
102
+ ...describeError(error),
103
+ }));
104
+ return null;
105
+ }
106
+ }
107
+ /**
108
+ * BLI-4303: what a backfill that did not reach `complete` means for the row.
109
+ *
110
+ * Three different machines used to read `❌ backfill did not complete`, and
111
+ * only one of them was broken:
112
+ *
113
+ * busy another collection run owned the lock, so this one never started
114
+ * progress it uploaded history and left more of it for the next run
115
+ * broken it could not do the work — auth, the server, a read failure
116
+ *
117
+ * Only the third is a `fail`. The first two are `needs_fix`, which is the same
118
+ * word `checkBackfillState` already uses for an unfinished catch-up: the fix
119
+ * side had been the only place that called an unfinished backfill a failure.
120
+ * Pure so the three verdicts are testable off a recorded `--json` payload.
121
+ */
122
+ export function backfillFixVerdict(parsed, scrapedReason) {
123
+ const reason = (typeof parsed?.failure_reason === "string" ? parsed.failure_reason : null) ??
124
+ scrapedReason;
125
+ if (isCollectionBusyReason(reason)) {
126
+ return needsFix("backfill-complete", reason ?? "collection_busy", "another collection run is using your sessions right now, so the catch-up " +
127
+ "stood aside; it runs on the next `cockpit doctor` or `cockpit backfill --all --yes`");
128
+ }
129
+ const counts = asRecord(parsed?.counts);
130
+ const batches = asRecord(parsed?.batches);
131
+ const backfilled = positiveNumberOrZero(counts?.["backfilled"]);
132
+ const remaining = positiveNumberOrZero(counts?.["remaining"]);
133
+ const completedBatches = positiveNumberOrZero(batches?.["completed"]);
134
+ if (backfilled > 0 || completedBatches > 0) {
135
+ return needsFix("backfill-complete", reason ?? "backfill_incomplete", `caught up on ${backfilled} old session${backfilled === 1 ? "" : "s"} this run` +
136
+ (remaining > 0 ? `, ${remaining} still to go` : "") +
137
+ "; run `cockpit backfill --all --yes` again to continue");
138
+ }
139
+ return fail("backfill-complete", reason ?? "backfill_failed", "backfill did not complete");
140
+ }
141
+ export function jsonField(output, field) {
142
+ const escaped = field.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
143
+ const match = output.match(new RegExp(`"${escaped}"\\s*:\\s*"([^"]+)"`, "u"));
144
+ return match?.[1] ?? null;
145
+ }
146
+ /**
147
+ * `cockpit sync --json` prints exactly one JSON document to stdout (stderr is
148
+ * for human text; see AGENTS.md logging conventions), so this is a real parse
149
+ * rather than the doctor module's usual regex field-scrape — which cannot
150
+ * disambiguate same-named fields nested under `codex_sessions.codex` vs
151
+ * `codex_sessions.claude` (BLI-2728).
152
+ */
153
+ export function parseDoctorSyncJson(stdout) {
154
+ try {
155
+ const parsed = JSON.parse(stdout.trim());
156
+ return parsed && typeof parsed === "object" ? parsed : null;
157
+ }
158
+ catch (error) {
159
+ // `null` sends doctor back to its regex field-scrape, quietly losing the
160
+ // BLI-2728 disambiguation. Something wrote to stdout that was not the one
161
+ // JSON document the contract promises — a stray console.log in the
162
+ // collector would do exactly this and look like nothing at all.
163
+ console.error("[cockpit-doctor] sync --json stdout was not one JSON document", JSON.stringify({
164
+ reason: "sync_json_unparseable",
165
+ byte_size: stdout.length,
166
+ ...describeError(error),
167
+ }));
168
+ return null;
169
+ }
170
+ }
171
+ /**
172
+ * BLI-2728: a tick that only deferred objects past the per-tick raw-evidence
173
+ * object budget (`RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET`, adapters/raw-evidence.ts)
174
+ * is a backlog that is draining, not a failure — `attributedSyncRunStatus`
175
+ * marks the run not-fully-`ok` (so `cockpit sync` exits non-zero and doctor's
176
+ * exec sees `code !== 0`) purely because objects remain queued, with the
177
+ * per-repo upload itself still having succeeded. A genuine failure (auth,
178
+ * server rejection, network, a real upload_failed outcome, an unposted
179
+ * session report) must still read red — this only fires when NOTHING else in
180
+ * the tick's own summary looks wrong. Pure so it is unit-testable without a
181
+ * live exec/fs harness; the remaining-object count is read straight from the
182
+ * tick's own summary, never recomputed.
183
+ */
184
+ export function syncBacklogDrainingVerdict(parsed) {
185
+ if (!parsed)
186
+ return null;
187
+ const deferredObjects = positiveNumberOrZero(parsed.raw_evidence_deferred_object_budget);
188
+ if (deferredObjects <= 0)
189
+ return null;
190
+ const deferredBytes = positiveNumberOrZero(parsed.raw_evidence_deferred_byte_budget);
191
+ const failedCount = positiveNumberOrZero(parsed.raw_evidence_failed_count);
192
+ const retryReasons = Array.isArray(parsed.raw_evidence_retry_reasons)
193
+ ? parsed.raw_evidence_retry_reasons.length
194
+ : 0;
195
+ const sessions = asRecord(parsed.codex_sessions);
196
+ const reportPosted = sessions?.["report_posted"];
197
+ const codexReadFailures = positiveNumberOrZero(asRecord(sessions?.["codex"])?.["read_failures"]);
198
+ const claudeSessions = asRecord(sessions?.["claude"]);
199
+ const claudeReadFailures = positiveNumberOrZero(claudeSessions?.["read_failures"]);
200
+ const claudeSidecarsFailed = positiveNumberOrZero(claudeSessions?.["sidecars_failed"]);
201
+ const onlyDeferredObjectBudget = deferredBytes === 0 &&
202
+ failedCount === 0 &&
203
+ retryReasons === 0 &&
204
+ reportPosted === true &&
205
+ codexReadFailures === 0 &&
206
+ claudeReadFailures === 0 &&
207
+ claudeSidecarsFailed === 0;
208
+ return onlyDeferredObjectBudget ? { remainingObjects: deferredObjects } : null;
209
+ }
210
+ /**
211
+ * BLI-4303: a sync that stood aside for whoever already owns the collection
212
+ * lock, and whether that owner is still alive.
213
+ *
214
+ * The receipt's `held_since` is the holder's last heartbeat, refreshed every
215
+ * 30 s by the running process itself, and a lock whose heartbeat goes older
216
+ * than `SYNC_LOCK_STALE_TAKEOVER_MS` is taken over by the next sync. So a
217
+ * heartbeat inside that window is a live collection run and the honest answer
218
+ * for the row is "this machine is collecting"; a heartbeat outside it (or none
219
+ * at all, which is what a lock file this machine could not create looks like —
220
+ * see `sync-lock.ts`) is a machine that owes a person a look.
221
+ *
222
+ * Pure so both verdicts are testable off a recorded receipt.
223
+ */
224
+ export function syncStandAsideVerdict(parsed, status, now = new Date()) {
225
+ const code = (typeof parsed?.status === "string" ? parsed.status : null) ?? status;
226
+ if (!isCollectionBusyReason(code))
227
+ return null;
228
+ const heldSince = typeof parsed?.held_since === "string" ? parsed.held_since : null;
229
+ const heartbeatMs = heldSince ? Date.parse(heldSince) : Number.NaN;
230
+ const ownerAlive = Number.isFinite(heartbeatMs) &&
231
+ now.getTime() - heartbeatMs <= SYNC_LOCK_STALE_TAKEOVER_MS;
232
+ return { code: code, heldSince, ownerAlive };
233
+ }
234
+ function positiveNumberOrZero(value) {
235
+ return typeof value === "number" && Number.isFinite(value) && value > 0
236
+ ? value
237
+ : 0;
238
+ }
@@ -1,9 +1,9 @@
1
+ import { withDoctorLockWait } from "./doctor-lock-wait.js";
1
2
  import fs from "node:fs/promises";
2
3
  import path from "node:path";
3
4
  import { inspectBackfillLock } from "../backfill-lock.js";
4
- import { backfillCompletionCovers, readBackfillCompletionMarker, readBackfillCursor, } from "../cursors/backfill-cursor.js";
5
+ import { readBackfillCompletionMarker, readBackfillCursor, } from "../cursors/backfill-cursor.js";
5
6
  import { savedDiscoveryLimitArgs } from "../discovery-limits.js";
6
- import { describeError } from "../health-detail.js";
7
7
  import { runStagingPrune } from "../disk-prune.js";
8
8
  import { retentionOptionsFromEnv } from "../disk-retention.js";
9
9
  import { readDiskFootprint } from "../disk-usage.js";
@@ -13,7 +13,8 @@ import { runRawEvidenceLocalGc, rawEvidenceGcSummary } from "../raw-evidence-gc.
13
13
  import { runBackfillCommand } from "./backfill.js";
14
14
  import { diskRowMessage, mib, redeliveryLine } from "./doctor-disk-words.js";
15
15
  import { doctorRoots } from "./doctor-access.js";
16
- import { asRecord, fail, needsFix, ok, skipped } from "./doctor-report.js";
16
+ import { backfillCompletionStepState, backfillFixVerdict, jsonField, parseDoctorBackfillJson, parseDoctorSyncJson, syncBacklogDrainingVerdict, syncStandAsideVerdict, } from "./doctor-pipeline-verdicts.js";
17
+ import { fail, needsFix, ok, skipped } from "./doctor-report.js";
17
18
  /**
18
19
  * The `backfill-complete`, `gc-checked`, `disk-bounded`, and `sync-fresh`
19
20
  * check family: does the collection pipeline itself have everything it
@@ -22,33 +23,17 @@ import { asRecord, fail, needsFix, ok, skipped } from "./doctor-report.js";
22
23
  * is committed, disk-bounded reads per-file against the upload ledger
23
24
  * (BLI-3619), and sync-fresh is the one check whose "fix" is a live proof
24
25
  * (`cockpit sync`) rather than a local computation.
25
- */
26
- const GC_MIN_INTERVAL_MS = 24 * 60 * 60 * 1000;
27
- /**
28
- * Pure so it can be unit-tested without touching the real machine's home
29
- * directory (`getCollectorRuntimePaths()` defaults to `os.homedir()` and
30
- * doctor never threads `--home` through the backfill steps). Returns `null`
31
- * when the marker does not cover the roots/sources — the caller falls
32
- * through to the lock/never-run diagnosis in that case.
33
26
  *
34
- * BLI-2727: a marker whose only outstanding entries are deterministic
35
- * oversized-file skips is still a completed backfill it reads green with a
36
- * named note, never a red `needs_fix`/`fail`, so an unliftable file cap never
37
- * reads as "backfill never completed" on repeat doctor runs.
27
+ * This module is the half that TOUCHES the machine: it runs the catch-up,
28
+ * prunes the disk, execs a sync. The verdicts those steps reach what a
29
+ * completion marker proves, what a stand-aside receipt means, when an
30
+ * unfinished backfill is busy rather than broken — are pure functions in
31
+ * `doctor-pipeline-verdicts.ts`, and every one of them is re-exported from
32
+ * here so callers keep one address.
38
33
  */
39
- export function backfillCompletionStepState(marker, roots) {
40
- if (!backfillCompletionCovers(marker, roots, ["codex", "claude_code"])) {
41
- return null;
42
- }
43
- const oversized = marker?.oversized_skips;
44
- if (oversized && oversized.count > 0) {
45
- return ok("backfill-complete", "complete_with_oversized_skips", `caught up on old Codex and Claude sessions in every saved folder ` +
46
- `(complete_with_oversized_skips · ${oversized.count} file${oversized.count === 1 ? "" : "s"} too big to upload)`);
47
- }
48
- return ok("backfill-complete", "complete", "caught up on old Codex and Claude sessions in every saved folder");
49
- }
34
+ const GC_MIN_INTERVAL_MS = 24 * 60 * 60 * 1000;
50
35
  export async function checkBackfillState(context) {
51
- const paths = getCollectorRuntimePaths();
36
+ const paths = getCollectorRuntimePaths(context.command.homeDir);
52
37
  const roots = await doctorRoots(context);
53
38
  const marker = await readBackfillCompletionMarker(paths);
54
39
  const covered = backfillCompletionStepState(marker, roots);
@@ -62,15 +47,20 @@ export async function checkBackfillState(context) {
62
47
  return needsFix("backfill-complete", cursor.updated_at ? "partial" : "never_run", "the catch-up over your old sessions has not finished");
63
48
  }
64
49
  export async function fixBackfillState(context) {
50
+ return withDoctorLockWait(context, () => runBackfillRepair(context));
51
+ }
52
+ async function runBackfillRepair(context) {
65
53
  const capture = capturedIo(context.io, !context.command.json);
66
54
  const code = await runBackfillCommand({
55
+ homeDir: context.command.homeDir,
67
56
  repoRoot: context.command.repoRoot,
68
57
  all: true,
69
58
  dryRun: false,
70
59
  yes: true,
71
60
  json: true,
72
61
  }, capture.io);
73
- const output = capture.stdout() + "\n" + capture.stderr();
62
+ const stdout = capture.stdout();
63
+ const output = `${stdout}\n${capture.stderr()}`;
74
64
  if (code === 0) {
75
65
  // Re-read the marker this run just wrote instead of hand-rolling a second
76
66
  // message: `checkBackfillState`'s pure core already knows how to say
@@ -79,19 +69,17 @@ export async function fixBackfillState(context) {
79
69
  const recheck = await checkBackfillState(context);
80
70
  if (recheck.status === "ok")
81
71
  return recheck;
82
- return ok("backfill-complete", "completed", "ran `cockpit backfill --all --yes`");
72
+ return recheck;
83
73
  }
84
- const reason = jsonField(output, "failure_reason");
85
- if (reason === "backfill_already_running") {
86
- return fail("backfill-complete", "backfill_already_running", "backfill lock is still held and no scope-valid completion marker exists");
87
- }
88
- return fail("backfill-complete", reason ?? "backfill_failed", "backfill did not complete");
74
+ const verdict = backfillFixVerdict(parseDoctorBackfillJson(stdout), jsonField(output, "failure_reason"));
75
+ console.error("[cockpit-doctor] catch-up run did not finish", JSON.stringify({ reason: verdict.code, row_status: verdict.status, exit_code: code }));
76
+ return verdict;
89
77
  }
90
78
  export async function checkGcState(context) {
91
79
  if (context.io.env["COCKPIT_DISABLE_GC"] === "1") {
92
80
  return skipped("gc-checked", "skipped_disabled", "cleanup is switched off");
93
81
  }
94
- const paths = getCollectorRuntimePaths();
82
+ const paths = getCollectorRuntimePaths(context.command.homeDir);
95
83
  const marker = path.join(paths.state_dir, ".last-raw-evidence-gc");
96
84
  const info = await fs.stat(marker).catch(() => null);
97
85
  if (info && Date.now() - info.mtimeMs < GC_MIN_INTERVAL_MS) {
@@ -100,7 +88,7 @@ export async function checkGcState(context) {
100
88
  return needsFix("gc-checked", "due", "cleanup is due");
101
89
  }
102
90
  export async function fixGcState(context) {
103
- const result = await runRawEvidenceLocalGc(getCollectorRuntimePaths(), context.io.env);
91
+ const result = await runRawEvidenceLocalGc(getCollectorRuntimePaths(context.command.homeDir), context.io.env);
104
92
  if (result.skipped) {
105
93
  return skipped("gc-checked", "skipped_throttled", "cleanup already ran today");
106
94
  }
@@ -196,11 +184,6 @@ function captureStream(target, chunks, forward) {
196
184
  },
197
185
  };
198
186
  }
199
- function jsonField(output, field) {
200
- const escaped = field.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
201
- const match = output.match(new RegExp(`"${escaped}"\\s*:\\s*"([^"]+)"`, "u"));
202
- return match?.[1] ?? null;
203
- }
204
187
  export async function checkSyncState(context) {
205
188
  const roots = await doctorRoots(context);
206
189
  if (roots.length === 0) {
@@ -211,76 +194,10 @@ export async function checkSyncState(context) {
211
194
  // and only turns green from those command receipts.
212
195
  return needsFix("sync-fresh", "per_root_verification_required", `fresh upload proof is required for ${roots.length} saved root${roots.length === 1 ? "" : "s"}`);
213
196
  }
214
- /**
215
- * `cockpit sync --json` prints exactly one JSON document to stdout (stderr is
216
- * for human text; see AGENTS.md logging conventions), so this is a real parse
217
- * rather than the doctor module's usual regex field-scrape — which cannot
218
- * disambiguate same-named fields nested under `codex_sessions.codex` vs
219
- * `codex_sessions.claude` (BLI-2728).
220
- */
221
- function parseDoctorSyncJson(stdout) {
222
- try {
223
- const parsed = JSON.parse(stdout.trim());
224
- return parsed && typeof parsed === "object" ? parsed : null;
225
- }
226
- catch (error) {
227
- // `null` sends doctor back to its regex field-scrape, quietly losing the
228
- // BLI-2728 disambiguation. Something wrote to stdout that was not the one
229
- // JSON document the contract promises — a stray console.log in the
230
- // collector would do exactly this and look like nothing at all.
231
- console.error("[cockpit-doctor] sync --json stdout was not one JSON document", JSON.stringify({
232
- reason: "sync_json_unparseable",
233
- byte_size: stdout.length,
234
- ...describeError(error),
235
- }));
236
- return null;
237
- }
238
- }
239
- /**
240
- * BLI-2728: a tick that only deferred objects past the per-tick raw-evidence
241
- * object budget (`RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET`, adapters/raw-evidence.ts)
242
- * is a backlog that is draining, not a failure — `attributedSyncRunStatus`
243
- * marks the run not-fully-`ok` (so `cockpit sync` exits non-zero and doctor's
244
- * exec sees `code !== 0`) purely because objects remain queued, with the
245
- * per-repo upload itself still having succeeded. A genuine failure (auth,
246
- * server rejection, network, a real upload_failed outcome, an unposted
247
- * session report) must still read red — this only fires when NOTHING else in
248
- * the tick's own summary looks wrong. Pure so it is unit-testable without a
249
- * live exec/fs harness; the remaining-object count is read straight from the
250
- * tick's own summary, never recomputed.
251
- */
252
- export function syncBacklogDrainingVerdict(parsed) {
253
- if (!parsed)
254
- return null;
255
- const deferredObjects = positiveNumberOrZero(parsed.raw_evidence_deferred_object_budget);
256
- if (deferredObjects <= 0)
257
- return null;
258
- const deferredBytes = positiveNumberOrZero(parsed.raw_evidence_deferred_byte_budget);
259
- const failedCount = positiveNumberOrZero(parsed.raw_evidence_failed_count);
260
- const retryReasons = Array.isArray(parsed.raw_evidence_retry_reasons)
261
- ? parsed.raw_evidence_retry_reasons.length
262
- : 0;
263
- const sessions = asRecord(parsed.codex_sessions);
264
- const reportPosted = sessions?.["report_posted"];
265
- const codexReadFailures = positiveNumberOrZero(asRecord(sessions?.["codex"])?.["read_failures"]);
266
- const claudeSessions = asRecord(sessions?.["claude"]);
267
- const claudeReadFailures = positiveNumberOrZero(claudeSessions?.["read_failures"]);
268
- const claudeSidecarsFailed = positiveNumberOrZero(claudeSessions?.["sidecars_failed"]);
269
- const onlyDeferredObjectBudget = deferredBytes === 0 &&
270
- failedCount === 0 &&
271
- retryReasons === 0 &&
272
- reportPosted === true &&
273
- codexReadFailures === 0 &&
274
- claudeReadFailures === 0 &&
275
- claudeSidecarsFailed === 0;
276
- return onlyDeferredObjectBudget ? { remainingObjects: deferredObjects } : null;
277
- }
278
- function positiveNumberOrZero(value) {
279
- return typeof value === "number" && Number.isFinite(value) && value > 0
280
- ? value
281
- : 0;
282
- }
283
197
  export async function fixSyncState(context) {
198
+ return withDoctorLockWait(context, () => runSyncRepair(context));
199
+ }
200
+ async function runSyncRepair(context) {
284
201
  const exec = context.io.exec;
285
202
  if (!exec)
286
203
  return fail("sync-fresh", "runner_unavailable", "sync runner unavailable");
@@ -294,6 +211,8 @@ export async function fixSyncState(context) {
294
211
  const discoveryArgs = await savedDiscoveryLimitArgs(context.command.homeDir);
295
212
  for (const repoRoot of roots) {
296
213
  const args = ["sync", "--json", "--workspace", repoRoot, ...discoveryArgs];
214
+ if (context.command.homeDir)
215
+ args.push("--home", context.command.homeDir);
297
216
  if (context.command.dashboardUrl !== DEFAULT_DASHBOARD_URL) {
298
217
  args.push("--dashboard-url", context.command.dashboardUrl);
299
218
  }
@@ -301,6 +220,21 @@ export async function fixSyncState(context) {
301
220
  const parsed = parseDoctorSyncJson(result.stdout);
302
221
  const output = `${result.stdout}\n${result.stderr}`;
303
222
  const status = jsonField(output, "status");
223
+ // BLI-4303, before any verdict: a sync that stood aside for the collection
224
+ // lock never ran, so neither its exit code nor its missing `uploaded`
225
+ // receipt says anything about this machine's health.
226
+ const standAside = syncStandAsideVerdict(parsed, status);
227
+ if (standAside) {
228
+ console.error("[cockpit-doctor] sync stood aside for a running collection run", JSON.stringify({
229
+ reason: standAside.code,
230
+ owner_alive: standAside.ownerAlive,
231
+ held_since: standAside.heldSince,
232
+ }));
233
+ return needsFix("sync-fresh", standAside.code, "waiting for the collection lock");
234
+ }
235
+ if (status === "live_sync_paused_during_backfill") {
236
+ return needsFix("sync-fresh", status, "waiting for historical catch-up");
237
+ }
304
238
  if (result.code !== 0) {
305
239
  const draining = syncBacklogDrainingVerdict(parsed);
306
240
  if (draining) {
@@ -317,4 +251,8 @@ export async function fixSyncState(context) {
317
251
  return ok("sync-fresh", "synced", roots.length === 1
318
252
  ? "ran `cockpit sync` and received an uploaded receipt"
319
253
  : `received uploaded receipts for ${roots.length} saved roots`);
320
- }
254
+ }
255
+ // Re-exported so `doctor.ts`, the doctor tests and every other caller keep
256
+ // importing these from `./doctor-pipeline.js` regardless of which sibling
257
+ // computes them.
258
+ export { backfillCompletionStepState, backfillFixVerdict, isCollectionBusyReason, syncBacklogDrainingVerdict, syncStandAsideVerdict, } from "./doctor-pipeline-verdicts.js";
@@ -1,3 +1,5 @@
1
+ import { inspectAgentRules, installAgentRules } from "../agent-rules.js";
2
+ import { doctorRoots } from "./doctor-access.js";
1
3
  import { installAutostartAgent } from "../autostart.js";
2
4
  import { fail, needsFix, ok, skipped } from "./doctor-report.js";
3
5
  import { autostartDoctorStatus, autostartRegistrationRoots, readAutostartRegistration, } from "./autostart-reading.js";
@@ -38,6 +40,8 @@ export async function fixAutostartState(context) {
38
40
  // fix that registers a different boundary than the check inspected would
39
41
  // leave the row red forever (BLI-3793).
40
42
  const roots = await autostartRegistrationRoots(context.command.homeDir, context.command.repoRoot);
43
+ if (roots.length === 0)
44
+ return needsFix("autostart-alive", "no_roots", "Run `cockpit doctor` to choose approved collection roots first.");
41
45
  const result = await installAutostartAgent({
42
46
  ...(roots[0] ? { repoRoot: roots[0] } : {}),
43
47
  repoRoots: roots,
@@ -67,7 +71,13 @@ export async function checkMemoryState(context) {
67
71
  return memoryStepState(outcome, "check");
68
72
  }
69
73
  export async function fixMemoryState(context) {
70
- const outcome = await installMemoryIntegration(memoryCommandFor(context), context.io);
74
+ let outcome = await installMemoryIntegration(memoryCommandFor(context), context.io);
75
+ if (!outcome.bin_found) {
76
+ // The public CLI declares the memory and Tower MCP packages as dependencies.
77
+ // Reinstall through the existing updater to restore missing package files.
78
+ await context.deps.selfUpdate(context.io, { json: true, tag: context.command.updateTag });
79
+ outcome = await installMemoryIntegration(memoryCommandFor(context), context.io);
80
+ }
71
81
  return memoryStepState(outcome, "fix");
72
82
  }
73
83
  function memoryCommandFor(context) {
@@ -95,7 +105,18 @@ function memoryStepState(outcome, phase) {
95
105
  return needsFix("memory-registered", "registration_incomplete", `BLI Memory is not registered with ${pending.map((target) => target.target).join(", ")}`);
96
106
  }
97
107
  if (!outcome.bin_found) {
98
- return skipped("memory-registered", "bin_missing", "bli-memory-mcp is not on this machine yet; nothing was written and the next run will try again");
108
+ return needsFix("memory-registered", "bin_missing", "bli-memory-mcp is not on this machine yet; nothing was written and the next run will try again");
99
109
  }
100
110
  return ok("memory-registered", phase === "fix" ? "installed" : "already_installed", "BLI Memory is registered with both agent hosts");
111
+ }
112
+ export async function checkDoctorAgentRules(context) {
113
+ const result = await inspectAgentRules({ homeDir: context.command.homeDir, scopePaths: await doctorRoots(context) });
114
+ return result.installed ? ok("agent-rules", "installed", "agent rules are installed for both hosts") : needsFix("agent-rules", "missing", "run `cockpit agent-rules install`");
115
+ }
116
+ export async function fixDoctorAgentRules(context) {
117
+ const roots = await doctorRoots(context);
118
+ if (roots.length === 0)
119
+ return needsFix("agent-rules", "no_roots", "Run `cockpit doctor` to choose approved roots first.");
120
+ await installAgentRules({ homeDir: context.command.homeDir, scopePaths: roots });
121
+ return checkDoctorAgentRules(context);
101
122
  }
@@ -1,3 +1,4 @@
1
+ import { setupReceiptPieces } from "@bli-cockpit/telemetry-core";
1
2
  import { redactedHealthDetail } from "../health-detail.js";
2
3
  import { setupReceiptBlock } from "./setup-receipt-lines.js";
3
4
  export function ok(id, code, message) {
@@ -28,9 +29,9 @@ export function isInteractiveDoctorFix(context) {
28
29
  return !context.command.json && Boolean(context.io.stdin.isTTY);
29
30
  }
30
31
  export async function maybeReportDoctorEvents(context, rows) {
31
- if (context.command.dryRun)
32
- return;
33
- await context.deps.reportInstallEvents({
32
+ if (context.command.dryRun || context.command.checkOnly)
33
+ return null;
34
+ return context.deps.reportInstallEvents({
34
35
  dashboardUrl: context.command.dashboardUrl,
35
36
  command: "doctor",
36
37
  events: rows.map(doctorEvent),
@@ -40,14 +41,17 @@ export async function maybeReportDoctorEvents(context, rows) {
40
41
  }
41
42
  export function writeDoctorOutput(command, io, rows,
42
43
  /** BLI-3731. Absent means it could not be read; doctor says so rather than nothing. */
43
- setupReceipt) {
44
+ setupReceipt, repairs = []) {
45
+ const needsPerson = doctorNeedsPerson(rows, setupReceipt);
44
46
  if (command.json) {
45
47
  writeLine(io.stdout, JSON.stringify({
46
- status: rows.some((row) => row.status === "fail" || row.hardStop)
48
+ status: needsPerson.length > 0
47
49
  ? "blocked"
48
50
  : "pass",
49
51
  dry_run: command.dryRun,
50
52
  steps: rows,
53
+ repairs,
54
+ needs_person: needsPerson,
51
55
  setup_receipt: setupReceipt?.receipt ?? null,
52
56
  memory_install: setupReceipt?.memory ?? null,
53
57
  }, null, 2));
@@ -58,13 +62,13 @@ setupReceipt) {
58
62
  // sentence, not the label (BLI-3194).
59
63
  writeLine(io.stdout, "state step result");
60
64
  for (const row of rows) {
61
- writeLine(io.stdout, `${doctorMark(row)} ${row.id.padEnd(20)} ${oneLine(row.message)}`);
65
+ writeLine(io.stdout, `${doctorMark(row)} ${row.id.padEnd(20)} ${oneLine(row.message).replaceAll("—", ";")}`);
62
66
  }
63
67
  const explanations = rows.filter((row) => (row.hardStop || row.status === "fail") && row.message.includes("\n"));
64
68
  for (const row of explanations) {
65
69
  writeLine(io.stderr, "");
66
70
  writeLine(io.stderr, `${row.id}:`);
67
- writeLine(io.stderr, row.message);
71
+ writeLine(io.stderr, row.message.replaceAll("—", ";"));
68
72
  }
69
73
  // BLI-3731. The invariant table says whether collection is healthy; this
70
74
  // says whether the machine is CONNECTED — the browser sign-in, the device
@@ -75,11 +79,14 @@ setupReceipt) {
75
79
  for (const line of setupReceipt
76
80
  ? setupReceiptBlock(setupReceipt, { indent: " " })
77
81
  : [" unknown — this machine could not be read this run."]) {
78
- writeLine(io.stdout, line);
82
+ writeLine(io.stdout, line.replaceAll("—", ";"));
79
83
  }
84
+ writeLine(io.stdout, needsPerson.length === 0 ? "Everything is fixed." : `${needsPerson.length} things still need you:`);
85
+ for (const item of needsPerson)
86
+ writeLine(io.stdout, ` ${item.step}: ${item.action}`);
80
87
  }
81
88
  function doctorEvent(row) {
82
- const status = row.status === "fail" || row.hardStop
89
+ const status = row.status === "fail" || row.status === "needs_fix" || row.hardStop
83
90
  ? "fail"
84
91
  : row.status === "skipped"
85
92
  ? "skipped"
@@ -124,4 +131,36 @@ export function asRecord(value) {
124
131
  }
125
132
  export function writeLine(stream, text) {
126
133
  stream.write(`${text}\n`);
134
+ }
135
+ export function doctorNeedsPerson(rows, receipt) {
136
+ const defaults = {
137
+ "cli-latest": "npm install -g @bli-cockpit/cli@latest",
138
+ "authed": "cockpit login",
139
+ "roots-ok": "cockpit doctor",
140
+ "single-install": "Run npm uninstall -g @bli-cockpit/cli with the Node installation that owns the extra CLI listed above.",
141
+ "autostart-alive": "cockpit autostart install",
142
+ "memory-registered": "cockpit memory install",
143
+ "agent-rules": "cockpit agent-rules install",
144
+ "mcp-answers": "cockpit doctor",
145
+ "memory-daemon": "cockpit memory install",
146
+ "backfill-complete": "cockpit backfill --all --yes",
147
+ "sync-fresh": "cockpit sync --json",
148
+ "gc-checked": "cockpit doctor",
149
+ "disk-bounded": "cockpit clean --all-committed",
150
+ };
151
+ const items = rows.filter((row) => row.status === "fail" || row.status === "needs_fix" || row.hardStop).map((row) => ({
152
+ step: row.id, reason: row.code,
153
+ action: row.nextAction ?? row.message.match(/`([^`]+)`/u)?.[1] ?? defaults[row.id],
154
+ }));
155
+ if (receipt) {
156
+ for (const { key, piece } of setupReceiptPieces(receipt.receipt, receipt.memory)) {
157
+ if (piece.status === "ok" || piece.status === "skipped" || piece.status === "unsupported")
158
+ continue;
159
+ const step = key === "device" || key === "browser" ? "authed" : key === "collector.autostart" ? "autostart-alive" : "memory-registered";
160
+ if (piece.status !== "needs_trust" && items.some((item) => item.step === step))
161
+ continue;
162
+ items.push({ step: key, reason: piece.reason ?? piece.status, action: piece.status === "needs_trust" ? "Open Codex and run /hooks to trust the installed hooks." : defaults[step] });
163
+ }
164
+ }
165
+ return items;
127
166
  }