@bli-cockpit/cli 0.2.168 → 0.2.173

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/dist/adapters/raw-evidence-collection-state.js +52 -6
  2. package/dist/adapters/raw-evidence-completeness.js +6 -0
  3. package/dist/adapters/raw-evidence-git-diff-reader.js +2 -2
  4. package/dist/adapters/raw-evidence-image-reader.js +5 -2
  5. package/dist/adapters/raw-evidence-transcript-file.js +2 -2
  6. package/dist/autostart-opt-out.js +43 -0
  7. package/dist/autostart-self-heal.js +23 -10
  8. package/dist/commands/autostart-command.js +10 -0
  9. package/dist/commands/autostart-heal.js +6 -4
  10. package/dist/commands/backfill-upload.js +12 -0
  11. package/dist/commands/doctor-access.js +17 -21
  12. package/dist/commands/doctor-disk-words.js +20 -0
  13. package/dist/commands/doctor-mcp.js +27 -0
  14. package/dist/commands/doctor-pipeline.js +34 -7
  15. package/dist/commands/doctor-registration.js +9 -3
  16. package/dist/commands/doctor-report.js +28 -0
  17. package/dist/commands/doctor.js +20 -5
  18. package/dist/commands/heartbeat-token.js +14 -0
  19. package/dist/commands/heartbeat.js +2 -1
  20. package/dist/commands/local-args-tower-logs.js +53 -0
  21. package/dist/commands/local-args-tower.js +1 -0
  22. package/dist/commands/local-args.js +3 -1
  23. package/dist/commands/local-command-shapes-logs.js +6 -0
  24. package/dist/commands/local-help-commands-tower-ops.js +26 -0
  25. package/dist/commands/local-help.js +2 -0
  26. package/dist/commands/local.js +3 -0
  27. package/dist/commands/logs.js +68 -0
  28. package/dist/commands/public-root.js +1 -1
  29. package/dist/commands/sync-followups-staging.js +8 -1
  30. package/dist/commands/sync-heal.js +144 -0
  31. package/dist/commands/sync.js +9 -1
  32. package/dist/disk-prune.js +35 -3
  33. package/dist/disk-retention.js +31 -1
  34. package/dist/sign-in-action.js +7 -0
  35. package/dist/staging-gate-receipt.js +131 -0
  36. package/dist/staging-gate.js +214 -0
  37. package/dist/state-dir-audit.js +5 -4
  38. package/dist/state-dir-prune.js +84 -0
  39. package/dist/sync-tick-stamp.js +22 -0
  40. package/package.json +4 -4
@@ -26,7 +26,8 @@ import { RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES } from "@bli-cockpit/telemetry-core"
26
26
  import crypto from "node:crypto";
27
27
  import fs from "node:fs/promises";
28
28
  import path from "node:path";
29
- import { DELIVERY_BACKOFF_BYPASS_REASON, deliveryBackoffApplies, heldSourceKeys, readRawEvidenceStagingState, resolveStagedObject, } from "../raw-evidence-staging.js";
29
+ import { DELIVERY_BACKOFF_BYPASS_REASON, deliveryBackoffApplies, heldSourceKeys, readRawEvidenceStagingState, resolveStagedObject, terminalDeliveryReason, } from "../raw-evidence-staging.js";
30
+ import { admitToStaging, openStagingAllowance, } from "../staging-gate.js";
30
31
  import { chmodPrivate } from "./raw-evidence-pack-store.js";
31
32
  import { GIT_DIFF_TIMEOUT_MS, MAX_GIT_DIFF_BYTES, } from "./raw-evidence-git-diff.js";
32
33
  export const DEFAULT_SINCE_MINUTES = 24 * 60;
@@ -54,6 +55,11 @@ export async function openCollection(context, options, places) {
54
55
  const byteBudget = options.byteBudget ?? RAW_EVIDENCE_DEFAULT_BYTE_BUDGET;
55
56
  const objectBudget = options.objectBudget ?? RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET;
56
57
  const staging = await readRawEvidenceStagingState(options.stateDir);
58
+ const budget = options.budget ?? {
59
+ remainingBytes: byteBudget,
60
+ remainingObjects: objectBudget,
61
+ };
62
+ const stagingAllowance = (budget.staging ??= await openStagingAllowance(options.stateDir, options.env ?? process.env, { statfs: options.statfs }));
57
63
  return {
58
64
  context,
59
65
  filesDir: path.join(places.stagingDir, "files"),
@@ -71,14 +77,16 @@ export async function openCollection(context, options, places) {
71
77
  redacted: [],
72
78
  reused: [],
73
79
  scanned: new Map(),
74
- caps: startingCaps({ byteBudget, objectBudget, budget: options.budget }),
80
+ caps: startingCaps({
81
+ byteBudget,
82
+ objectBudget,
83
+ budget: options.budget,
84
+ stagingCapBytes: stagingAllowance.snapshot.cap_bytes,
85
+ }),
75
86
  skipContentHashes: options.skipContentHashes ?? new Set(),
76
87
  priorSnapshots: options.priorSnapshots ?? new Map(),
77
88
  stubReoffers: options.stubReoffers ?? new Map(),
78
- budget: options.budget ?? {
79
- remainingBytes: byteBudget,
80
- remainingObjects: objectBudget,
81
- },
89
+ budget,
82
90
  index: { value: 0 },
83
91
  };
84
92
  }
@@ -118,6 +126,12 @@ function startingCaps(options) {
118
126
  observed: options.budget?.remainingObjects ?? options.objectBudget,
119
127
  applied: false,
120
128
  },
129
+ {
130
+ source: "raw_evidence",
131
+ cap_type: "staging_gate",
132
+ limit: options.stagingCapBytes,
133
+ applied: false,
134
+ },
121
135
  {
122
136
  source: "git_diff",
123
137
  cap_type: "max_bytes_per_diff",
@@ -185,6 +199,38 @@ export async function stageEvidenceBytes(collection, options) {
185
199
  collection.stagedNewCount += 1;
186
200
  return { local_path: destination, staged_in_pack: true };
187
201
  }
202
+ /**
203
+ * The one door a file passes before its bytes may be copied: the per-sync
204
+ * budget below, then the disk (BLI-12832). A refusal from the disk hands the
205
+ * budget back, because nothing was staged and the next tick re-reads the source.
206
+ * A copy already staged costs the disk nothing and is never refused by it.
207
+ */
208
+ export function admitEvidence(collection, byteLength, file) {
209
+ const budgetReason = admitToBudget(collection.budget, byteLength);
210
+ if (budgetReason)
211
+ return budgetReason;
212
+ const allowance = collection.budget.staging;
213
+ if (!allowance)
214
+ return null;
215
+ // BLI-12832: bytes the server refused for good are reclaimed by the prune
216
+ // (the refusal stays as a receipt), so copying them again would be churn.
217
+ if (collection.staging.staged[file.contentHash] === undefined &&
218
+ terminalDeliveryReason(collection.staging.delivery_attempts[file.contentHash]?.last_reason)) {
219
+ collection.budget.remainingObjects += 1;
220
+ collection.budget.remainingBytes += byteLength;
221
+ console.error("[staging-gate] not copying bytes the server refused for good", JSON.stringify({ reason: "delivery_terminal_refused", kind: file.kind, file_bytes: byteLength }));
222
+ return "delivery_terminal_refused";
223
+ }
224
+ const diskReason = admitToStaging(allowance, byteLength, {
225
+ alreadyStaged: collection.staging.staged[file.contentHash] !== undefined,
226
+ kind: file.kind,
227
+ });
228
+ if (!diskReason)
229
+ return null;
230
+ collection.budget.remainingObjects += 1;
231
+ collection.budget.remainingBytes += byteLength;
232
+ return `staging_deferred:${diskReason}`;
233
+ }
188
234
  /**
189
235
  * Decrements the per-sync budget when a file fits, or returns a deferred-skip
190
236
  * reason when it does not. The object budget bounds request count; the byte
@@ -21,6 +21,12 @@ export function markCapApplied(ledger, source, capType) {
21
21
  cap.applied = true;
22
22
  }
23
23
  export function markBudgetCapApplied(ledger, reason) {
24
+ if (reason === "delivery_terminal_refused")
25
+ return;
26
+ if (reason.startsWith("staging_deferred:")) {
27
+ markCapApplied(ledger, "raw_evidence", "staging_gate");
28
+ return;
29
+ }
24
30
  markCapApplied(ledger, "raw_evidence", reason === "deferred_object_budget" ? "object_budget" : "byte_budget");
25
31
  }
26
32
  // ---------------------------------------------------------------------------
@@ -19,7 +19,7 @@ import { sha256 } from "./raw-evidence-keys.js";
19
19
  import { maskSecretsInGitDiff, } from "./raw-evidence-sanitize.js";
20
20
  import { markBudgetCapApplied, markCapApplied, recordScanned, } from "./raw-evidence-completeness.js";
21
21
  import { evidenceEntry } from "./raw-evidence-manifest.js";
22
- import { admitToBudget, stageEvidenceBytes, } from "./raw-evidence-collection-state.js";
22
+ import { admitEvidence, stageEvidenceBytes, } from "./raw-evidence-collection-state.js";
23
23
  import { MAX_GIT_DIFF_BYTES, REPO_ROOT_MISSING_REASON, RepoRootMissingError, repoRootExists, runGitDiff, } from "./raw-evidence-git-diff.js";
24
24
  const GIT_DIFF_TARGETS = [
25
25
  { label: "unstaged", args: ["diff", "--no-ext-diff", "--"] },
@@ -130,7 +130,7 @@ async function stageOneGitDiff(collection, target) {
130
130
  });
131
131
  return;
132
132
  }
133
- const deferReason = admitToBudget(collection.budget, sanitized.bytes.byteLength);
133
+ const deferReason = admitEvidence(collection, sanitized.bytes.byteLength, { contentHash, kind: "git_diff" });
134
134
  if (deferReason) {
135
135
  markBudgetCapApplied(collection, deferReason);
136
136
  collection.skipped.push({
@@ -13,7 +13,7 @@ import { evidenceSourceKey } from "../raw-evidence-staging.js";
13
13
  import { sha256 } from "./raw-evidence-keys.js";
14
14
  import { markBudgetCapApplied, recordScanned, } from "./raw-evidence-completeness.js";
15
15
  import { evidenceEntry } from "./raw-evidence-manifest.js";
16
- import { admitToBudget, stageEvidenceBytes, } from "./raw-evidence-collection-state.js";
16
+ import { admitEvidence, stageEvidenceBytes, } from "./raw-evidence-collection-state.js";
17
17
  export async function collectAgentImagesFromTranscript(collection, options) {
18
18
  const result = await collectAgentImageEvidenceFromJsonlFile({
19
19
  filePath: options.filePath,
@@ -56,7 +56,10 @@ async function collectOneAgentImageFile(collection, options) {
56
56
  });
57
57
  return;
58
58
  }
59
- const deferReason = admitToBudget(collection.budget, raw.byteLength);
59
+ const deferReason = admitEvidence(collection, raw.byteLength, {
60
+ contentHash,
61
+ kind: options.kind,
62
+ });
60
63
  if (deferReason) {
61
64
  markBudgetCapApplied(collection, deferReason);
62
65
  collection.skipped.push({
@@ -20,7 +20,7 @@ import { maskSecretsInTranscript, } from "./raw-evidence-sanitize.js";
20
20
  import { markBudgetCapApplied, markCapApplied, } from "./raw-evidence-completeness.js";
21
21
  import { evidenceEntry } from "./raw-evidence-manifest.js";
22
22
  import { describeSnapshot, stubReplacementClaim, supersedeClaimFor, } from "./raw-evidence-snapshot-supersede.js";
23
- import { admitToBudget, stageEvidenceBytes, } from "./raw-evidence-collection-state.js";
23
+ import { admitEvidence, stageEvidenceBytes, } from "./raw-evidence-collection-state.js";
24
24
  /**
25
25
  * Reads, secret-guards, content-addresses, budget-checks, and copies one
26
26
  * attributed transcript into the pack.
@@ -82,7 +82,7 @@ export async function collectOneEvidenceFile(collection, options) {
82
82
  });
83
83
  return true;
84
84
  }
85
- const deferReason = admitToBudget(collection.budget, sanitized.bytes.byteLength);
85
+ const deferReason = admitEvidence(collection, sanitized.bytes.byteLength, { contentHash, kind: options.kind });
86
86
  if (deferReason) {
87
87
  markBudgetCapApplied(collection, deferReason);
88
88
  return skip(deferReason);
@@ -0,0 +1,43 @@
1
+ import { AUTOSTART_OPT_OUT_MARKER_FILE_NAME } from "@bli-cockpit/telemetry-core";
2
+ import fs from "node:fs/promises";
3
+ import path from "node:path";
4
+ /**
5
+ * BLI-12828. The tick now CREATES a missing scheduler registration (an absent
6
+ * LaunchAgent / Scheduled Task is how a Mac goes quiet after a reboot), so
7
+ * "absent" can no longer double as "the operator removed it on purpose". This
8
+ * marker is that deliberate act: `cockpit autostart uninstall` writes it,
9
+ * `cockpit autostart install` and `cockpit doctor` clear it, and the tick's
10
+ * create-when-absent repair refuses while it stands.
11
+ *
12
+ * One small file under the state dir; it holds a timestamp, never a path.
13
+ */
14
+ export const AUTOSTART_OPT_OUT_MARKER = AUTOSTART_OPT_OUT_MARKER_FILE_NAME;
15
+ function markerPath(paths) {
16
+ return path.join(paths.state_dir, AUTOSTART_OPT_OUT_MARKER);
17
+ }
18
+ export async function isAutostartOptedOut(paths) {
19
+ return fs
20
+ .stat(markerPath(paths))
21
+ .then(() => true)
22
+ .catch(() => false);
23
+ }
24
+ export async function markAutostartOptedOut(paths, now = new Date()) {
25
+ try {
26
+ await fs.mkdir(paths.state_dir, { recursive: true, mode: 0o700 });
27
+ await fs.writeFile(markerPath(paths), now.toISOString());
28
+ console.error("[autostart] opt-out recorded; the sync tick will not re-create the scheduler until `cockpit autostart install`", JSON.stringify({ reason: "operator_uninstalled" }));
29
+ }
30
+ catch (error) {
31
+ console.error("[autostart] opt-out could not be recorded; the tick may re-create the scheduler", JSON.stringify({
32
+ reason: "opt_out_write_failed",
33
+ error_name: error instanceof Error ? error.name : typeof error,
34
+ }));
35
+ }
36
+ }
37
+ export async function clearAutostartOptOut(paths) {
38
+ const had = await isAutostartOptedOut(paths);
39
+ await fs.rm(markerPath(paths), { force: true }).catch(() => undefined);
40
+ if (had) {
41
+ console.error("[autostart] opt-out cleared; background sync is wanted again", JSON.stringify({ reason: "operator_installed" }));
42
+ }
43
+ }
@@ -3,8 +3,12 @@ import path from "node:path";
3
3
  import { spawn } from "node:child_process";
4
4
  import { autostartStatus, installAutostartAgent, registeredRuntimePathProblems, } from "./autostart.js";
5
5
  import { detectSecondCockpitInstall, secondInstallReason, } from "./second-install.js";
6
+ import { isAutostartOptedOut } from "./autostart-opt-out.js";
6
7
  export const AUTOSTART_REPAIR_THROTTLE_MARKER = ".last-autostart-repair";
7
- const AUTOSTART_REPAIR_MIN_INTERVAL_MS = 24 * 60 * 60 * 1000;
8
+ // BLI-12828: the FIRST repair of a break never waits. The marker only spaces
9
+ // out RETRIES of a repair that keeps failing, and a healthy reading clears it,
10
+ // so the next break is repaired on the next tick, not a day later.
11
+ const AUTOSTART_REPAIR_RETRY_AFTER_FAILURE_MS = 60 * 60 * 1000;
8
12
  const DETAIL_MAX_CHARS = 300;
9
13
  export async function runAutostartSelfHeal(paths, options) {
10
14
  const platform = options.platform ?? process.platform;
@@ -20,9 +24,12 @@ export async function runAutostartSelfHeal(paths, options) {
20
24
  exec: options.exec,
21
25
  platform,
22
26
  });
23
- // Absent means the operator (or onboarding) owns the decision, not this tick.
24
- if (status.status === "absent")
27
+ const absent = status.status === "absent";
28
+ if (absent && (await isAutostartOptedOut(paths))) {
29
+ // Said on the tick's own log so "why is this Mac not ticking" is answerable.
30
+ console.error("[autostart-self-heal] scheduler absent; not re-creating it, the operator uninstalled it on purpose", JSON.stringify({ reason: "autostart_uninstalled_by_operator", platform }));
25
31
  return null;
32
+ }
26
33
  // A registration can read as "loaded" and still name a binary that no longer
27
34
  // exists — that is precisely what `brew upgrade node` leaves behind. Ask the
28
35
  // filesystem about the paths the SCHEDULER holds, not the ones this process
@@ -34,11 +41,16 @@ export async function runAutostartSelfHeal(paths, options) {
34
41
  platform,
35
42
  })
36
43
  : [];
37
- if (status.status !== "not_loaded" && runtimeProblems.length === 0) {
38
- // Healthy is the steady state and stays silent.
44
+ const marker = path.join(paths.state_dir, AUTOSTART_REPAIR_THROTTLE_MARKER);
45
+ if (!absent && status.status !== "not_loaded" && runtimeProblems.length === 0) {
46
+ // Healthy is the steady state and stays silent. A marker left by an
47
+ // earlier attempt is spent: that repair worked, so the next break starts
48
+ // fresh instead of waiting out a retry window.
49
+ await fs.rm(marker, { force: true }).catch(() => undefined);
39
50
  return null;
40
51
  }
41
52
  const problem = [
53
+ ...(absent ? ["registration absent"] : []),
42
54
  ...(status.message ? [status.message] : []),
43
55
  ...runtimeProblems,
44
56
  ]
@@ -59,15 +71,16 @@ export async function runAutostartSelfHeal(paths, options) {
59
71
  step: platform === "darwin" ? "autostart_heal" : "autostart_repair",
60
72
  };
61
73
  }
62
- // Attempts are throttled (marker mtime, written for the attempt not the
63
- // outcome) so a persistently failing repair cannot run every 15 minutes. The
74
+ // Only a RETRY waits (marker mtime, written for the attempt not the
75
+ // outcome, cleared by the next healthy reading): a repair that keeps failing
76
+ // is retried hourly instead of churning the scheduler every 15 minutes. The
64
77
  // status probe above still runs every tick — it is one launchctl/schtasks
65
78
  // query.
66
79
  const now = options.now ?? new Date();
67
- const marker = path.join(paths.state_dir, AUTOSTART_REPAIR_THROTTLE_MARKER);
68
80
  const lastAttempt = await fs.stat(marker).catch(() => null);
69
81
  if (lastAttempt &&
70
- now.getTime() - lastAttempt.mtimeMs < AUTOSTART_REPAIR_MIN_INTERVAL_MS) {
82
+ now.getTime() - lastAttempt.mtimeMs < AUTOSTART_REPAIR_RETRY_AFTER_FAILURE_MS) {
83
+ console.error("[autostart-self-heal] scheduler still broken; the last repair attempt was under an hour ago, retrying after that", JSON.stringify({ reason: "repair_retry_waiting", platform }));
71
84
  return { status: "skipped", reason: "repair_throttled_recent_attempt" };
72
85
  }
73
86
  await fs.mkdir(paths.state_dir, { recursive: true }).catch(() => undefined);
@@ -85,7 +98,7 @@ export async function runAutostartSelfHeal(paths, options) {
85
98
  if (repaired.loaded) {
86
99
  return {
87
100
  status: "ok",
88
- reason: "autostart_repaired",
101
+ reason: absent ? "autostart_created" : "autostart_repaired",
89
102
  detail: problem,
90
103
  step: "autostart_repair",
91
104
  };
@@ -1,6 +1,7 @@
1
1
  import { defaultExec, writeLine } from "./cli-io.js";
2
2
  import { runAutostartHealDetached } from "./autostart-heal.js";
3
3
  import { autostartStatus, installAutostartAgent, uninstallAutostartAgent, } from "../autostart.js";
4
+ import { clearAutostartOptOut, markAutostartOptedOut, } from "../autostart-opt-out.js";
4
5
  import { getCollectorRuntimePaths, readLocalCollectorConfig, } from "../local-state.js";
5
6
  export async function runAutostart(command, io) {
6
7
  if (command.action === "heal-detached") {
@@ -37,6 +38,15 @@ export async function runAutostart(command, io) {
37
38
  intervalSeconds: command.intervalSeconds,
38
39
  exec,
39
40
  });
41
+ // BLI-12828: the tick re-creates an absent scheduler, so a deliberate
42
+ // uninstall has to leave a mark the tick can read.
43
+ const optOutPaths = getCollectorRuntimePaths(command.homeDir);
44
+ if (command.action === "uninstall" && result.status !== "unsupported") {
45
+ await markAutostartOptedOut(optOutPaths);
46
+ }
47
+ else if (command.action === "install" && result.loaded !== false) {
48
+ await clearAutostartOptOut(optOutPaths);
49
+ }
40
50
  if (command.json) {
41
51
  writeLine(io.stdout, JSON.stringify(result, null, 2));
42
52
  return autostartExitCode(command.action, result);
@@ -1,6 +1,7 @@
1
1
  import path from "node:path";
2
2
  import { appendFile } from "node:fs/promises";
3
3
  import { AUTOSTART_LABEL, autostartStatus, reregisterDarwinAgent, writeDarwinAutostartPlist, } from "../autostart.js";
4
+ import { isAutostartOptedOut } from "../autostart-opt-out.js";
4
5
  import { redactedHealthDetail } from "../health-detail.js";
5
6
  import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, readLocalCollectorConfig, } from "../local-state.js";
6
7
  import { normalizeCollectionRoots } from "../root-normalization.js";
@@ -72,9 +73,10 @@ async function healDetached(command, io, deps) {
72
73
  exec,
73
74
  platform,
74
75
  });
75
- if (before.status === "absent") {
76
- // Repair what exists, never create.
77
- return { status: "skipped", reason: "autostart_absent" };
76
+ if (before.status === "absent" &&
77
+ (await isAutostartOptedOut(getCollectorRuntimePaths(command.homeDir)))) {
78
+ // BLI-12828: absent is created now; only a deliberate uninstall stops it.
79
+ return { status: "skipped", reason: "autostart_uninstalled_by_operator" };
78
80
  }
79
81
  let written;
80
82
  try {
@@ -125,7 +127,7 @@ async function healDetached(command, io, deps) {
125
127
  }
126
128
  return {
127
129
  status: "ok",
128
- reason: "autostart_healed",
130
+ reason: before.status === "absent" ? "autostart_created" : "autostart_healed",
129
131
  detail: `node_path_reason=${written.node_path_reason}`,
130
132
  };
131
133
  }
@@ -7,6 +7,7 @@ import { countSessionUploadFailures, deferredEvidenceCount, durableBackfillCandi
7
7
  import { blockingScanIssues } from "./backfill-issues.js";
8
8
  import { ensureOutsideRootsWorkspace, isOutsideRootsWorkspace, } from "./backfill-outside-roots.js";
9
9
  import { writeLine } from "./backfill-output.js";
10
+ import { stagingDeferredFiles } from "../staging-gate.js";
10
11
  import { stubClaimsForCandidates } from "./backfill-stub-reoffer.js";
11
12
  const BACKFILL_MAX_CONSECUTIVE_FAILURES = 3;
12
13
  /**
@@ -92,6 +93,17 @@ export async function uploadBackfillBatches(command, io, lock, ctx) {
92
93
  failureReason = "deferred_budget_exhausted";
93
94
  break;
94
95
  }
96
+ // BLI-12832: the disk gate refused a copy. Every later batch would be
97
+ // refused the same way, so stop here; the cursor only advanced through
98
+ // sessions that earned a durable pointer, and the next run starts from
99
+ // the same place once room returns.
100
+ const stagingDeferred = stagingDeferredFiles(rawEvidenceBudget.staging);
101
+ if (stagingDeferred > 0) {
102
+ blockedAt = stoppedAt("staging paused: not enough room to copy more evidence", index);
103
+ failureReason = "deferred_budget_exhausted";
104
+ console.error("[backfill] stopped, the staging gate is holding new copies", JSON.stringify({ reason: "staging_deferred", files_deferred: stagingDeferred, batch_index: index + 1 }));
105
+ break;
106
+ }
95
107
  if (consecutiveFailures >= BACKFILL_MAX_CONSECUTIVE_FAILURES) {
96
108
  blockedAt = stoppedAt("consecutive upload failures", index);
97
109
  failureReason = failedBatchReason(outcome.sync, outcome.missingDurableMain);
@@ -8,6 +8,7 @@ import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, readLocalCollectorConf
8
8
  import { normalizeCollectionRoots } from "../root-normalization.js";
9
9
  import { detectSecondCockpitInstall } from "../second-install.js";
10
10
  import { hardStop, needsFix, ok, skipped } from "./doctor-report.js";
11
+ import { SIGN_IN_ACTION } from "../sign-in-action.js";
11
12
  /**
12
13
  * The `authed`, `roots-ok`, and `single-install` check family: who is this
13
14
  * machine and where does it collect. Auth and roots are hard stops with no
@@ -31,7 +32,7 @@ export async function fixAuthState(context, state) {
31
32
  return checked;
32
33
  }
33
34
  if (!isInteractiveDoctorFix(context))
34
- return { ...state, nextAction: "cockpit login" };
35
+ return { ...state, nextAction: SIGN_IN_ACTION };
35
36
  const code = await context.deps.runLogin(context).catch((error) => {
36
37
  // Exit code 1 with no reason at all is what an operator saw when doctor
37
38
  // tried and failed to repair their auth — the same output as a login that
@@ -73,13 +74,10 @@ export async function readAuthState(context) {
73
74
  }
74
75
  return ok("authed", "device_token_present", "this machine is signed in");
75
76
  }
76
- return hardStop("authed", "pairing_required", [
77
- "device is not signed in.",
78
- "What you can do:",
79
- " 1) Run `cockpit login` and complete the email/device approval.",
80
- ` 2) If this is a reused machine, run \`${onboardOneLiner(context.command)}\` to refresh onboarding.`,
81
- " 3) Send this output to Edward if approval is blocked.",
82
- ].join("\n"));
77
+ return {
78
+ ...hardStop("authed", "pairing_required", `this machine is signed out. ${SIGN_IN_ACTION}`),
79
+ nextAction: SIGN_IN_ACTION,
80
+ };
83
81
  }
84
82
  export async function readRootState(context) {
85
83
  const paths = getCollectorRuntimePaths(context.command.homeDir);
@@ -92,13 +90,11 @@ export async function readRootState(context) {
92
90
  roots,
93
91
  };
94
92
  }
95
- return hardStop("roots-ok", "no_roots", [
96
- "no saved collection roots were found.",
97
- "What you can do:",
98
- ` 1) Run \`${onboardOneLiner(context.command)}\` to save the workspace roots again.`,
99
- " 2) If this is the wrong folder, rerun from the BLI workspace or pass `--workspace <path>`.",
100
- " 3) Run `cockpit doctor` interactively to choose approved roots.",
101
- ].join("\n"));
93
+ const action = `Run \`${onboardOneLiner(context.command)}\` in a terminal from the folder where you work.`;
94
+ return {
95
+ ...hardStop("roots-ok", "no_roots", `no collection folders are saved yet. ${action}`),
96
+ nextAction: action,
97
+ };
102
98
  }
103
99
  /**
104
100
  * BLI-3218/BLI-3553. Two Tower installs on one machine make each one's tick
@@ -164,10 +160,10 @@ function shellQuote(value) {
164
160
  * a gate on it, so a session sitting outside one is not a defect doctor found
165
161
  * and cannot repair. This row is therefore always informational (`ok`): it
166
162
  * reports how many of this scan's sessions were inside vs. outside, never
167
- * asks for approval, and attempts no fix. Widening the boundary still buys
168
- * the redacted transcript BYTES for those outside directories (those stay
169
- * root-gated), so the message still names `cockpit roots add` as something an
170
- * operator can choose to do, never something doctor is stuck waiting on. The
163
+ * asks for approval, and attempts no fix. Widening a root only adds the redacted transcript BYTES for those
164
+ * directories, so the message names `cockpit roots add` as an OPTIONAL step an
165
+ * operator can choose, never something doctor, or any person, is waiting on to
166
+ * approve. The
171
167
  * row only turns red the ordinary way every other doctor check does: the scan
172
168
  * itself throwing (`work_directory_scan_incomplete` from unreadable session
173
169
  * folders), which `runDoctorWithDeps`'s existing `check_threw` handling
@@ -182,9 +178,9 @@ export async function readRootsCoverWork(context) {
182
178
  const outsideCount = outside.reduce((sum, entry) => sum + entry.count, 0);
183
179
  const insideCount = scan.observed - outsideCount;
184
180
  if (!outsideCount)
185
- return ok("roots-cover-work", "work_covered", `${scan.observed} sessions seen; no known working directories outside approved roots. ${sessionCoverageLine(scan, outsideCount)}`);
181
+ return ok("roots-cover-work", "work_covered", `${scan.observed} sessions seen; no working folders outside your saved roots. ${sessionCoverageLine(scan, outsideCount)}`);
186
182
  return {
187
- ...ok("roots-cover-work", "work_outside_roots_labelled", `${insideCount} of ${scan.observed} sessions inside approved roots, ${outsideCount} outside — every one already reached the server, the outside ones carrying an outside label. Their redacted transcript content stays unuploaded until a root widens (scan ${scan.generated_at}):\n${outside.map(entry => `${entry.count}: ${entry.directory}\n ${rootsAddCommand(entry.directory, process.platform, homeDir)}`).join("\n")}\n\n${sessionCoverageLine(scan, outsideCount)}`),
183
+ ...ok("roots-cover-work", "work_outside_roots_labelled", `${insideCount} of ${scan.observed} sessions inside your saved roots, ${outsideCount} outside — every one already reached the server, the outside ones carrying an outside label. Nothing needs approving. Optional: add a folder as a root to also upload its redacted transcript text (scan ${scan.generated_at}):\n${outside.map(entry => `${entry.count}: ${entry.directory}\n ${rootsAddCommand(entry.directory, process.platform, homeDir)}`).join("\n")}\n\n${sessionCoverageLine(scan, outsideCount)}`),
188
184
  nextAction: rootsAddCommand(outside[0].directory, process.platform, homeDir),
189
185
  };
190
186
  }
@@ -57,6 +57,26 @@ export function footprintBuckets(footprint) {
57
57
  ];
58
58
  return buckets.filter((bucket) => bucket.bytes > 0).sort((a, b) => b.bytes - a.bytes);
59
59
  }
60
+ /**
61
+ * BLI-12832: the sentence for a paused staging gate, in plain words. Every
62
+ * variant says the same two true things: nothing is lost (the source file is
63
+ * still on disk), and copying resumes by itself.
64
+ */
65
+ export function stagingPausedWords(gate, holder) {
66
+ if (gate.reason === "low_disk") {
67
+ return `Tower paused copying new evidence: the disk has ${gib(gate.free_bytes ?? 0)} GB free. Nothing is lost; it resumes when space returns.`;
68
+ }
69
+ if (gate.reason === "state_over_ceiling") {
70
+ return `Tower paused copying new evidence: everything it keeps here is ${mib(gate.collector_bytes)} MB, over its ${mib(gate.ceiling_bytes)} MB ceiling${holder ? `, mostly ${holder.label} (${mib(holder.bytes)} MB)` : ""}. Nothing is lost; it resumes when that shrinks.`;
71
+ }
72
+ if (gate.reason === "staging_at_cap") {
73
+ return `Tower paused copying new evidence: its staging folder is full (${mib(gate.staging_bytes)} MB of a ${mib(gate.cap_bytes)} MB cap) with evidence the server has not accepted yet. Nothing is lost; the sources stay on disk and copying resumes once uploads succeed.`;
74
+ }
75
+ return null;
76
+ }
77
+ export function gib(bytes) {
78
+ return (bytes / (1024 * 1024 * 1024)).toFixed(1);
79
+ }
60
80
  export function footprintTotalBytes(footprint) {
61
81
  return footprintBuckets(footprint).reduce((sum, bucket) => sum + bucket.bytes, 0);
62
82
  }
@@ -30,6 +30,12 @@
30
30
  * ledger with its reason label (`doctor-report.ts`'s `doctorEvent`
31
31
  * maps every other status to `ok`).
32
32
  *
33
+ * BLI-12828: a `fail` (nothing answered) now has a repair. The server ships
34
+ * inside the CLI and is wired in by `memory install`, so the fix re-registers,
35
+ * takes the newest CLI if that alone did not wake it, and asks again; the row's
36
+ * receipt shows the before and after. A `needs_fix` (Tower refused) has none:
37
+ * the refusal is the sign-in's, and the `authed` row owns that.
38
+ *
33
39
  * `skipped` when the bin is not on this machine at all, for the same reason
34
40
  * `memory-registered` skips: nothing to probe is not a broken machine.
35
41
  */
@@ -95,4 +101,25 @@ function probeEnv(context) {
95
101
  if (context.command.dashboardUrl)
96
102
  base[TOWER_DASHBOARD_URL_ENV] = context.command.dashboardUrl;
97
103
  return base;
104
+ }
105
+ /**
106
+ * The `mcp-answers` repair (BLI-12828): re-register, then take the newest CLI,
107
+ * asking the server again after each. Heavy (it spawns the server and may run
108
+ * the updater), so doctor runs it only on a `fail` and the tick leaves it to
109
+ * the daily memory install and the CLI floor, which do the same two things.
110
+ */
111
+ export async function fixMcpAnswersState(context, state) {
112
+ if (state.status !== "fail") {
113
+ console.error("[doctor] mcp-answers not repaired; Tower refused the call and the sign-in row owns that", JSON.stringify({ reason: state.code }));
114
+ return state;
115
+ }
116
+ await context.deps.fixMemory(context);
117
+ let checked = await context.deps.checkMcpAnswers(context);
118
+ console.error("[doctor] mcp-answers re-registered and asked again", JSON.stringify({ reason: checked.code, status: checked.status }));
119
+ if (checked.status === "fail") {
120
+ await context.deps.selfUpdate(context.io, { json: true, tag: context.command.updateTag });
121
+ checked = await context.deps.checkMcpAnswers(context);
122
+ console.error("[doctor] mcp-answers updated the CLI and asked again", JSON.stringify({ reason: checked.code, status: checked.status }));
123
+ }
124
+ return checked;
98
125
  }
@@ -5,12 +5,13 @@ import { readBackfillCompletionMarker, readBackfillCursor, } from "../cursors/ba
5
5
  import { savedDiscoveryLimitArgs } from "../discovery-limits.js";
6
6
  import { runStagingPrune } from "../disk-prune.js";
7
7
  import { retentionOptionsFromEnv } from "../disk-retention.js";
8
+ import { readStagingGate, stagingGateLimitsFromEnv } from "../staging-gate.js";
8
9
  import { readDiskFootprint } from "../disk-usage.js";
9
10
  import { runEvidenceRedelivery } from "../evidence-redelivery.js";
10
11
  import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths } from "../local-state.js";
11
12
  import { runRawEvidenceLocalGc, rawEvidenceGcSummary } from "../raw-evidence-gc.js";
12
13
  import { runBackfillCommand } from "./backfill.js";
13
- import { FOOTPRINT_NEEDS_FIX_BYTES, diskRowMessage, footprintHolder, footprintTotalBytes, mib, redeliveryLine, } from "./doctor-disk-words.js";
14
+ import { FOOTPRINT_NEEDS_FIX_BYTES, stagingPausedWords, diskRowMessage, footprintHolder, footprintTotalBytes, mib, redeliveryLine, } from "./doctor-disk-words.js";
14
15
  import { doctorRoots } from "./doctor-access.js";
15
16
  import { backfillCompletionStepState, backfillFixVerdict, jsonField, parseDoctorBackfillJson, parseDoctorSyncJson, syncFailureMessage, syncBacklogDrainingVerdict, syncStandAsideVerdict, } from "./doctor-pipeline-verdicts.js";
16
17
  import { asRecord, fail, needsFix, ok, skipped } from "./doctor-report.js";
@@ -162,34 +163,55 @@ export async function checkDiskState(context) {
162
163
  const footprint = await readDiskFootprint(getCollectorRuntimePaths(context.command.homeDir));
163
164
  const { capBytes } = retentionOptionsFromEnv(context.io.env);
164
165
  const message = diskRowMessage(footprint, capBytes);
166
+ const paused = await stagingPausedRow(context, footprint, message);
167
+ if (paused)
168
+ return paused;
165
169
  if (footprint.staging.total_bytes - (footprint.staging.terminal_bytes ?? 0) > capBytes) {
166
170
  return needsFix("disk-bounded", "over_staging_cap", message);
167
171
  }
168
- const over = footprintOverSentence(footprint);
172
+ const over = footprintOverSentence(footprint, stagingGateLimitsFromEnv(context.io.env).ceilingBytes);
169
173
  if (over)
170
174
  return needsFix("disk-bounded", "footprint_over_512mb", `${over}; ${message}`);
171
175
  return ok("disk-bounded", "within_staging_cap", message);
172
176
  }
177
+ /**
178
+ * BLI-12832: the row for a paused staging gate, or null when copying is open.
179
+ * Plain words first, then the numbers; logged either way so a green row can
180
+ * still say the gate was checked.
181
+ */
182
+ async function stagingPausedRow(context, footprint, message) {
183
+ const gate = await readStagingGate(getCollectorRuntimePaths(context.command.homeDir).state_dir, context.io.env);
184
+ const words = stagingPausedWords(gate, footprintHolder(footprint));
185
+ console.error("[cockpit-doctor] staging gate", JSON.stringify({
186
+ reason: gate.reason ?? "open",
187
+ staging_bytes: gate.staging_bytes,
188
+ collector_bytes: gate.collector_bytes,
189
+ free_bytes: gate.free_bytes,
190
+ }));
191
+ if (!words || !gate.reason)
192
+ return null;
193
+ return needsFix("disk-bounded", `staging_paused_${gate.reason}`, `${words} ${message}`);
194
+ }
173
195
  /**
174
196
  * "812.0 MB held, over the 512 MB line; most of it is sync logs (600.0 MB)",
175
197
  * or null when the footprint is under the line. Logged either way, because a
176
198
  * row that only speaks when it is red cannot say the check ran.
177
199
  */
178
- export function footprintOverSentence(footprint) {
200
+ export function footprintOverSentence(footprint, ceilingBytes = FOOTPRINT_NEEDS_FIX_BYTES) {
179
201
  const total = footprintTotalBytes(footprint);
180
202
  const holder = footprintHolder(footprint);
181
- const over = total > FOOTPRINT_NEEDS_FIX_BYTES;
203
+ const over = total > ceilingBytes;
182
204
  console.error("[cockpit-doctor] collector footprint", JSON.stringify({
183
205
  reason: over ? "footprint_over_512mb" : "footprint_within_512mb",
184
206
  total_bytes: total,
185
- limit_bytes: FOOTPRINT_NEEDS_FIX_BYTES,
207
+ limit_bytes: ceilingBytes,
186
208
  holder: holder?.label ?? null,
187
209
  holder_bytes: holder?.bytes ?? 0,
188
210
  study_entries_not_counted: footprint.state.study_entries,
189
211
  }));
190
212
  if (!over)
191
213
  return null;
192
- return `${mib(total)} MB held, over the ${mib(FOOTPRINT_NEEDS_FIX_BYTES)} MB line; most of it is ${holder?.label ?? "unknown"} (${mib(holder?.bytes ?? 0)} MB)`;
214
+ return `${mib(total)} MB held, over the ${mib(ceilingBytes)} MB line; most of it is ${holder?.label ?? "unknown"} (${mib(holder?.bytes ?? 0)} MB)`;
193
215
  }
194
216
  export async function fixDiskState(context) {
195
217
  const paths = getCollectorRuntimePaths(context.command.homeDir);
@@ -216,11 +238,16 @@ export async function fixDiskState(context) {
216
238
  }
217
239
  const footprint = await readDiskFootprint(paths);
218
240
  const message = diskRowMessage(footprint, pruned.cap_bytes);
219
- const over = pruned.cap_blocked_by_uncommitted ? null : footprintOverSentence(footprint);
241
+ const over = pruned.cap_blocked_by_uncommitted
242
+ ? null
243
+ : footprintOverSentence(footprint, stagingGateLimitsFromEnv(context.io.env).ceilingBytes);
220
244
  if (over) {
221
245
  return needsFix("disk-bounded", "footprint_over_512mb", `freed ${mib(pruned.deleted_bytes)} MB; ${over}; ${message}`);
222
246
  }
223
247
  if (!pruned.cap_blocked_by_uncommitted) {
248
+ const stillPaused = await stagingPausedRow(context, footprint, `freed ${mib(pruned.deleted_bytes)} MB; ${message}`);
249
+ if (stillPaused)
250
+ return stillPaused;
224
251
  return ok("disk-bounded", `freed_${pruned.deleted_files}`, `freed ${mib(pruned.deleted_bytes)} MB; ${message}`);
225
252
  }
226
253
  // Unaccepted evidence still occupies the cap. Keep the disk row actionable.