@bli-cockpit/cli 0.2.168 → 0.2.173
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/raw-evidence-collection-state.js +52 -6
- package/dist/adapters/raw-evidence-completeness.js +6 -0
- package/dist/adapters/raw-evidence-git-diff-reader.js +2 -2
- package/dist/adapters/raw-evidence-image-reader.js +5 -2
- package/dist/adapters/raw-evidence-transcript-file.js +2 -2
- package/dist/autostart-opt-out.js +43 -0
- package/dist/autostart-self-heal.js +23 -10
- package/dist/commands/autostart-command.js +10 -0
- package/dist/commands/autostart-heal.js +6 -4
- package/dist/commands/backfill-upload.js +12 -0
- package/dist/commands/doctor-access.js +17 -21
- package/dist/commands/doctor-disk-words.js +20 -0
- package/dist/commands/doctor-mcp.js +27 -0
- package/dist/commands/doctor-pipeline.js +34 -7
- package/dist/commands/doctor-registration.js +9 -3
- package/dist/commands/doctor-report.js +28 -0
- package/dist/commands/doctor.js +20 -5
- package/dist/commands/heartbeat-token.js +14 -0
- package/dist/commands/heartbeat.js +2 -1
- package/dist/commands/local-args-tower-logs.js +53 -0
- package/dist/commands/local-args-tower.js +1 -0
- package/dist/commands/local-args.js +3 -1
- package/dist/commands/local-command-shapes-logs.js +6 -0
- package/dist/commands/local-help-commands-tower-ops.js +26 -0
- package/dist/commands/local-help.js +2 -0
- package/dist/commands/local.js +3 -0
- package/dist/commands/logs.js +68 -0
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/sync-followups-staging.js +8 -1
- package/dist/commands/sync-heal.js +144 -0
- package/dist/commands/sync.js +9 -1
- package/dist/disk-prune.js +35 -3
- package/dist/disk-retention.js +31 -1
- package/dist/sign-in-action.js +7 -0
- package/dist/staging-gate-receipt.js +131 -0
- package/dist/staging-gate.js +214 -0
- package/dist/state-dir-audit.js +5 -4
- package/dist/state-dir-prune.js +84 -0
- package/dist/sync-tick-stamp.js +22 -0
- package/package.json +4 -4
|
@@ -26,7 +26,8 @@ import { RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES } from "@bli-cockpit/telemetry-core"
|
|
|
26
26
|
import crypto from "node:crypto";
|
|
27
27
|
import fs from "node:fs/promises";
|
|
28
28
|
import path from "node:path";
|
|
29
|
-
import { DELIVERY_BACKOFF_BYPASS_REASON, deliveryBackoffApplies, heldSourceKeys, readRawEvidenceStagingState, resolveStagedObject, } from "../raw-evidence-staging.js";
|
|
29
|
+
import { DELIVERY_BACKOFF_BYPASS_REASON, deliveryBackoffApplies, heldSourceKeys, readRawEvidenceStagingState, resolveStagedObject, terminalDeliveryReason, } from "../raw-evidence-staging.js";
|
|
30
|
+
import { admitToStaging, openStagingAllowance, } from "../staging-gate.js";
|
|
30
31
|
import { chmodPrivate } from "./raw-evidence-pack-store.js";
|
|
31
32
|
import { GIT_DIFF_TIMEOUT_MS, MAX_GIT_DIFF_BYTES, } from "./raw-evidence-git-diff.js";
|
|
32
33
|
export const DEFAULT_SINCE_MINUTES = 24 * 60;
|
|
@@ -54,6 +55,11 @@ export async function openCollection(context, options, places) {
|
|
|
54
55
|
const byteBudget = options.byteBudget ?? RAW_EVIDENCE_DEFAULT_BYTE_BUDGET;
|
|
55
56
|
const objectBudget = options.objectBudget ?? RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET;
|
|
56
57
|
const staging = await readRawEvidenceStagingState(options.stateDir);
|
|
58
|
+
const budget = options.budget ?? {
|
|
59
|
+
remainingBytes: byteBudget,
|
|
60
|
+
remainingObjects: objectBudget,
|
|
61
|
+
};
|
|
62
|
+
const stagingAllowance = (budget.staging ??= await openStagingAllowance(options.stateDir, options.env ?? process.env, { statfs: options.statfs }));
|
|
57
63
|
return {
|
|
58
64
|
context,
|
|
59
65
|
filesDir: path.join(places.stagingDir, "files"),
|
|
@@ -71,14 +77,16 @@ export async function openCollection(context, options, places) {
|
|
|
71
77
|
redacted: [],
|
|
72
78
|
reused: [],
|
|
73
79
|
scanned: new Map(),
|
|
74
|
-
caps: startingCaps({
|
|
80
|
+
caps: startingCaps({
|
|
81
|
+
byteBudget,
|
|
82
|
+
objectBudget,
|
|
83
|
+
budget: options.budget,
|
|
84
|
+
stagingCapBytes: stagingAllowance.snapshot.cap_bytes,
|
|
85
|
+
}),
|
|
75
86
|
skipContentHashes: options.skipContentHashes ?? new Set(),
|
|
76
87
|
priorSnapshots: options.priorSnapshots ?? new Map(),
|
|
77
88
|
stubReoffers: options.stubReoffers ?? new Map(),
|
|
78
|
-
budget
|
|
79
|
-
remainingBytes: byteBudget,
|
|
80
|
-
remainingObjects: objectBudget,
|
|
81
|
-
},
|
|
89
|
+
budget,
|
|
82
90
|
index: { value: 0 },
|
|
83
91
|
};
|
|
84
92
|
}
|
|
@@ -118,6 +126,12 @@ function startingCaps(options) {
|
|
|
118
126
|
observed: options.budget?.remainingObjects ?? options.objectBudget,
|
|
119
127
|
applied: false,
|
|
120
128
|
},
|
|
129
|
+
{
|
|
130
|
+
source: "raw_evidence",
|
|
131
|
+
cap_type: "staging_gate",
|
|
132
|
+
limit: options.stagingCapBytes,
|
|
133
|
+
applied: false,
|
|
134
|
+
},
|
|
121
135
|
{
|
|
122
136
|
source: "git_diff",
|
|
123
137
|
cap_type: "max_bytes_per_diff",
|
|
@@ -185,6 +199,38 @@ export async function stageEvidenceBytes(collection, options) {
|
|
|
185
199
|
collection.stagedNewCount += 1;
|
|
186
200
|
return { local_path: destination, staged_in_pack: true };
|
|
187
201
|
}
|
|
202
|
+
/**
|
|
203
|
+
* The one door a file passes before its bytes may be copied: the per-sync
|
|
204
|
+
* budget below, then the disk (BLI-12832). A refusal from the disk hands the
|
|
205
|
+
* budget back, because nothing was staged and the next tick re-reads the source.
|
|
206
|
+
* A copy already staged costs the disk nothing and is never refused by it.
|
|
207
|
+
*/
|
|
208
|
+
export function admitEvidence(collection, byteLength, file) {
|
|
209
|
+
const budgetReason = admitToBudget(collection.budget, byteLength);
|
|
210
|
+
if (budgetReason)
|
|
211
|
+
return budgetReason;
|
|
212
|
+
const allowance = collection.budget.staging;
|
|
213
|
+
if (!allowance)
|
|
214
|
+
return null;
|
|
215
|
+
// BLI-12832: bytes the server refused for good are reclaimed by the prune
|
|
216
|
+
// (the refusal stays as a receipt), so copying them again would be churn.
|
|
217
|
+
if (collection.staging.staged[file.contentHash] === undefined &&
|
|
218
|
+
terminalDeliveryReason(collection.staging.delivery_attempts[file.contentHash]?.last_reason)) {
|
|
219
|
+
collection.budget.remainingObjects += 1;
|
|
220
|
+
collection.budget.remainingBytes += byteLength;
|
|
221
|
+
console.error("[staging-gate] not copying bytes the server refused for good", JSON.stringify({ reason: "delivery_terminal_refused", kind: file.kind, file_bytes: byteLength }));
|
|
222
|
+
return "delivery_terminal_refused";
|
|
223
|
+
}
|
|
224
|
+
const diskReason = admitToStaging(allowance, byteLength, {
|
|
225
|
+
alreadyStaged: collection.staging.staged[file.contentHash] !== undefined,
|
|
226
|
+
kind: file.kind,
|
|
227
|
+
});
|
|
228
|
+
if (!diskReason)
|
|
229
|
+
return null;
|
|
230
|
+
collection.budget.remainingObjects += 1;
|
|
231
|
+
collection.budget.remainingBytes += byteLength;
|
|
232
|
+
return `staging_deferred:${diskReason}`;
|
|
233
|
+
}
|
|
188
234
|
/**
|
|
189
235
|
* Decrements the per-sync budget when a file fits, or returns a deferred-skip
|
|
190
236
|
* reason when it does not. The object budget bounds request count; the byte
|
|
@@ -21,6 +21,12 @@ export function markCapApplied(ledger, source, capType) {
|
|
|
21
21
|
cap.applied = true;
|
|
22
22
|
}
|
|
23
23
|
export function markBudgetCapApplied(ledger, reason) {
|
|
24
|
+
if (reason === "delivery_terminal_refused")
|
|
25
|
+
return;
|
|
26
|
+
if (reason.startsWith("staging_deferred:")) {
|
|
27
|
+
markCapApplied(ledger, "raw_evidence", "staging_gate");
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
24
30
|
markCapApplied(ledger, "raw_evidence", reason === "deferred_object_budget" ? "object_budget" : "byte_budget");
|
|
25
31
|
}
|
|
26
32
|
// ---------------------------------------------------------------------------
|
|
@@ -19,7 +19,7 @@ import { sha256 } from "./raw-evidence-keys.js";
|
|
|
19
19
|
import { maskSecretsInGitDiff, } from "./raw-evidence-sanitize.js";
|
|
20
20
|
import { markBudgetCapApplied, markCapApplied, recordScanned, } from "./raw-evidence-completeness.js";
|
|
21
21
|
import { evidenceEntry } from "./raw-evidence-manifest.js";
|
|
22
|
-
import {
|
|
22
|
+
import { admitEvidence, stageEvidenceBytes, } from "./raw-evidence-collection-state.js";
|
|
23
23
|
import { MAX_GIT_DIFF_BYTES, REPO_ROOT_MISSING_REASON, RepoRootMissingError, repoRootExists, runGitDiff, } from "./raw-evidence-git-diff.js";
|
|
24
24
|
const GIT_DIFF_TARGETS = [
|
|
25
25
|
{ label: "unstaged", args: ["diff", "--no-ext-diff", "--"] },
|
|
@@ -130,7 +130,7 @@ async function stageOneGitDiff(collection, target) {
|
|
|
130
130
|
});
|
|
131
131
|
return;
|
|
132
132
|
}
|
|
133
|
-
const deferReason =
|
|
133
|
+
const deferReason = admitEvidence(collection, sanitized.bytes.byteLength, { contentHash, kind: "git_diff" });
|
|
134
134
|
if (deferReason) {
|
|
135
135
|
markBudgetCapApplied(collection, deferReason);
|
|
136
136
|
collection.skipped.push({
|
|
@@ -13,7 +13,7 @@ import { evidenceSourceKey } from "../raw-evidence-staging.js";
|
|
|
13
13
|
import { sha256 } from "./raw-evidence-keys.js";
|
|
14
14
|
import { markBudgetCapApplied, recordScanned, } from "./raw-evidence-completeness.js";
|
|
15
15
|
import { evidenceEntry } from "./raw-evidence-manifest.js";
|
|
16
|
-
import {
|
|
16
|
+
import { admitEvidence, stageEvidenceBytes, } from "./raw-evidence-collection-state.js";
|
|
17
17
|
export async function collectAgentImagesFromTranscript(collection, options) {
|
|
18
18
|
const result = await collectAgentImageEvidenceFromJsonlFile({
|
|
19
19
|
filePath: options.filePath,
|
|
@@ -56,7 +56,10 @@ async function collectOneAgentImageFile(collection, options) {
|
|
|
56
56
|
});
|
|
57
57
|
return;
|
|
58
58
|
}
|
|
59
|
-
const deferReason =
|
|
59
|
+
const deferReason = admitEvidence(collection, raw.byteLength, {
|
|
60
|
+
contentHash,
|
|
61
|
+
kind: options.kind,
|
|
62
|
+
});
|
|
60
63
|
if (deferReason) {
|
|
61
64
|
markBudgetCapApplied(collection, deferReason);
|
|
62
65
|
collection.skipped.push({
|
|
@@ -20,7 +20,7 @@ import { maskSecretsInTranscript, } from "./raw-evidence-sanitize.js";
|
|
|
20
20
|
import { markBudgetCapApplied, markCapApplied, } from "./raw-evidence-completeness.js";
|
|
21
21
|
import { evidenceEntry } from "./raw-evidence-manifest.js";
|
|
22
22
|
import { describeSnapshot, stubReplacementClaim, supersedeClaimFor, } from "./raw-evidence-snapshot-supersede.js";
|
|
23
|
-
import {
|
|
23
|
+
import { admitEvidence, stageEvidenceBytes, } from "./raw-evidence-collection-state.js";
|
|
24
24
|
/**
|
|
25
25
|
* Reads, secret-guards, content-addresses, budget-checks, and copies one
|
|
26
26
|
* attributed transcript into the pack.
|
|
@@ -82,7 +82,7 @@ export async function collectOneEvidenceFile(collection, options) {
|
|
|
82
82
|
});
|
|
83
83
|
return true;
|
|
84
84
|
}
|
|
85
|
-
const deferReason =
|
|
85
|
+
const deferReason = admitEvidence(collection, sanitized.bytes.byteLength, { contentHash, kind: options.kind });
|
|
86
86
|
if (deferReason) {
|
|
87
87
|
markBudgetCapApplied(collection, deferReason);
|
|
88
88
|
return skip(deferReason);
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { AUTOSTART_OPT_OUT_MARKER_FILE_NAME } from "@bli-cockpit/telemetry-core";
|
|
2
|
+
import fs from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
/**
|
|
5
|
+
* BLI-12828. The tick now CREATES a missing scheduler registration (an absent
|
|
6
|
+
* LaunchAgent / Scheduled Task is how a Mac goes quiet after a reboot), so
|
|
7
|
+
* "absent" can no longer double as "the operator removed it on purpose". This
|
|
8
|
+
* marker is that deliberate act: `cockpit autostart uninstall` writes it,
|
|
9
|
+
* `cockpit autostart install` and `cockpit doctor` clear it, and the tick's
|
|
10
|
+
* create-when-absent repair refuses while it stands.
|
|
11
|
+
*
|
|
12
|
+
* One small file under the state dir; it holds a timestamp, never a path.
|
|
13
|
+
*/
|
|
14
|
+
export const AUTOSTART_OPT_OUT_MARKER = AUTOSTART_OPT_OUT_MARKER_FILE_NAME;
|
|
15
|
+
function markerPath(paths) {
|
|
16
|
+
return path.join(paths.state_dir, AUTOSTART_OPT_OUT_MARKER);
|
|
17
|
+
}
|
|
18
|
+
export async function isAutostartOptedOut(paths) {
|
|
19
|
+
return fs
|
|
20
|
+
.stat(markerPath(paths))
|
|
21
|
+
.then(() => true)
|
|
22
|
+
.catch(() => false);
|
|
23
|
+
}
|
|
24
|
+
export async function markAutostartOptedOut(paths, now = new Date()) {
|
|
25
|
+
try {
|
|
26
|
+
await fs.mkdir(paths.state_dir, { recursive: true, mode: 0o700 });
|
|
27
|
+
await fs.writeFile(markerPath(paths), now.toISOString());
|
|
28
|
+
console.error("[autostart] opt-out recorded; the sync tick will not re-create the scheduler until `cockpit autostart install`", JSON.stringify({ reason: "operator_uninstalled" }));
|
|
29
|
+
}
|
|
30
|
+
catch (error) {
|
|
31
|
+
console.error("[autostart] opt-out could not be recorded; the tick may re-create the scheduler", JSON.stringify({
|
|
32
|
+
reason: "opt_out_write_failed",
|
|
33
|
+
error_name: error instanceof Error ? error.name : typeof error,
|
|
34
|
+
}));
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
export async function clearAutostartOptOut(paths) {
|
|
38
|
+
const had = await isAutostartOptedOut(paths);
|
|
39
|
+
await fs.rm(markerPath(paths), { force: true }).catch(() => undefined);
|
|
40
|
+
if (had) {
|
|
41
|
+
console.error("[autostart] opt-out cleared; background sync is wanted again", JSON.stringify({ reason: "operator_installed" }));
|
|
42
|
+
}
|
|
43
|
+
}
|
|
@@ -3,8 +3,12 @@ import path from "node:path";
|
|
|
3
3
|
import { spawn } from "node:child_process";
|
|
4
4
|
import { autostartStatus, installAutostartAgent, registeredRuntimePathProblems, } from "./autostart.js";
|
|
5
5
|
import { detectSecondCockpitInstall, secondInstallReason, } from "./second-install.js";
|
|
6
|
+
import { isAutostartOptedOut } from "./autostart-opt-out.js";
|
|
6
7
|
export const AUTOSTART_REPAIR_THROTTLE_MARKER = ".last-autostart-repair";
|
|
7
|
-
|
|
8
|
+
// BLI-12828: the FIRST repair of a break never waits. The marker only spaces
|
|
9
|
+
// out RETRIES of a repair that keeps failing, and a healthy reading clears it,
|
|
10
|
+
// so the next break is repaired on the next tick, not a day later.
|
|
11
|
+
const AUTOSTART_REPAIR_RETRY_AFTER_FAILURE_MS = 60 * 60 * 1000;
|
|
8
12
|
const DETAIL_MAX_CHARS = 300;
|
|
9
13
|
export async function runAutostartSelfHeal(paths, options) {
|
|
10
14
|
const platform = options.platform ?? process.platform;
|
|
@@ -20,9 +24,12 @@ export async function runAutostartSelfHeal(paths, options) {
|
|
|
20
24
|
exec: options.exec,
|
|
21
25
|
platform,
|
|
22
26
|
});
|
|
23
|
-
|
|
24
|
-
if (
|
|
27
|
+
const absent = status.status === "absent";
|
|
28
|
+
if (absent && (await isAutostartOptedOut(paths))) {
|
|
29
|
+
// Said on the tick's own log so "why is this Mac not ticking" is answerable.
|
|
30
|
+
console.error("[autostart-self-heal] scheduler absent; not re-creating it, the operator uninstalled it on purpose", JSON.stringify({ reason: "autostart_uninstalled_by_operator", platform }));
|
|
25
31
|
return null;
|
|
32
|
+
}
|
|
26
33
|
// A registration can read as "loaded" and still name a binary that no longer
|
|
27
34
|
// exists — that is precisely what `brew upgrade node` leaves behind. Ask the
|
|
28
35
|
// filesystem about the paths the SCHEDULER holds, not the ones this process
|
|
@@ -34,11 +41,16 @@ export async function runAutostartSelfHeal(paths, options) {
|
|
|
34
41
|
platform,
|
|
35
42
|
})
|
|
36
43
|
: [];
|
|
37
|
-
|
|
38
|
-
|
|
44
|
+
const marker = path.join(paths.state_dir, AUTOSTART_REPAIR_THROTTLE_MARKER);
|
|
45
|
+
if (!absent && status.status !== "not_loaded" && runtimeProblems.length === 0) {
|
|
46
|
+
// Healthy is the steady state and stays silent. A marker left by an
|
|
47
|
+
// earlier attempt is spent: that repair worked, so the next break starts
|
|
48
|
+
// fresh instead of waiting out a retry window.
|
|
49
|
+
await fs.rm(marker, { force: true }).catch(() => undefined);
|
|
39
50
|
return null;
|
|
40
51
|
}
|
|
41
52
|
const problem = [
|
|
53
|
+
...(absent ? ["registration absent"] : []),
|
|
42
54
|
...(status.message ? [status.message] : []),
|
|
43
55
|
...runtimeProblems,
|
|
44
56
|
]
|
|
@@ -59,15 +71,16 @@ export async function runAutostartSelfHeal(paths, options) {
|
|
|
59
71
|
step: platform === "darwin" ? "autostart_heal" : "autostart_repair",
|
|
60
72
|
};
|
|
61
73
|
}
|
|
62
|
-
//
|
|
63
|
-
// outcome
|
|
74
|
+
// Only a RETRY waits (marker mtime, written for the attempt not the
|
|
75
|
+
// outcome, cleared by the next healthy reading): a repair that keeps failing
|
|
76
|
+
// is retried hourly instead of churning the scheduler every 15 minutes. The
|
|
64
77
|
// status probe above still runs every tick — it is one launchctl/schtasks
|
|
65
78
|
// query.
|
|
66
79
|
const now = options.now ?? new Date();
|
|
67
|
-
const marker = path.join(paths.state_dir, AUTOSTART_REPAIR_THROTTLE_MARKER);
|
|
68
80
|
const lastAttempt = await fs.stat(marker).catch(() => null);
|
|
69
81
|
if (lastAttempt &&
|
|
70
|
-
now.getTime() - lastAttempt.mtimeMs <
|
|
82
|
+
now.getTime() - lastAttempt.mtimeMs < AUTOSTART_REPAIR_RETRY_AFTER_FAILURE_MS) {
|
|
83
|
+
console.error("[autostart-self-heal] scheduler still broken; the last repair attempt was under an hour ago, retrying after that", JSON.stringify({ reason: "repair_retry_waiting", platform }));
|
|
71
84
|
return { status: "skipped", reason: "repair_throttled_recent_attempt" };
|
|
72
85
|
}
|
|
73
86
|
await fs.mkdir(paths.state_dir, { recursive: true }).catch(() => undefined);
|
|
@@ -85,7 +98,7 @@ export async function runAutostartSelfHeal(paths, options) {
|
|
|
85
98
|
if (repaired.loaded) {
|
|
86
99
|
return {
|
|
87
100
|
status: "ok",
|
|
88
|
-
reason: "autostart_repaired",
|
|
101
|
+
reason: absent ? "autostart_created" : "autostart_repaired",
|
|
89
102
|
detail: problem,
|
|
90
103
|
step: "autostart_repair",
|
|
91
104
|
};
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { defaultExec, writeLine } from "./cli-io.js";
|
|
2
2
|
import { runAutostartHealDetached } from "./autostart-heal.js";
|
|
3
3
|
import { autostartStatus, installAutostartAgent, uninstallAutostartAgent, } from "../autostart.js";
|
|
4
|
+
import { clearAutostartOptOut, markAutostartOptedOut, } from "../autostart-opt-out.js";
|
|
4
5
|
import { getCollectorRuntimePaths, readLocalCollectorConfig, } from "../local-state.js";
|
|
5
6
|
export async function runAutostart(command, io) {
|
|
6
7
|
if (command.action === "heal-detached") {
|
|
@@ -37,6 +38,15 @@ export async function runAutostart(command, io) {
|
|
|
37
38
|
intervalSeconds: command.intervalSeconds,
|
|
38
39
|
exec,
|
|
39
40
|
});
|
|
41
|
+
// BLI-12828: the tick re-creates an absent scheduler, so a deliberate
|
|
42
|
+
// uninstall has to leave a mark the tick can read.
|
|
43
|
+
const optOutPaths = getCollectorRuntimePaths(command.homeDir);
|
|
44
|
+
if (command.action === "uninstall" && result.status !== "unsupported") {
|
|
45
|
+
await markAutostartOptedOut(optOutPaths);
|
|
46
|
+
}
|
|
47
|
+
else if (command.action === "install" && result.loaded !== false) {
|
|
48
|
+
await clearAutostartOptOut(optOutPaths);
|
|
49
|
+
}
|
|
40
50
|
if (command.json) {
|
|
41
51
|
writeLine(io.stdout, JSON.stringify(result, null, 2));
|
|
42
52
|
return autostartExitCode(command.action, result);
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import path from "node:path";
|
|
2
2
|
import { appendFile } from "node:fs/promises";
|
|
3
3
|
import { AUTOSTART_LABEL, autostartStatus, reregisterDarwinAgent, writeDarwinAutostartPlist, } from "../autostart.js";
|
|
4
|
+
import { isAutostartOptedOut } from "../autostart-opt-out.js";
|
|
4
5
|
import { redactedHealthDetail } from "../health-detail.js";
|
|
5
6
|
import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, readLocalCollectorConfig, } from "../local-state.js";
|
|
6
7
|
import { normalizeCollectionRoots } from "../root-normalization.js";
|
|
@@ -72,9 +73,10 @@ async function healDetached(command, io, deps) {
|
|
|
72
73
|
exec,
|
|
73
74
|
platform,
|
|
74
75
|
});
|
|
75
|
-
if (before.status === "absent"
|
|
76
|
-
|
|
77
|
-
|
|
76
|
+
if (before.status === "absent" &&
|
|
77
|
+
(await isAutostartOptedOut(getCollectorRuntimePaths(command.homeDir)))) {
|
|
78
|
+
// BLI-12828: absent is created now; only a deliberate uninstall stops it.
|
|
79
|
+
return { status: "skipped", reason: "autostart_uninstalled_by_operator" };
|
|
78
80
|
}
|
|
79
81
|
let written;
|
|
80
82
|
try {
|
|
@@ -125,7 +127,7 @@ async function healDetached(command, io, deps) {
|
|
|
125
127
|
}
|
|
126
128
|
return {
|
|
127
129
|
status: "ok",
|
|
128
|
-
reason: "autostart_healed",
|
|
130
|
+
reason: before.status === "absent" ? "autostart_created" : "autostart_healed",
|
|
129
131
|
detail: `node_path_reason=${written.node_path_reason}`,
|
|
130
132
|
};
|
|
131
133
|
}
|
|
@@ -7,6 +7,7 @@ import { countSessionUploadFailures, deferredEvidenceCount, durableBackfillCandi
|
|
|
7
7
|
import { blockingScanIssues } from "./backfill-issues.js";
|
|
8
8
|
import { ensureOutsideRootsWorkspace, isOutsideRootsWorkspace, } from "./backfill-outside-roots.js";
|
|
9
9
|
import { writeLine } from "./backfill-output.js";
|
|
10
|
+
import { stagingDeferredFiles } from "../staging-gate.js";
|
|
10
11
|
import { stubClaimsForCandidates } from "./backfill-stub-reoffer.js";
|
|
11
12
|
const BACKFILL_MAX_CONSECUTIVE_FAILURES = 3;
|
|
12
13
|
/**
|
|
@@ -92,6 +93,17 @@ export async function uploadBackfillBatches(command, io, lock, ctx) {
|
|
|
92
93
|
failureReason = "deferred_budget_exhausted";
|
|
93
94
|
break;
|
|
94
95
|
}
|
|
96
|
+
// BLI-12832: the disk gate refused a copy. Every later batch would be
|
|
97
|
+
// refused the same way, so stop here; the cursor only advanced through
|
|
98
|
+
// sessions that earned a durable pointer, and the next run starts from
|
|
99
|
+
// the same place once room returns.
|
|
100
|
+
const stagingDeferred = stagingDeferredFiles(rawEvidenceBudget.staging);
|
|
101
|
+
if (stagingDeferred > 0) {
|
|
102
|
+
blockedAt = stoppedAt("staging paused: not enough room to copy more evidence", index);
|
|
103
|
+
failureReason = "deferred_budget_exhausted";
|
|
104
|
+
console.error("[backfill] stopped, the staging gate is holding new copies", JSON.stringify({ reason: "staging_deferred", files_deferred: stagingDeferred, batch_index: index + 1 }));
|
|
105
|
+
break;
|
|
106
|
+
}
|
|
95
107
|
if (consecutiveFailures >= BACKFILL_MAX_CONSECUTIVE_FAILURES) {
|
|
96
108
|
blockedAt = stoppedAt("consecutive upload failures", index);
|
|
97
109
|
failureReason = failedBatchReason(outcome.sync, outcome.missingDurableMain);
|
|
@@ -8,6 +8,7 @@ import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, readLocalCollectorConf
|
|
|
8
8
|
import { normalizeCollectionRoots } from "../root-normalization.js";
|
|
9
9
|
import { detectSecondCockpitInstall } from "../second-install.js";
|
|
10
10
|
import { hardStop, needsFix, ok, skipped } from "./doctor-report.js";
|
|
11
|
+
import { SIGN_IN_ACTION } from "../sign-in-action.js";
|
|
11
12
|
/**
|
|
12
13
|
* The `authed`, `roots-ok`, and `single-install` check family: who is this
|
|
13
14
|
* machine and where does it collect. Auth and roots are hard stops with no
|
|
@@ -31,7 +32,7 @@ export async function fixAuthState(context, state) {
|
|
|
31
32
|
return checked;
|
|
32
33
|
}
|
|
33
34
|
if (!isInteractiveDoctorFix(context))
|
|
34
|
-
return { ...state, nextAction:
|
|
35
|
+
return { ...state, nextAction: SIGN_IN_ACTION };
|
|
35
36
|
const code = await context.deps.runLogin(context).catch((error) => {
|
|
36
37
|
// Exit code 1 with no reason at all is what an operator saw when doctor
|
|
37
38
|
// tried and failed to repair their auth — the same output as a login that
|
|
@@ -73,13 +74,10 @@ export async function readAuthState(context) {
|
|
|
73
74
|
}
|
|
74
75
|
return ok("authed", "device_token_present", "this machine is signed in");
|
|
75
76
|
}
|
|
76
|
-
return
|
|
77
|
-
"
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
` 2) If this is a reused machine, run \`${onboardOneLiner(context.command)}\` to refresh onboarding.`,
|
|
81
|
-
" 3) Send this output to Edward if approval is blocked.",
|
|
82
|
-
].join("\n"));
|
|
77
|
+
return {
|
|
78
|
+
...hardStop("authed", "pairing_required", `this machine is signed out. ${SIGN_IN_ACTION}`),
|
|
79
|
+
nextAction: SIGN_IN_ACTION,
|
|
80
|
+
};
|
|
83
81
|
}
|
|
84
82
|
export async function readRootState(context) {
|
|
85
83
|
const paths = getCollectorRuntimePaths(context.command.homeDir);
|
|
@@ -92,13 +90,11 @@ export async function readRootState(context) {
|
|
|
92
90
|
roots,
|
|
93
91
|
};
|
|
94
92
|
}
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
"
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
" 3) Run `cockpit doctor` interactively to choose approved roots.",
|
|
101
|
-
].join("\n"));
|
|
93
|
+
const action = `Run \`${onboardOneLiner(context.command)}\` in a terminal from the folder where you work.`;
|
|
94
|
+
return {
|
|
95
|
+
...hardStop("roots-ok", "no_roots", `no collection folders are saved yet. ${action}`),
|
|
96
|
+
nextAction: action,
|
|
97
|
+
};
|
|
102
98
|
}
|
|
103
99
|
/**
|
|
104
100
|
* BLI-3218/BLI-3553. Two Tower installs on one machine make each one's tick
|
|
@@ -164,10 +160,10 @@ function shellQuote(value) {
|
|
|
164
160
|
* a gate on it, so a session sitting outside one is not a defect doctor found
|
|
165
161
|
* and cannot repair. This row is therefore always informational (`ok`): it
|
|
166
162
|
* reports how many of this scan's sessions were inside vs. outside, never
|
|
167
|
-
* asks for approval, and attempts no fix. Widening the
|
|
168
|
-
* the
|
|
169
|
-
*
|
|
170
|
-
*
|
|
163
|
+
* asks for approval, and attempts no fix. Widening a root only adds the redacted transcript BYTES for those
|
|
164
|
+
* directories, so the message names `cockpit roots add` as an OPTIONAL step an
|
|
165
|
+
* operator can choose, never something doctor, or any person, is waiting on to
|
|
166
|
+
* approve. The
|
|
171
167
|
* row only turns red the ordinary way every other doctor check does: the scan
|
|
172
168
|
* itself throwing (`work_directory_scan_incomplete` from unreadable session
|
|
173
169
|
* folders), which `runDoctorWithDeps`'s existing `check_threw` handling
|
|
@@ -182,9 +178,9 @@ export async function readRootsCoverWork(context) {
|
|
|
182
178
|
const outsideCount = outside.reduce((sum, entry) => sum + entry.count, 0);
|
|
183
179
|
const insideCount = scan.observed - outsideCount;
|
|
184
180
|
if (!outsideCount)
|
|
185
|
-
return ok("roots-cover-work", "work_covered", `${scan.observed} sessions seen; no
|
|
181
|
+
return ok("roots-cover-work", "work_covered", `${scan.observed} sessions seen; no working folders outside your saved roots. ${sessionCoverageLine(scan, outsideCount)}`);
|
|
186
182
|
return {
|
|
187
|
-
...ok("roots-cover-work", "work_outside_roots_labelled", `${insideCount} of ${scan.observed} sessions inside
|
|
183
|
+
...ok("roots-cover-work", "work_outside_roots_labelled", `${insideCount} of ${scan.observed} sessions inside your saved roots, ${outsideCount} outside — every one already reached the server, the outside ones carrying an outside label. Nothing needs approving. Optional: add a folder as a root to also upload its redacted transcript text (scan ${scan.generated_at}):\n${outside.map(entry => `${entry.count}: ${entry.directory}\n ${rootsAddCommand(entry.directory, process.platform, homeDir)}`).join("\n")}\n\n${sessionCoverageLine(scan, outsideCount)}`),
|
|
188
184
|
nextAction: rootsAddCommand(outside[0].directory, process.platform, homeDir),
|
|
189
185
|
};
|
|
190
186
|
}
|
|
@@ -57,6 +57,26 @@ export function footprintBuckets(footprint) {
|
|
|
57
57
|
];
|
|
58
58
|
return buckets.filter((bucket) => bucket.bytes > 0).sort((a, b) => b.bytes - a.bytes);
|
|
59
59
|
}
|
|
60
|
+
/**
|
|
61
|
+
* BLI-12832: the sentence for a paused staging gate, in plain words. Every
|
|
62
|
+
* variant says the same two true things: nothing is lost (the source file is
|
|
63
|
+
* still on disk), and copying resumes by itself.
|
|
64
|
+
*/
|
|
65
|
+
export function stagingPausedWords(gate, holder) {
|
|
66
|
+
if (gate.reason === "low_disk") {
|
|
67
|
+
return `Tower paused copying new evidence: the disk has ${gib(gate.free_bytes ?? 0)} GB free. Nothing is lost; it resumes when space returns.`;
|
|
68
|
+
}
|
|
69
|
+
if (gate.reason === "state_over_ceiling") {
|
|
70
|
+
return `Tower paused copying new evidence: everything it keeps here is ${mib(gate.collector_bytes)} MB, over its ${mib(gate.ceiling_bytes)} MB ceiling${holder ? `, mostly ${holder.label} (${mib(holder.bytes)} MB)` : ""}. Nothing is lost; it resumes when that shrinks.`;
|
|
71
|
+
}
|
|
72
|
+
if (gate.reason === "staging_at_cap") {
|
|
73
|
+
return `Tower paused copying new evidence: its staging folder is full (${mib(gate.staging_bytes)} MB of a ${mib(gate.cap_bytes)} MB cap) with evidence the server has not accepted yet. Nothing is lost; the sources stay on disk and copying resumes once uploads succeed.`;
|
|
74
|
+
}
|
|
75
|
+
return null;
|
|
76
|
+
}
|
|
77
|
+
export function gib(bytes) {
|
|
78
|
+
return (bytes / (1024 * 1024 * 1024)).toFixed(1);
|
|
79
|
+
}
|
|
60
80
|
export function footprintTotalBytes(footprint) {
|
|
61
81
|
return footprintBuckets(footprint).reduce((sum, bucket) => sum + bucket.bytes, 0);
|
|
62
82
|
}
|
|
@@ -30,6 +30,12 @@
|
|
|
30
30
|
* ledger with its reason label (`doctor-report.ts`'s `doctorEvent`
|
|
31
31
|
* maps every other status to `ok`).
|
|
32
32
|
*
|
|
33
|
+
* BLI-12828: a `fail` (nothing answered) now has a repair. The server ships
|
|
34
|
+
* inside the CLI and is wired in by `memory install`, so the fix re-registers,
|
|
35
|
+
* takes the newest CLI if that alone did not wake it, and asks again; the row's
|
|
36
|
+
* receipt shows the before and after. A `needs_fix` (Tower refused) has none:
|
|
37
|
+
* the refusal is the sign-in's, and the `authed` row owns that.
|
|
38
|
+
*
|
|
33
39
|
* `skipped` when the bin is not on this machine at all, for the same reason
|
|
34
40
|
* `memory-registered` skips: nothing to probe is not a broken machine.
|
|
35
41
|
*/
|
|
@@ -95,4 +101,25 @@ function probeEnv(context) {
|
|
|
95
101
|
if (context.command.dashboardUrl)
|
|
96
102
|
base[TOWER_DASHBOARD_URL_ENV] = context.command.dashboardUrl;
|
|
97
103
|
return base;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* The `mcp-answers` repair (BLI-12828): re-register, then take the newest CLI,
|
|
107
|
+
* asking the server again after each. Heavy (it spawns the server and may run
|
|
108
|
+
* the updater), so doctor runs it only on a `fail` and the tick leaves it to
|
|
109
|
+
* the daily memory install and the CLI floor, which do the same two things.
|
|
110
|
+
*/
|
|
111
|
+
export async function fixMcpAnswersState(context, state) {
|
|
112
|
+
if (state.status !== "fail") {
|
|
113
|
+
console.error("[doctor] mcp-answers not repaired; Tower refused the call and the sign-in row owns that", JSON.stringify({ reason: state.code }));
|
|
114
|
+
return state;
|
|
115
|
+
}
|
|
116
|
+
await context.deps.fixMemory(context);
|
|
117
|
+
let checked = await context.deps.checkMcpAnswers(context);
|
|
118
|
+
console.error("[doctor] mcp-answers re-registered and asked again", JSON.stringify({ reason: checked.code, status: checked.status }));
|
|
119
|
+
if (checked.status === "fail") {
|
|
120
|
+
await context.deps.selfUpdate(context.io, { json: true, tag: context.command.updateTag });
|
|
121
|
+
checked = await context.deps.checkMcpAnswers(context);
|
|
122
|
+
console.error("[doctor] mcp-answers updated the CLI and asked again", JSON.stringify({ reason: checked.code, status: checked.status }));
|
|
123
|
+
}
|
|
124
|
+
return checked;
|
|
98
125
|
}
|
|
@@ -5,12 +5,13 @@ import { readBackfillCompletionMarker, readBackfillCursor, } from "../cursors/ba
|
|
|
5
5
|
import { savedDiscoveryLimitArgs } from "../discovery-limits.js";
|
|
6
6
|
import { runStagingPrune } from "../disk-prune.js";
|
|
7
7
|
import { retentionOptionsFromEnv } from "../disk-retention.js";
|
|
8
|
+
import { readStagingGate, stagingGateLimitsFromEnv } from "../staging-gate.js";
|
|
8
9
|
import { readDiskFootprint } from "../disk-usage.js";
|
|
9
10
|
import { runEvidenceRedelivery } from "../evidence-redelivery.js";
|
|
10
11
|
import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths } from "../local-state.js";
|
|
11
12
|
import { runRawEvidenceLocalGc, rawEvidenceGcSummary } from "../raw-evidence-gc.js";
|
|
12
13
|
import { runBackfillCommand } from "./backfill.js";
|
|
13
|
-
import { FOOTPRINT_NEEDS_FIX_BYTES, diskRowMessage, footprintHolder, footprintTotalBytes, mib, redeliveryLine, } from "./doctor-disk-words.js";
|
|
14
|
+
import { FOOTPRINT_NEEDS_FIX_BYTES, stagingPausedWords, diskRowMessage, footprintHolder, footprintTotalBytes, mib, redeliveryLine, } from "./doctor-disk-words.js";
|
|
14
15
|
import { doctorRoots } from "./doctor-access.js";
|
|
15
16
|
import { backfillCompletionStepState, backfillFixVerdict, jsonField, parseDoctorBackfillJson, parseDoctorSyncJson, syncFailureMessage, syncBacklogDrainingVerdict, syncStandAsideVerdict, } from "./doctor-pipeline-verdicts.js";
|
|
16
17
|
import { asRecord, fail, needsFix, ok, skipped } from "./doctor-report.js";
|
|
@@ -162,34 +163,55 @@ export async function checkDiskState(context) {
|
|
|
162
163
|
const footprint = await readDiskFootprint(getCollectorRuntimePaths(context.command.homeDir));
|
|
163
164
|
const { capBytes } = retentionOptionsFromEnv(context.io.env);
|
|
164
165
|
const message = diskRowMessage(footprint, capBytes);
|
|
166
|
+
const paused = await stagingPausedRow(context, footprint, message);
|
|
167
|
+
if (paused)
|
|
168
|
+
return paused;
|
|
165
169
|
if (footprint.staging.total_bytes - (footprint.staging.terminal_bytes ?? 0) > capBytes) {
|
|
166
170
|
return needsFix("disk-bounded", "over_staging_cap", message);
|
|
167
171
|
}
|
|
168
|
-
const over = footprintOverSentence(footprint);
|
|
172
|
+
const over = footprintOverSentence(footprint, stagingGateLimitsFromEnv(context.io.env).ceilingBytes);
|
|
169
173
|
if (over)
|
|
170
174
|
return needsFix("disk-bounded", "footprint_over_512mb", `${over}; ${message}`);
|
|
171
175
|
return ok("disk-bounded", "within_staging_cap", message);
|
|
172
176
|
}
|
|
177
|
+
/**
|
|
178
|
+
* BLI-12832: the row for a paused staging gate, or null when copying is open.
|
|
179
|
+
* Plain words first, then the numbers; logged either way so a green row can
|
|
180
|
+
* still say the gate was checked.
|
|
181
|
+
*/
|
|
182
|
+
async function stagingPausedRow(context, footprint, message) {
|
|
183
|
+
const gate = await readStagingGate(getCollectorRuntimePaths(context.command.homeDir).state_dir, context.io.env);
|
|
184
|
+
const words = stagingPausedWords(gate, footprintHolder(footprint));
|
|
185
|
+
console.error("[cockpit-doctor] staging gate", JSON.stringify({
|
|
186
|
+
reason: gate.reason ?? "open",
|
|
187
|
+
staging_bytes: gate.staging_bytes,
|
|
188
|
+
collector_bytes: gate.collector_bytes,
|
|
189
|
+
free_bytes: gate.free_bytes,
|
|
190
|
+
}));
|
|
191
|
+
if (!words || !gate.reason)
|
|
192
|
+
return null;
|
|
193
|
+
return needsFix("disk-bounded", `staging_paused_${gate.reason}`, `${words} ${message}`);
|
|
194
|
+
}
|
|
173
195
|
/**
|
|
174
196
|
* "812.0 MB held, over the 512 MB line; most of it is sync logs (600.0 MB)",
|
|
175
197
|
* or null when the footprint is under the line. Logged either way, because a
|
|
176
198
|
* row that only speaks when it is red cannot say the check ran.
|
|
177
199
|
*/
|
|
178
|
-
export function footprintOverSentence(footprint) {
|
|
200
|
+
export function footprintOverSentence(footprint, ceilingBytes = FOOTPRINT_NEEDS_FIX_BYTES) {
|
|
179
201
|
const total = footprintTotalBytes(footprint);
|
|
180
202
|
const holder = footprintHolder(footprint);
|
|
181
|
-
const over = total >
|
|
203
|
+
const over = total > ceilingBytes;
|
|
182
204
|
console.error("[cockpit-doctor] collector footprint", JSON.stringify({
|
|
183
205
|
reason: over ? "footprint_over_512mb" : "footprint_within_512mb",
|
|
184
206
|
total_bytes: total,
|
|
185
|
-
limit_bytes:
|
|
207
|
+
limit_bytes: ceilingBytes,
|
|
186
208
|
holder: holder?.label ?? null,
|
|
187
209
|
holder_bytes: holder?.bytes ?? 0,
|
|
188
210
|
study_entries_not_counted: footprint.state.study_entries,
|
|
189
211
|
}));
|
|
190
212
|
if (!over)
|
|
191
213
|
return null;
|
|
192
|
-
return `${mib(total)} MB held, over the ${mib(
|
|
214
|
+
return `${mib(total)} MB held, over the ${mib(ceilingBytes)} MB line; most of it is ${holder?.label ?? "unknown"} (${mib(holder?.bytes ?? 0)} MB)`;
|
|
193
215
|
}
|
|
194
216
|
export async function fixDiskState(context) {
|
|
195
217
|
const paths = getCollectorRuntimePaths(context.command.homeDir);
|
|
@@ -216,11 +238,16 @@ export async function fixDiskState(context) {
|
|
|
216
238
|
}
|
|
217
239
|
const footprint = await readDiskFootprint(paths);
|
|
218
240
|
const message = diskRowMessage(footprint, pruned.cap_bytes);
|
|
219
|
-
const over = pruned.cap_blocked_by_uncommitted
|
|
241
|
+
const over = pruned.cap_blocked_by_uncommitted
|
|
242
|
+
? null
|
|
243
|
+
: footprintOverSentence(footprint, stagingGateLimitsFromEnv(context.io.env).ceilingBytes);
|
|
220
244
|
if (over) {
|
|
221
245
|
return needsFix("disk-bounded", "footprint_over_512mb", `freed ${mib(pruned.deleted_bytes)} MB; ${over}; ${message}`);
|
|
222
246
|
}
|
|
223
247
|
if (!pruned.cap_blocked_by_uncommitted) {
|
|
248
|
+
const stillPaused = await stagingPausedRow(context, footprint, `freed ${mib(pruned.deleted_bytes)} MB; ${message}`);
|
|
249
|
+
if (stillPaused)
|
|
250
|
+
return stillPaused;
|
|
224
251
|
return ok("disk-bounded", `freed_${pruned.deleted_files}`, `freed ${mib(pruned.deleted_bytes)} MB; ${message}`);
|
|
225
252
|
}
|
|
226
253
|
// Unaccepted evidence still occupies the cap. Keep the disk row actionable.
|