@bli-cockpit/cli 0.2.46 → 0.2.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/raw-evidence-git-diff.js +50 -0
- package/dist/adapters/raw-evidence-keys.js +4 -1
- package/dist/adapters/raw-evidence-pack-store.js +13 -4
- package/dist/adapters/raw-evidence.js +43 -1
- package/dist/autostart-node-path.js +141 -0
- package/dist/autostart-self-heal.js +115 -11
- package/dist/autostart.js +213 -41
- package/dist/commands/autostart-heal.js +162 -0
- package/dist/commands/collection-roots.js +4 -4
- package/dist/commands/doctor.js +47 -1
- package/dist/commands/heartbeat.js +175 -0
- package/dist/commands/install-receipts.js +11 -19
- package/dist/commands/install-update.js +3 -3
- package/dist/commands/local-args.js +7 -1
- package/dist/commands/local.js +116 -10
- package/dist/commands/ops-render.js +29 -0
- package/dist/commands/ops.js +6 -0
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/session-sync.js +146 -25
- package/dist/evidence-upload-client.js +112 -4
- package/dist/evidence-upload-rekey.js +40 -0
- package/dist/log-rotation.js +144 -0
- package/dist/onboarding-roots.js +23 -6
- package/dist/raw-evidence-gc.js +9 -23
- package/dist/scheduled-self-update.js +1 -0
- package/dist/second-install.js +160 -0
- package/dist/sync-health-class.js +242 -0
- package/dist/upload.js +2 -0
- package/package.json +2 -2
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* "I am alive, and these are the folders I may look at." One POST per tick.
|
|
3
|
+
*
|
|
4
|
+
* BLI-3551. The dashboard learned a device was alive only when an ambient
|
|
5
|
+
* envelope arrived, and an envelope only arrives when there was something to
|
|
6
|
+
* collect. A machine whose operator works entirely outside the approved roots
|
|
7
|
+
* therefore went silent while working perfectly: 377 ticks in 38 hours without
|
|
8
|
+
* `last_seen_at` moving once. Dead and quiet looked identical.
|
|
9
|
+
*
|
|
10
|
+
* Three deliberate choices:
|
|
11
|
+
*
|
|
12
|
+
* - **Not spooled.** The install-events outbox retries for days, which is right
|
|
13
|
+
* for a receipt and wrong for a heartbeat: a heartbeat delivered tomorrow is
|
|
14
|
+
* a lie about today. A failed send is logged and dropped, and the next tick
|
|
15
|
+
* is fifteen minutes away.
|
|
16
|
+
* - **Never throws, never fails the sync.** Same rule as the self-update tail.
|
|
17
|
+
* - **Roots travel as labels.** `{ basename, path_sha256 }` — the folder name a
|
|
18
|
+
* person recognises plus the stable identity of the normalized absolute path.
|
|
19
|
+
* The path itself never leaves the machine, exactly as with `cwd_hash` on the
|
|
20
|
+
* session rows.
|
|
21
|
+
*/
|
|
22
|
+
import crypto from "node:crypto";
|
|
23
|
+
import fs from "node:fs";
|
|
24
|
+
import os from "node:os";
|
|
25
|
+
import path from "node:path";
|
|
26
|
+
import { COLLECTOR_HEARTBEAT_SCHEMA_VERSION, } from "@bli-cockpit/telemetry-core";
|
|
27
|
+
import { describeError } from "../health-detail.js";
|
|
28
|
+
import { getCollectorRuntimePaths, readLocalCollectorSessionFile, LOCAL_COLLECTOR_VERSION, } from "../local-state.js";
|
|
29
|
+
const HEARTBEAT_TIMEOUT_MS = 5_000;
|
|
30
|
+
/**
|
|
31
|
+
* The label form of one approved root.
|
|
32
|
+
*
|
|
33
|
+
* `path.resolve` first so `~/BLI`, `~/BLI/` and `~/BLI/.` are one identity, and
|
|
34
|
+
* on Windows so `c:\users\…` and `C:\Users\…` do not hash apart — the platform
|
|
35
|
+
* treats them as the same folder, and two machines pointing at it must compare
|
|
36
|
+
* equal. Case is folded ONLY on Windows, where the filesystem is
|
|
37
|
+
* case-insensitive; folding on macOS would merge two genuinely distinct roots.
|
|
38
|
+
*/
|
|
39
|
+
export function collectionRootLabel(root, platform = process.platform) {
|
|
40
|
+
const pathApi = platform === "win32" ? path.win32 : path.posix;
|
|
41
|
+
const resolved = platform === "win32"
|
|
42
|
+
? path.win32.normalize(root).replace(/[\\/]+$/u, "")
|
|
43
|
+
: path.posix.normalize(root).replace(/\/+$/u, "");
|
|
44
|
+
const canonical = platform === "win32" ? resolved.toLowerCase() : resolved;
|
|
45
|
+
const basename = pathApi.basename(resolved) || resolved;
|
|
46
|
+
return {
|
|
47
|
+
basename: basename.slice(0, 160),
|
|
48
|
+
path_sha256: crypto
|
|
49
|
+
.createHash("sha256")
|
|
50
|
+
.update(canonical, "utf8")
|
|
51
|
+
.digest("hex"),
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* The label set for a machine's approved roots, one entry per real folder.
|
|
56
|
+
*
|
|
57
|
+
* The collector deliberately keeps CONSENT ALIASES — on macOS an approved
|
|
58
|
+
* `/var/folders/…` root is held alongside its `/private/var/folders/…`
|
|
59
|
+
* realpath, so attribution matches either spelling. Both are true, and both
|
|
60
|
+
* name one folder; a heartbeat that shipped both would make every Mac look like
|
|
61
|
+
* it had twice the roots it has. Symlinks are resolved before hashing so the
|
|
62
|
+
* aliases collapse into the one folder they describe, and a path that cannot be
|
|
63
|
+
* resolved (deleted, or a Windows spelling on a POSIX host) keeps its own
|
|
64
|
+
* spelling rather than being dropped.
|
|
65
|
+
*/
|
|
66
|
+
export function collectionRootLabels(roots, platform = process.platform, realpath = defaultRealpath) {
|
|
67
|
+
const byHash = new Map();
|
|
68
|
+
for (const root of roots) {
|
|
69
|
+
if (!root.trim())
|
|
70
|
+
continue;
|
|
71
|
+
let resolved = root;
|
|
72
|
+
try {
|
|
73
|
+
resolved = realpath(root);
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
// Keep the spelling we were given: a root that is not on disk right now
|
|
77
|
+
// is still a root this machine is allowed to look at.
|
|
78
|
+
}
|
|
79
|
+
const label = collectionRootLabel(resolved, platform);
|
|
80
|
+
if (!byHash.has(label.path_sha256))
|
|
81
|
+
byHash.set(label.path_sha256, label);
|
|
82
|
+
}
|
|
83
|
+
return [...byHash.values()];
|
|
84
|
+
}
|
|
85
|
+
function defaultRealpath(input) {
|
|
86
|
+
return fs.realpathSync.native(input);
|
|
87
|
+
}
|
|
88
|
+
export function buildCollectorHeartbeat(options) {
|
|
89
|
+
const platform = options.platform ?? os.platform();
|
|
90
|
+
return {
|
|
91
|
+
schema_version: COLLECTOR_HEARTBEAT_SCHEMA_VERSION,
|
|
92
|
+
generated_at: (options.now ?? new Date()).toISOString(),
|
|
93
|
+
collector_version: options.collectorVersion ?? LOCAL_COLLECTOR_VERSION,
|
|
94
|
+
os_platform: platform,
|
|
95
|
+
roots: collectionRootLabels(options.roots, platform),
|
|
96
|
+
last_sync_status: options.facts.status,
|
|
97
|
+
...(options.facts.reason ? { last_sync_reason: options.facts.reason } : {}),
|
|
98
|
+
...(typeof options.facts.sessionsObserved === "number"
|
|
99
|
+
? { sessions_observed: options.facts.sessionsObserved }
|
|
100
|
+
: {}),
|
|
101
|
+
...(typeof options.facts.sessionsOutsideRoot === "number"
|
|
102
|
+
? { sessions_outside_root: options.facts.sessionsOutsideRoot }
|
|
103
|
+
: {}),
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Sends the heartbeat. Returns whether it landed; never throws.
|
|
108
|
+
*
|
|
109
|
+
* Every branch says something (BLI-3551 / the logging contract): a missing
|
|
110
|
+
* device session, a refusal, a transport failure and the success all get one
|
|
111
|
+
* line, because "did this machine check in today?" must be answerable from
|
|
112
|
+
* `sync.err.log` alone when the dashboard says a device is quiet.
|
|
113
|
+
*/
|
|
114
|
+
export async function sendCollectorHeartbeatBestEffort(options) {
|
|
115
|
+
const paths = getCollectorRuntimePaths(options.homeDir);
|
|
116
|
+
const session = await readLocalCollectorSessionFile(paths).catch(() => null);
|
|
117
|
+
if (!session ||
|
|
118
|
+
session.session_state !== "valid" ||
|
|
119
|
+
typeof session.device_token !== "string" ||
|
|
120
|
+
!session.device_token) {
|
|
121
|
+
console.error("[heartbeat] no valid device session on this machine; the dashboard will show it as quiet", JSON.stringify({
|
|
122
|
+
reason: "no_device_session",
|
|
123
|
+
next_action: "run `cockpit do-everything` to pair this machine again",
|
|
124
|
+
}));
|
|
125
|
+
return false;
|
|
126
|
+
}
|
|
127
|
+
const heartbeat = buildCollectorHeartbeat({
|
|
128
|
+
roots: options.roots,
|
|
129
|
+
facts: options.facts,
|
|
130
|
+
...(options.now ? { now: options.now } : {}),
|
|
131
|
+
});
|
|
132
|
+
const controller = new AbortController();
|
|
133
|
+
const timeout = setTimeout(() => controller.abort(), HEARTBEAT_TIMEOUT_MS);
|
|
134
|
+
try {
|
|
135
|
+
const response = await options.io.fetch(`${options.dashboardUrl}/api/ambient/heartbeat`, {
|
|
136
|
+
method: "POST",
|
|
137
|
+
headers: {
|
|
138
|
+
"Content-Type": "application/json",
|
|
139
|
+
Authorization: `Bearer ${session.device_token}`,
|
|
140
|
+
},
|
|
141
|
+
body: JSON.stringify(heartbeat),
|
|
142
|
+
signal: controller.signal,
|
|
143
|
+
});
|
|
144
|
+
if (!response.ok) {
|
|
145
|
+
// A heartbeat is not retried, so the reason has to be said once, here.
|
|
146
|
+
console.error("[heartbeat] the dashboard refused this tick's heartbeat", JSON.stringify({
|
|
147
|
+
reason: "heartbeat_rejected",
|
|
148
|
+
http_status: response.status,
|
|
149
|
+
root_count: heartbeat.roots.length,
|
|
150
|
+
}));
|
|
151
|
+
return false;
|
|
152
|
+
}
|
|
153
|
+
console.error("[heartbeat] checked in", JSON.stringify({
|
|
154
|
+
reason: "heartbeat_recorded",
|
|
155
|
+
http_status: response.status,
|
|
156
|
+
root_count: heartbeat.roots.length,
|
|
157
|
+
sync_status: heartbeat.last_sync_status,
|
|
158
|
+
sync_reason: heartbeat.last_sync_reason ?? null,
|
|
159
|
+
sessions_observed: heartbeat.sessions_observed ?? null,
|
|
160
|
+
sessions_outside_root: heartbeat.sessions_outside_root ?? null,
|
|
161
|
+
}));
|
|
162
|
+
return true;
|
|
163
|
+
}
|
|
164
|
+
catch (error) {
|
|
165
|
+
console.error("[heartbeat] could not reach the dashboard this tick", JSON.stringify({
|
|
166
|
+
reason: "heartbeat_transport_failed",
|
|
167
|
+
root_count: heartbeat.roots.length,
|
|
168
|
+
...describeError(error),
|
|
169
|
+
}));
|
|
170
|
+
return false;
|
|
171
|
+
}
|
|
172
|
+
finally {
|
|
173
|
+
clearTimeout(timeout);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
@@ -14,7 +14,6 @@ import { errorMessage, writeLine } from "./cli-io.js";
|
|
|
14
14
|
import { describeError, maskLocalIdentifiers, redactedHealthDetail, } from "../health-detail.js";
|
|
15
15
|
import { getCollectorRuntimePaths, readLocalCollectorSessionFile, LOCAL_COLLECTOR_VERSION, } from "../local-state.js";
|
|
16
16
|
import { redactSecretLikeContent } from "@bli-cockpit/telemetry-core";
|
|
17
|
-
import { COLLECTION_ROOT_REQUIRED } from "../onboarding-roots.js";
|
|
18
17
|
import { enqueueInstallEventEntry, readPendingInstallEventEntries, recordInstallEventAttemptFailure, removeInstallEventEntry, } from "../spool/install-event-outbox.js";
|
|
19
18
|
export function addInstallEvent(events, step, status, errorCode,
|
|
20
19
|
// BLI-2542: the bucket alone cannot be acted on. Callers that hold the reason
|
|
@@ -188,24 +187,17 @@ function classifyInstallTelemetryError(error) {
|
|
|
188
187
|
return "network";
|
|
189
188
|
return "failed";
|
|
190
189
|
}
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
// as a collection-root failure and the real reason was lost (BLI-2492).
|
|
203
|
-
if (message.includes(COLLECTION_ROOT_REQUIRED) ||
|
|
204
|
-
/collection root/iu.test(message)) {
|
|
205
|
-
return "collection_root_failed";
|
|
206
|
-
}
|
|
207
|
-
return "sync_failed";
|
|
208
|
-
}
|
|
190
|
+
/**
|
|
191
|
+
* The bucket a `sync_complete` receipt carries.
|
|
192
|
+
*
|
|
193
|
+
* The implementation moved to `../sync-health-class.js` in BLI-3551 and is
|
|
194
|
+
* re-exported here because this module's export surface is what `local.ts` and
|
|
195
|
+
* the tests import. It no longer reads the error's message: matching
|
|
196
|
+
* `/auth|token|session|…/` against the text filed the sync's own
|
|
197
|
+
* `session_report_unposted:no_successful_sync` label as `auth_failed` on every
|
|
198
|
+
* tick of a machine whose credentials were fine.
|
|
199
|
+
*/
|
|
200
|
+
export { classifySyncHealthError, classifySyncFailureRecords, } from "../sync-health-class.js";
|
|
209
201
|
// The bucket above is for aggregation. This is the reason — the actual message,
|
|
210
202
|
// redacted on the machine that produced it, before it ever leaves.
|
|
211
203
|
//
|
|
@@ -14,7 +14,7 @@ import { defaultExec, defaultInteractiveExec, writeExecOutput, writeLine } from
|
|
|
14
14
|
import { assertCollectionRootPersisted } from "./collection-roots.js";
|
|
15
15
|
import { addInstallEvent, reportInstallEventsBestEffort } from "./install-receipts.js";
|
|
16
16
|
import { installLocalCollector, LOCAL_COLLECTOR_VERSION, } from "../local-state.js";
|
|
17
|
-
import { COLLECTION_ROOT_REQUIRED, missingCollectionRootMessage, normalizeRootsDetailed, rootRejectionExplanation, } from "../onboarding-roots.js";
|
|
17
|
+
import { COLLECTION_ROOT_REQUIRED, CollectionRootRequiredError, missingCollectionRootMessage, normalizeRootsDetailed, rootRejectionExplanation, } from "../onboarding-roots.js";
|
|
18
18
|
export class SelfUpdateError extends Error {
|
|
19
19
|
result;
|
|
20
20
|
eacces;
|
|
@@ -75,9 +75,9 @@ function resolveInstallCommandRoots(command) {
|
|
|
75
75
|
};
|
|
76
76
|
}
|
|
77
77
|
if (detailed.rejected.length > 0) {
|
|
78
|
-
throw new
|
|
78
|
+
throw new CollectionRootRequiredError(rootRejectionExplanation(detailed.rejected[0], command));
|
|
79
79
|
}
|
|
80
|
-
throw new
|
|
80
|
+
throw new CollectionRootRequiredError(missingCollectionRootMessage(command));
|
|
81
81
|
}
|
|
82
82
|
export async function runUpdate(command, io) {
|
|
83
83
|
const installEvents = [];
|
|
@@ -776,6 +776,7 @@ function parseAutostartArgs(args) {
|
|
|
776
776
|
"--workspace",
|
|
777
777
|
"--dashboard-url",
|
|
778
778
|
"--interval-seconds",
|
|
779
|
+
"--parent-pid",
|
|
779
780
|
"--json",
|
|
780
781
|
],
|
|
781
782
|
valueFlags: [
|
|
@@ -784,13 +785,17 @@ function parseAutostartArgs(args) {
|
|
|
784
785
|
"--workspace",
|
|
785
786
|
"--dashboard-url",
|
|
786
787
|
"--interval-seconds",
|
|
788
|
+
"--parent-pid",
|
|
787
789
|
],
|
|
788
790
|
});
|
|
789
791
|
if (values.positionals.length > 1) {
|
|
790
792
|
throw new Error("autostart accepts at most one action (install|uninstall|status).");
|
|
791
793
|
}
|
|
792
794
|
const action = values.positionals[0] ?? "install";
|
|
793
|
-
if (action !== "install" &&
|
|
795
|
+
if (action !== "install" &&
|
|
796
|
+
action !== "uninstall" &&
|
|
797
|
+
action !== "status" &&
|
|
798
|
+
action !== "heal-detached") {
|
|
794
799
|
throw new Error("autostart action must be install, uninstall, or status.");
|
|
795
800
|
}
|
|
796
801
|
return {
|
|
@@ -800,6 +805,7 @@ function parseAutostartArgs(args) {
|
|
|
800
805
|
repoRoot: optionalNonEmpty(workRootFlagValue(values)),
|
|
801
806
|
dashboardUrl: normalizeUrl(values.flags.get("--dashboard-url") ?? DEFAULT_DASHBOARD_URL),
|
|
802
807
|
intervalSeconds: optionalPositiveInteger(values.flags.get("--interval-seconds"), "--interval-seconds") ?? DEFAULT_AUTOSTART_INTERVAL_SECONDS,
|
|
808
|
+
parentPid: optionalPositiveInteger(values.flags.get("--parent-pid"), "--parent-pid"),
|
|
803
809
|
json: values.booleans.has("--json"),
|
|
804
810
|
};
|
|
805
811
|
}
|
package/dist/commands/local.js
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
* local-help.ts what --help prints, and the recognised command names
|
|
10
10
|
* cli-io.ts stdout/stderr/stdin plumbing and the production CliIo
|
|
11
11
|
* install-receipts.ts the named step results this machine reports back
|
|
12
|
+
* heartbeat.ts the per-tick "I am alive, and these are my roots"
|
|
12
13
|
* local-auth.ts which email, the OTP exchange, pairing with fallback
|
|
13
14
|
* collection-roots.ts which folders may be collected, saved and read back
|
|
14
15
|
* local-discovery.ts finding git worktrees inside those roots
|
|
@@ -36,6 +37,8 @@
|
|
|
36
37
|
* settings.ts `cockpit settings` and `cockpit model`
|
|
37
38
|
* settings-render.ts how a settings answer reads in a terminal
|
|
38
39
|
* team.ts `cockpit team` — members, invite, role
|
|
40
|
+
* autostart-heal.ts the internal detached one-shot the macOS tick spawns
|
|
41
|
+
* to re-register launchd after the tick itself exits
|
|
39
42
|
*
|
|
40
43
|
* What stays here is orchestration: onboard, login/logout/start, the sync tick,
|
|
41
44
|
* analyze, serve, autostart and agent-rules.
|
|
@@ -44,7 +47,8 @@ import path from "node:path";
|
|
|
44
47
|
import { bufferedWritable, defaultExec, defaultIo, errorMessage, parseCapturedJson, replayCaptured, writeLine, } from "./cli-io.js";
|
|
45
48
|
import { isLocalHelpRequest, localCommandHelp } from "./local-help.js";
|
|
46
49
|
import { describeError, isMissingFileFailure } from "../health-detail.js";
|
|
47
|
-
import {
|
|
50
|
+
import { sendCollectorHeartbeatBestEffort, } from "./heartbeat.js";
|
|
51
|
+
import { addInstallEvent, classifySyncFailureRecords, classifySyncHealthError, redactedSyncErrorDetail, reportInstallEventsBestEffort, } from "./install-receipts.js";
|
|
48
52
|
import { canReuseOnboardSession, pairLocalCollectorWithAuthFallback, readOnboardSessionReuseCandidate, requestPairingAccessToken, requestPairingAccessTokenDetailed, resolveInteractiveLoginEmail, resolveOnboardEmail, } from "./local-auth.js";
|
|
49
53
|
import { collectionRootConsentAliases, persistOnboardingRootConfig, resolveOnboardingRootsForCommand, } from "./collection-roots.js";
|
|
50
54
|
import { discoverCommandWorktrees, rememberDiscoveryLimits } from "./local-discovery.js";
|
|
@@ -75,16 +79,19 @@ import { autostartStatus, installAutostartAgent, uninstallAutostartAgent, } from
|
|
|
75
79
|
import { DEFAULT_DASHBOARD_URL, ensureLocalCollectorConfig, getCollectorRuntimePaths, inspectLocalCollectorStatus, logoutLocalCollector, LOCAL_COLLECTOR_VERSION, readLocalCollectorConfig, readLocalCollectorSessionFile, startLocalWorkContext, } from "../local-state.js";
|
|
76
80
|
import { acquireSyncLock } from "../sync-lock.js";
|
|
77
81
|
import { runAttributedWorktreeSync, } from "./session-sync.js";
|
|
78
|
-
import { COLLECTION_ROOT_REQUIRED, missingCollectionRootMessage, } from "../onboarding-roots.js";
|
|
82
|
+
import { COLLECTION_ROOT_REQUIRED, CollectionRootRequiredError, missingCollectionRootMessage, } from "../onboarding-roots.js";
|
|
79
83
|
import { rawEvidenceDedupSummary, rawEvidenceGcSummary, runRawEvidenceLocalGc, sweepDuplicateStagedRawEvidence, } from "../raw-evidence-gc.js";
|
|
80
84
|
import { envWithNodeRuntimeOnPath, runScheduledSelfUpdate, } from "../scheduled-self-update.js";
|
|
81
85
|
import { runAutostartSelfHeal, } from "../autostart-self-heal.js";
|
|
86
|
+
import { rotateCollectorLogsBestEffort } from "../log-rotation.js";
|
|
87
|
+
import { runAutostartHealDetached } from "./autostart-heal.js";
|
|
82
88
|
import { normalizeCollectionRoots } from "../root-normalization.js";
|
|
83
89
|
// `./local.js` is the published entry point for this command surface: the
|
|
84
90
|
// public CLI's generated root, commands/root.ts, doctor.ts and the test suite
|
|
85
91
|
// all import from here. Splitting the file must not move a name off it.
|
|
86
92
|
export { rootCommandNames, localCommandHelp } from "./local-help.js";
|
|
87
|
-
export {
|
|
93
|
+
export { buildCollectorHeartbeat, collectionRootLabel, collectionRootLabels, sendCollectorHeartbeatBestEffort, } from "./heartbeat.js";
|
|
94
|
+
export { classifySyncFailureRecords, classifySyncHealthError, redactedSyncErrorDetail, reportInstallEventsBestEffort, SYNC_ERROR_DETAIL_MAX_CHARS, } from "./install-receipts.js";
|
|
88
95
|
export { assertCollectionRootPersisted } from "./collection-roots.js";
|
|
89
96
|
export { runSelfUpdate, SelfUpdateError, } from "./install-update.js";
|
|
90
97
|
export async function runLocalCockpitCli(argv, io = defaultIo()) {
|
|
@@ -1091,6 +1098,11 @@ async function runStart(command, io) {
|
|
|
1091
1098
|
}
|
|
1092
1099
|
async function runSync(command, io) {
|
|
1093
1100
|
const paths = getCollectorRuntimePaths(command.homeDir);
|
|
1101
|
+
// BLI-3553, first thing in the tick: cap the scheduler's own logs. Nothing
|
|
1102
|
+
// rotated them before, and two months of a 15-minute tick left 211 MB of
|
|
1103
|
+
// sync.log on the reference Mac. Best-effort by construction — a rotation
|
|
1104
|
+
// problem is its own log line, never a reason collection does not run.
|
|
1105
|
+
await rotateCollectorLogsBestEffort(paths);
|
|
1094
1106
|
const config = await readLocalCollectorConfig(paths).catch(() => null);
|
|
1095
1107
|
const dashboardUrl = command.dashboardUrl ?? config?.dashboard_url ?? DEFAULT_DASHBOARD_URL;
|
|
1096
1108
|
const minCliVersionAtStart = await reportInstallEventsBestEffort({
|
|
@@ -1103,6 +1115,10 @@ async function runSync(command, io) {
|
|
|
1103
1115
|
});
|
|
1104
1116
|
try {
|
|
1105
1117
|
const result = await runSyncWithHealthReceipt(command, io);
|
|
1118
|
+
// BLI-3551: every tick checks in, including one that collected nothing.
|
|
1119
|
+
// This is the only writer of `last_seen_at` that does not need an envelope,
|
|
1120
|
+
// so it is what separates a quiet machine from a dead one.
|
|
1121
|
+
await sendSyncHeartbeat(command, io, dashboardUrl, result.heartbeat);
|
|
1106
1122
|
const minCliVersion = await reportInstallEventsBestEffort({
|
|
1107
1123
|
homeDir: command.homeDir,
|
|
1108
1124
|
dashboardUrl,
|
|
@@ -1118,6 +1134,14 @@ async function runSync(command, io) {
|
|
|
1118
1134
|
return result.exitCode;
|
|
1119
1135
|
}
|
|
1120
1136
|
catch (error) {
|
|
1137
|
+
const errorCode = classifySyncHealthError(error);
|
|
1138
|
+
// A machine whose sync THREW is still alive, and that is worth knowing —
|
|
1139
|
+
// a device that stops checking in entirely is a different problem from one
|
|
1140
|
+
// checking in with a failure every fifteen minutes.
|
|
1141
|
+
await sendSyncHeartbeat(command, io, dashboardUrl, {
|
|
1142
|
+
status: "fail",
|
|
1143
|
+
reason: errorCode,
|
|
1144
|
+
});
|
|
1121
1145
|
const minCliVersion = await reportInstallEventsBestEffort({
|
|
1122
1146
|
homeDir: command.homeDir,
|
|
1123
1147
|
dashboardUrl,
|
|
@@ -1126,7 +1150,7 @@ async function runSync(command, io) {
|
|
|
1126
1150
|
{
|
|
1127
1151
|
step: "sync_complete",
|
|
1128
1152
|
status: "fail",
|
|
1129
|
-
error_code:
|
|
1153
|
+
error_code: errorCode,
|
|
1130
1154
|
error_detail: redactedSyncErrorDetail(error),
|
|
1131
1155
|
},
|
|
1132
1156
|
],
|
|
@@ -1138,6 +1162,28 @@ async function runSync(command, io) {
|
|
|
1138
1162
|
throw error;
|
|
1139
1163
|
}
|
|
1140
1164
|
}
|
|
1165
|
+
/**
|
|
1166
|
+
* The tick's check-in (BLI-3551).
|
|
1167
|
+
*
|
|
1168
|
+
* Resolving the roots is best-effort on purpose: a machine with NO approved
|
|
1169
|
+
* root is exactly the machine whose silence needs explaining, so it still
|
|
1170
|
+
* checks in — with an empty root list, which is itself the finding.
|
|
1171
|
+
*/
|
|
1172
|
+
async function sendSyncHeartbeat(command, io, dashboardUrl, facts) {
|
|
1173
|
+
const roots = await resolveSyncCollectionRoots(command).catch(() => []);
|
|
1174
|
+
await sendCollectorHeartbeatBestEffort({
|
|
1175
|
+
homeDir: command.homeDir,
|
|
1176
|
+
dashboardUrl,
|
|
1177
|
+
roots,
|
|
1178
|
+
facts,
|
|
1179
|
+
io,
|
|
1180
|
+
}).catch((error) => {
|
|
1181
|
+
// The sender already swallows everything it knows about; this is the net
|
|
1182
|
+
// for anything it does not, because a heartbeat must never fail a sync.
|
|
1183
|
+
console.error("[heartbeat] the check-in threw and was dropped", JSON.stringify({ reason: "heartbeat_threw", ...describeError(error) }));
|
|
1184
|
+
return false;
|
|
1185
|
+
});
|
|
1186
|
+
}
|
|
1141
1187
|
/**
|
|
1142
1188
|
* BLI-2721: after the tick's collection and self-update are done and
|
|
1143
1189
|
* reported, repair a broken/legacy autostart registration in place (Windows
|
|
@@ -1199,7 +1245,10 @@ async function reportAutostartSelfHealOutcome(command, io, dashboardUrl, result)
|
|
|
1199
1245
|
command: "sync",
|
|
1200
1246
|
events: [
|
|
1201
1247
|
{
|
|
1202
|
-
|
|
1248
|
+
// Windows repairs in place and keeps the name already in the receipts
|
|
1249
|
+
// and the runbook; the macOS path only SCHEDULES a detached repair, so
|
|
1250
|
+
// it reports under its own step (BLI-3553).
|
|
1251
|
+
step: result.step ?? "autostart_repair",
|
|
1203
1252
|
status: result.status,
|
|
1204
1253
|
...(result.status === "ok" ? {} : { error_code: result.reason }),
|
|
1205
1254
|
...(result.detail ? { error_detail: result.detail } : {}),
|
|
@@ -1298,10 +1347,24 @@ function scheduledSelfUpdateInstallEvent(result) {
|
|
|
1298
1347
|
? `forced_min_version ${result.min_version}`
|
|
1299
1348
|
: null;
|
|
1300
1349
|
if (result.status === "ok") {
|
|
1350
|
+
// BLI-3551: this used to be `update ok` with an empty detail unless the
|
|
1351
|
+
// floor forced it. One machine posted that receipt daily for nine releases
|
|
1352
|
+
// while sitting on 0.2.37, and nobody could tell "already current" from
|
|
1353
|
+
// "installed something" from "npm answered nothing" — three different
|
|
1354
|
+
// situations wearing one word. The success branch names itself now.
|
|
1355
|
+
const okDetail = [
|
|
1356
|
+
forcedDetail,
|
|
1357
|
+
result.reason === "updated" && result.previous_version && result.installed_version
|
|
1358
|
+
? `installed ${result.previous_version}→${result.installed_version}`
|
|
1359
|
+
: result.reason,
|
|
1360
|
+
result.target_version ? `target ${result.target_version}` : null,
|
|
1361
|
+
]
|
|
1362
|
+
.filter((part) => Boolean(part))
|
|
1363
|
+
.join("; ");
|
|
1301
1364
|
return {
|
|
1302
1365
|
step: "update",
|
|
1303
1366
|
status: "ok",
|
|
1304
|
-
...(
|
|
1367
|
+
...(okDetail ? { error_detail: okDetail } : {}),
|
|
1305
1368
|
};
|
|
1306
1369
|
}
|
|
1307
1370
|
const detail = [
|
|
@@ -1341,6 +1404,10 @@ async function runSyncWithHealthReceipt(command, io) {
|
|
|
1341
1404
|
status: "skipped",
|
|
1342
1405
|
error_code: "live_sync_paused_during_backfill",
|
|
1343
1406
|
},
|
|
1407
|
+
heartbeat: {
|
|
1408
|
+
status: "skipped",
|
|
1409
|
+
reason: "live_sync_paused_during_backfill",
|
|
1410
|
+
},
|
|
1344
1411
|
};
|
|
1345
1412
|
}
|
|
1346
1413
|
// Single-flight: a launchd timer and a manual sync must not interleave the
|
|
@@ -1367,14 +1434,28 @@ async function runSyncWithHealthReceipt(command, io) {
|
|
|
1367
1434
|
status: "skipped",
|
|
1368
1435
|
error_code: "sync_already_running",
|
|
1369
1436
|
},
|
|
1437
|
+
heartbeat: { status: "skipped", reason: "sync_already_running" },
|
|
1370
1438
|
};
|
|
1371
1439
|
}
|
|
1372
1440
|
try {
|
|
1373
|
-
const { exitCode, failureReasons } = await runSyncLocked(command, io);
|
|
1441
|
+
const { exitCode, failureReasons, failureRecords, notice, sessionsObserved, sessionsOutsideRoot, } = await runSyncLocked(command, io);
|
|
1442
|
+
const counts = {
|
|
1443
|
+
sessionsObserved,
|
|
1444
|
+
sessionsOutsideRoot,
|
|
1445
|
+
};
|
|
1374
1446
|
if (exitCode === 0) {
|
|
1447
|
+
// BLI-3551: an `ok` tick can still have something to say. `nothing_in_root`
|
|
1448
|
+
// is the receipt that separates "this machine is alive and its operator
|
|
1449
|
+
// works outside the approved roots" from "this machine is dead", which
|
|
1450
|
+
// until now looked identical from the dashboard.
|
|
1375
1451
|
return {
|
|
1376
1452
|
exitCode,
|
|
1377
|
-
completion: {
|
|
1453
|
+
completion: {
|
|
1454
|
+
step: "sync_complete",
|
|
1455
|
+
status: "ok",
|
|
1456
|
+
...(notice ? { error_detail: notice } : {}),
|
|
1457
|
+
},
|
|
1458
|
+
heartbeat: { status: "ok", reason: notice, ...counts },
|
|
1378
1459
|
};
|
|
1379
1460
|
}
|
|
1380
1461
|
// A sync that fails by exit code says exactly as much as one that throws.
|
|
@@ -1382,14 +1463,18 @@ async function runSyncWithHealthReceipt(command, io) {
|
|
|
1382
1463
|
// failure rows carried a null detail and the real reason was reachable only
|
|
1383
1464
|
// by running `cockpit status` on the machine itself (BLI-2526).
|
|
1384
1465
|
const reasonText = failureReasons.join("; ");
|
|
1466
|
+
// The bucket comes from the records the deciding branches wrote, not from
|
|
1467
|
+
// this sentence (BLI-3551). The sentence is still the detail.
|
|
1468
|
+
const errorCode = classifySyncFailureRecords(failureRecords);
|
|
1385
1469
|
return {
|
|
1386
1470
|
exitCode,
|
|
1387
1471
|
completion: {
|
|
1388
1472
|
step: "sync_complete",
|
|
1389
1473
|
status: "fail",
|
|
1390
|
-
error_code:
|
|
1474
|
+
error_code: errorCode,
|
|
1391
1475
|
error_detail: redactedSyncErrorDetail(reasonText),
|
|
1392
1476
|
},
|
|
1477
|
+
heartbeat: { status: "fail", reason: errorCode, ...counts },
|
|
1393
1478
|
};
|
|
1394
1479
|
}
|
|
1395
1480
|
finally {
|
|
@@ -1407,6 +1492,10 @@ function syncResult(run) {
|
|
|
1407
1492
|
return {
|
|
1408
1493
|
exitCode: run.ok ? 0 : 1,
|
|
1409
1494
|
failureReasons: run.ok ? [] : run.failure_reasons,
|
|
1495
|
+
failureRecords: run.ok ? [] : run.failure_records,
|
|
1496
|
+
notice: run.notice,
|
|
1497
|
+
sessionsObserved: run.sessions_observed,
|
|
1498
|
+
sessionsOutsideRoot: run.sessions_outside_root,
|
|
1410
1499
|
};
|
|
1411
1500
|
}
|
|
1412
1501
|
/**
|
|
@@ -1492,6 +1581,7 @@ async function reportNoWorktreeSync(command, io, run, dedup) {
|
|
|
1492
1581
|
mode: "no_worktrees",
|
|
1493
1582
|
status: collectionRunStatus,
|
|
1494
1583
|
collection_complete: run.ok,
|
|
1584
|
+
...(run.notice ? { notice: run.notice } : {}),
|
|
1495
1585
|
codex_sessions: run.summary,
|
|
1496
1586
|
raw_evidence_gc: gc,
|
|
1497
1587
|
raw_evidence_dedup: dedup,
|
|
@@ -1499,6 +1589,11 @@ async function reportNoWorktreeSync(command, io, run, dedup) {
|
|
|
1499
1589
|
return syncResult(run);
|
|
1500
1590
|
}
|
|
1501
1591
|
writeLine(run.ok ? io.stdout : io.stderr, `Tower sync ${collectionRunStatus}: no git worktrees under this root; session scan ran.`);
|
|
1592
|
+
if (run.notice) {
|
|
1593
|
+
// Says out loud what the receipt now says to the dashboard: the sessions
|
|
1594
|
+
// this machine ran were all outside the folders it is allowed to look at.
|
|
1595
|
+
writeLine(io.stdout, `Every session seen this run was outside your approved folders (${run.notice}). Nothing was collected, and nothing is broken.`);
|
|
1596
|
+
}
|
|
1502
1597
|
writeAgentSessionSummary(io, run.summary);
|
|
1503
1598
|
if (gc && !gc.skipped)
|
|
1504
1599
|
writeLine(io.stdout, rawEvidenceGcSummary(gc));
|
|
@@ -1558,7 +1653,7 @@ async function resolveSyncCollectionRoots(command) {
|
|
|
1558
1653
|
if (savedRoots.length > 0) {
|
|
1559
1654
|
return collectionRootConsentAliases(savedRoots);
|
|
1560
1655
|
}
|
|
1561
|
-
throw new
|
|
1656
|
+
throw new CollectionRootRequiredError(`no explicit or saved collection root is available.`);
|
|
1562
1657
|
}
|
|
1563
1658
|
async function runAnalyze(command, io) {
|
|
1564
1659
|
const syncStdout = [];
|
|
@@ -1701,6 +1796,17 @@ async function runServe(command, io) {
|
|
|
1701
1796
|
return 0;
|
|
1702
1797
|
}
|
|
1703
1798
|
async function runAutostart(command, io) {
|
|
1799
|
+
if (command.action === "heal-detached") {
|
|
1800
|
+
// BLI-3553: the internal one-shot the scheduled macOS tick spawns. It has
|
|
1801
|
+
// its own receipt and its own exit code; nothing else in this function
|
|
1802
|
+
// applies to it.
|
|
1803
|
+
return runAutostartHealDetached({
|
|
1804
|
+
homeDir: command.homeDir,
|
|
1805
|
+
dashboardUrl: command.dashboardUrl,
|
|
1806
|
+
parentPid: command.parentPid ?? null,
|
|
1807
|
+
json: command.json,
|
|
1808
|
+
}, io);
|
|
1809
|
+
}
|
|
1704
1810
|
const exec = io.exec ?? defaultExec();
|
|
1705
1811
|
const repoRoots = command.action === "install" || command.action === "status"
|
|
1706
1812
|
? await resolveAutostartRoots(command.homeDir, command.repoRoot)
|
|
@@ -80,6 +80,8 @@ export function renderOpsStatus(payload, dim) {
|
|
|
80
80
|
lines.push(dim(` scheduled by ${row.configFile} (${row.cron ?? "?"})`));
|
|
81
81
|
}
|
|
82
82
|
}
|
|
83
|
+
if (payload.fleet)
|
|
84
|
+
lines.push(...renderFleet(payload.fleet, dim));
|
|
83
85
|
const slack = payload.skips?.slack;
|
|
84
86
|
const external = payload.skips?.external;
|
|
85
87
|
if (slack || external) {
|
|
@@ -93,6 +95,33 @@ export function renderOpsStatus(payload, dim) {
|
|
|
93
95
|
}
|
|
94
96
|
return lines;
|
|
95
97
|
}
|
|
98
|
+
/**
|
|
99
|
+
* The laptops. Red first, because a person reading this at 9am should not have
|
|
100
|
+
* to scroll past nine healthy machines to find the dead one.
|
|
101
|
+
*/
|
|
102
|
+
export function renderFleet(fleet, dim) {
|
|
103
|
+
const lines = ["", `FLEET ${fleet.summary ?? "(no summary)"}`];
|
|
104
|
+
if (fleet.readError) {
|
|
105
|
+
lines.push(` the fleet could not be read (${fleet.readError}); nothing is known about any machine`);
|
|
106
|
+
return lines;
|
|
107
|
+
}
|
|
108
|
+
const devices = fleet.devices ?? [];
|
|
109
|
+
const rank = (colour) => colour === "red" ? 0 : colour === "amber" ? 1 : 2;
|
|
110
|
+
const ordered = [...devices].sort((left, right) => rank(left.colour) - rank(right.colour));
|
|
111
|
+
if (ordered.length === 0) {
|
|
112
|
+
lines.push(dim(" no live collector device is registered at all"));
|
|
113
|
+
}
|
|
114
|
+
for (const device of ordered) {
|
|
115
|
+
const text = ` ${device.line ?? device.deviceId ?? "(device)"}`;
|
|
116
|
+
// A healthy machine is dimmed, never dropped: "which laptops are fine" is
|
|
117
|
+
// the other half of the question, and a list that only shows failures
|
|
118
|
+
// cannot answer "is everybody else collecting?".
|
|
119
|
+
lines.push(device.colour === "red" || device.colour === "amber" ? text : dim(text));
|
|
120
|
+
}
|
|
121
|
+
if (fleet.noDeviceLine)
|
|
122
|
+
lines.push(dim(` ${fleet.noDeviceLine}`));
|
|
123
|
+
return lines;
|
|
124
|
+
}
|
|
96
125
|
function renderSkipLedger(ledger, dim) {
|
|
97
126
|
const lines = [];
|
|
98
127
|
const name = ledger.relation ?? "skips";
|
package/dist/commands/ops.js
CHANGED
|
@@ -78,6 +78,12 @@ async function runOpsStatus(command, io, tower) {
|
|
|
78
78
|
unhealthy: unhealthy.length,
|
|
79
79
|
unhealthy_ids: unhealthy.map((row) => row.id ?? "?"),
|
|
80
80
|
with_skips: Boolean(command.skips),
|
|
81
|
+
// The laptops (BLI-3550), counted on the same line: a run that shows a
|
|
82
|
+
// green board and says nothing about the fleet cannot answer "was
|
|
83
|
+
// anybody's machine dead this morning?".
|
|
84
|
+
fleet_devices: payload.fleet?.counts?.devices ?? null,
|
|
85
|
+
fleet_red: payload.fleet?.counts?.red ?? null,
|
|
86
|
+
fleet_amber: payload.fleet?.counts?.amber ?? null,
|
|
81
87
|
})}`);
|
|
82
88
|
return unhealthy.length > 0 ? 1 : 0;
|
|
83
89
|
}
|
|
@@ -15,7 +15,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
15
15
|
}
|
|
16
16
|
|
|
17
17
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
18
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
18
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.47");
|
|
19
19
|
return 0;
|
|
20
20
|
}
|
|
21
21
|
|