@bli-cockpit/cli 0.2.217 → 0.2.219
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/heartbeat-token-lock.js +138 -0
- package/dist/commands/heartbeat-token.js +41 -1
- package/dist/commands/heartbeat.js +23 -5
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/setup-receipt-lines.js +7 -6
- package/dist/commands/sync-heal.js +2 -2
- package/package.json +2 -2
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The single-flight lock around the heartbeat's token exchange (BLI-17994).
|
|
3
|
+
*
|
|
4
|
+
* The heartbeat is the one door a device token is offered a successor
|
|
5
|
+
* through, and it is called from three places: the launchd / Task Scheduler
|
|
6
|
+
* tick, a manual `cockpit sync`, and `cockpit doctor`. Two of them can fire in
|
|
7
|
+
* the same second. Without a lock both read `session.json` holding the same
|
|
8
|
+
* token, both knock with it, and the second's rotation request races the
|
|
9
|
+
* first's reply (the server answers "rotation_pending", but two writers of
|
|
10
|
+
* `session.json` still interleave). With it they take turns: the second one
|
|
11
|
+
* reads the file AFTER the first stored the pending token and knocks with that,
|
|
12
|
+
* which is exactly the "first use" that promotes it.
|
|
13
|
+
*
|
|
14
|
+
* The same code runs on macOS and Windows: an exclusive-create (`wx`) lock file
|
|
15
|
+
* in the state directory, no `flock`, no platform branch. A holder that died
|
|
16
|
+
* leaves a file that goes stale after a minute and is taken over.
|
|
17
|
+
*
|
|
18
|
+
* The lock is a courtesy, never a gate. The server is safe without it (a
|
|
19
|
+
* request can never destroy a token that is still in use), so a lock file that
|
|
20
|
+
* cannot be created at all (read-only volume, antivirus) lets the heartbeat go
|
|
21
|
+
* ahead, loudly. Only a lock that stays HELD past the wait skips this tick, and
|
|
22
|
+
* a heartbeat is not spooled: the next tick is fifteen minutes away.
|
|
23
|
+
*
|
|
24
|
+
* A new module here must also join `scripts/build-public-cli.mjs`
|
|
25
|
+
* `runtimeFiles`, or the repo tests stay green while the packed CLI breaks.
|
|
26
|
+
*/
|
|
27
|
+
import crypto from "node:crypto";
|
|
28
|
+
import fs from "node:fs/promises";
|
|
29
|
+
import path from "node:path";
|
|
30
|
+
import { describeError } from "../health-detail.js";
|
|
31
|
+
import { getCollectorRuntimePaths } from "../local-state.js";
|
|
32
|
+
const LOCK_FILENAME = "token-rotation.lock";
|
|
33
|
+
/** A heartbeat holds the lock for one HTTP call (5 s timeout) plus a few file reads. */
|
|
34
|
+
export const TOKEN_LOCK_STALE_MS = 60_000;
|
|
35
|
+
/** How long a second caller waits for the first before skipping this tick's heartbeat. */
|
|
36
|
+
export const TOKEN_LOCK_WAIT_MS = 8_000;
|
|
37
|
+
const POLL_MS = 150;
|
|
38
|
+
export async function acquireTokenRotationLock(homeDir, options = {}) {
|
|
39
|
+
const paths = getCollectorRuntimePaths(homeDir);
|
|
40
|
+
const lockPath = path.join(paths.state_dir, LOCK_FILENAME);
|
|
41
|
+
const waitMs = options.waitMs ?? TOKEN_LOCK_WAIT_MS;
|
|
42
|
+
const staleMs = options.staleMs ?? TOKEN_LOCK_STALE_MS;
|
|
43
|
+
const pollMs = options.pollMs ?? POLL_MS;
|
|
44
|
+
const owner = crypto.randomUUID();
|
|
45
|
+
const started = Date.now();
|
|
46
|
+
try {
|
|
47
|
+
await fs.mkdir(paths.state_dir, { recursive: true, mode: 0o700 });
|
|
48
|
+
}
|
|
49
|
+
catch (error) {
|
|
50
|
+
return unlocked("state_dir_unavailable", error, started);
|
|
51
|
+
}
|
|
52
|
+
for (;;) {
|
|
53
|
+
try {
|
|
54
|
+
const handle = await fs.open(lockPath, "wx", 0o600);
|
|
55
|
+
await handle.writeFile(`${JSON.stringify({ pid: process.pid, owner, at_ms: Date.now() })}\n`);
|
|
56
|
+
await handle.close();
|
|
57
|
+
const waited = Date.now() - started;
|
|
58
|
+
if (waited > pollMs) {
|
|
59
|
+
console.error("[heartbeat] waited for another heartbeat to finish its token exchange", JSON.stringify({ reason: "token_lock_acquired_after_wait", waited_ms: waited }));
|
|
60
|
+
}
|
|
61
|
+
return {
|
|
62
|
+
acquired: true,
|
|
63
|
+
waited_ms: waited,
|
|
64
|
+
mode: "held",
|
|
65
|
+
release: async () => {
|
|
66
|
+
// Only remove a lock that is still ours: a holder that was taken over
|
|
67
|
+
// as stale must not delete its successor's lock.
|
|
68
|
+
const current = await readOwner(lockPath);
|
|
69
|
+
if (current === owner)
|
|
70
|
+
await fs.rm(lockPath, { force: true }).catch(() => undefined);
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
if (error?.code !== "EEXIST") {
|
|
76
|
+
return unlocked("lock_file_unwritable", error, started);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
const heldAt = await readHeldAt(lockPath);
|
|
80
|
+
if (heldAt === "gone")
|
|
81
|
+
continue; // released between our create and our read: try again at once
|
|
82
|
+
if (Date.now() - heldAt > staleMs) {
|
|
83
|
+
// A holder that died: take the lock over.
|
|
84
|
+
console.error("[heartbeat] removing a stale token-exchange lock", JSON.stringify({ reason: "token_lock_stale_taken_over", age_ms: Date.now() - heldAt }));
|
|
85
|
+
await fs.rm(lockPath, { force: true }).catch(() => undefined);
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
if (Date.now() - started >= waitMs) {
|
|
89
|
+
console.error("[heartbeat] another heartbeat is still exchanging the device token; skipping this tick's check-in", JSON.stringify({ reason: "token_lock_busy", waited_ms: Date.now() - started, next_action: "the next tick checks in" }));
|
|
90
|
+
return { acquired: false, waited_ms: Date.now() - started };
|
|
91
|
+
}
|
|
92
|
+
await new Promise((resolve) => setTimeout(resolve, pollMs));
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
/** Run `work` under the lock; `skipped` is the answer when the lock stays held past the wait. */
|
|
96
|
+
export async function withTokenRotationLock(homeDir, options, skipped, work) {
|
|
97
|
+
const lock = await acquireTokenRotationLock(homeDir, options);
|
|
98
|
+
if (!lock.acquired)
|
|
99
|
+
return skipped;
|
|
100
|
+
try {
|
|
101
|
+
return await work();
|
|
102
|
+
}
|
|
103
|
+
finally {
|
|
104
|
+
await lock.release();
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
function unlocked(reason, error, started) {
|
|
108
|
+
console.error("[heartbeat] could not create the token-exchange lock; checking in without it (the dashboard stays safe without it)", JSON.stringify({ reason: `token_lock_${reason}`, ...describeError(error) }));
|
|
109
|
+
return { acquired: true, waited_ms: Date.now() - started, mode: "unlocked", release: async () => undefined };
|
|
110
|
+
}
|
|
111
|
+
async function readOwner(lockPath) {
|
|
112
|
+
try {
|
|
113
|
+
const raw = JSON.parse(await fs.readFile(lockPath, "utf8"));
|
|
114
|
+
return typeof raw["owner"] === "string" ? raw["owner"] : null;
|
|
115
|
+
}
|
|
116
|
+
catch {
|
|
117
|
+
// Gone or unreadable: not provably ours, so release leaves it alone.
|
|
118
|
+
return null;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
/** When the lock was taken (ms), `"gone"` if the file no longer exists. A half-written file is dated by its mtime. */
|
|
122
|
+
async function readHeldAt(lockPath) {
|
|
123
|
+
try {
|
|
124
|
+
const raw = JSON.parse(await fs.readFile(lockPath, "utf8"));
|
|
125
|
+
if (typeof raw["at_ms"] === "number" && Number.isFinite(raw["at_ms"]))
|
|
126
|
+
return raw["at_ms"];
|
|
127
|
+
}
|
|
128
|
+
catch (error) {
|
|
129
|
+
if (error?.code === "ENOENT")
|
|
130
|
+
return "gone";
|
|
131
|
+
}
|
|
132
|
+
try {
|
|
133
|
+
return (await fs.stat(lockPath)).mtimeMs;
|
|
134
|
+
}
|
|
135
|
+
catch {
|
|
136
|
+
return "gone";
|
|
137
|
+
}
|
|
138
|
+
}
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import { describeError } from "../health-detail.js";
|
|
2
|
+
import { writeJsonFileAtomic } from "../local-state-files.js";
|
|
2
3
|
import { getCollectorRuntimePaths, readLocalCollectorSessionFile, recordDeviceTokenExpiry, toSessionReference, } from "../local-state.js";
|
|
4
|
+
/** The request header that tells the dashboard this build stores a rotated token (BLI-17994). */
|
|
5
|
+
export const TOKEN_ROTATION_HEADER = "x-cockpit-token-rotation";
|
|
3
6
|
/**
|
|
4
7
|
* Read the session and decide whether this tick may check in.
|
|
5
8
|
*
|
|
@@ -33,7 +36,7 @@ export async function readHeartbeatSession(homeDir) {
|
|
|
33
36
|
console.error("[heartbeat] this machine's session looks expired; checking in anyway so the dashboard can renew it", JSON.stringify({
|
|
34
37
|
reason: "session_expired_locally",
|
|
35
38
|
expires_at: session.expires_at ?? null,
|
|
36
|
-
next_action: "
|
|
39
|
+
next_action: "a token inside its rotation window is swapped on this call; one past its date needs `cockpit doctor`",
|
|
37
40
|
}));
|
|
38
41
|
}
|
|
39
42
|
return { usable: true, session, state: state };
|
|
@@ -66,6 +69,43 @@ export async function recordHeartbeatTokenExpiry(homeDir, response) {
|
|
|
66
69
|
return null;
|
|
67
70
|
});
|
|
68
71
|
}
|
|
72
|
+
/**
|
|
73
|
+
* Store the PENDING device token the heartbeat door handed back (BLI-17994).
|
|
74
|
+
*
|
|
75
|
+
* `reply` is the parsed heartbeat body. No `pending_device_token` in it is the
|
|
76
|
+
* normal case (nothing was due, or a successor is already waiting) and says
|
|
77
|
+
* nothing. When there is one, it is written to `session.json` in ONE atomic
|
|
78
|
+
* rename (write a temp file, rename over the session), so no reader (the sync,
|
|
79
|
+
* the memory hooks, the MCP servers) ever sees a half-written file. The date is
|
|
80
|
+
* deliberately NOT touched: a pending token has not been used yet, so the
|
|
81
|
+
* server has not moved the date. The first request that carries it promotes it
|
|
82
|
+
* and retires the old token; the reply to that request carries the new date,
|
|
83
|
+
* which `recordHeartbeatTokenExpiry` stores.
|
|
84
|
+
*
|
|
85
|
+
* A reply that is malformed, or a write that fails, keeps the current token,
|
|
86
|
+
* which stays valid until its own date; the server offers another pending
|
|
87
|
+
* token after ten minutes. Never throws.
|
|
88
|
+
*/
|
|
89
|
+
export async function recordHeartbeatPendingToken(homeDir, reply) {
|
|
90
|
+
const incoming = reply?.["pending_device_token"];
|
|
91
|
+
if (incoming === undefined)
|
|
92
|
+
return { status: "none" };
|
|
93
|
+
if (typeof incoming !== "string" || !/^bli_dev_[A-Za-z0-9_-]{16,}$/.test(incoming)) {
|
|
94
|
+
console.error("[heartbeat] the dashboard sent a device token this build does not recognise; keeping the one it has", JSON.stringify({ reason: "pending_token_malformed" }));
|
|
95
|
+
return { status: "skipped", reason: "pending_token_malformed" };
|
|
96
|
+
}
|
|
97
|
+
const paths = getCollectorRuntimePaths(homeDir);
|
|
98
|
+
try {
|
|
99
|
+
const session = await readLocalCollectorSessionFile(paths);
|
|
100
|
+
await writeJsonFileAtomic(paths.session_file, { ...session, device_token: incoming });
|
|
101
|
+
console.error("[heartbeat] stored the pending device token; the first request that uses it makes it current", JSON.stringify({ reason: "pending_token_stored", expires_at: session.expires_at ?? null }));
|
|
102
|
+
return { status: "stored" };
|
|
103
|
+
}
|
|
104
|
+
catch (error) {
|
|
105
|
+
console.error("[heartbeat] the pending device token could not be stored; keeping the one this machine has, which stays valid until its date", JSON.stringify({ reason: "pending_token_write_failed", ...describeError(error) }));
|
|
106
|
+
return { status: "skipped", reason: "pending_token_write_failed" };
|
|
107
|
+
}
|
|
108
|
+
}
|
|
69
109
|
async function readTokenExpiry(response) {
|
|
70
110
|
try {
|
|
71
111
|
const body = (await response.json());
|
|
@@ -31,7 +31,8 @@ import { readStagingInventory } from "../disk-usage.js";
|
|
|
31
31
|
import { describeError } from "../health-detail.js";
|
|
32
32
|
import { shouldSuppressFleetReceipts, } from "../dev-build.js";
|
|
33
33
|
import { getCollectorRuntimePaths, LOCAL_COLLECTOR_VERSION, } from "../local-state.js";
|
|
34
|
-
import { noteHeartbeatRefusal, readHeartbeatSession, recordHeartbeatTokenExpiry, } from "./heartbeat-token.js";
|
|
34
|
+
import { noteHeartbeatRefusal, readHeartbeatSession, recordHeartbeatTokenExpiry, recordHeartbeatPendingToken, TOKEN_ROTATION_HEADER, } from "./heartbeat-token.js";
|
|
35
|
+
import { withTokenRotationLock } from "./heartbeat-token-lock.js";
|
|
35
36
|
import { classifySyncHealthError } from "../sync-health-class.js";
|
|
36
37
|
import { readMemoryReceiptFile } from "./memory-install-receipt.js";
|
|
37
38
|
import { readMemoryHookCounts } from "./memory-hook-counts.js";
|
|
@@ -220,6 +221,11 @@ export async function readHeartbeatMemoryReceipt(options) {
|
|
|
220
221
|
* `sync.err.log` alone when the dashboard says a device is quiet.
|
|
221
222
|
*/
|
|
222
223
|
export async function sendCollectorHeartbeatBestEffort(options) {
|
|
224
|
+
// BLI-17994. Single-flight: a launchd tick and a manual sync take turns, so
|
|
225
|
+
// the second reads `session.json` AFTER the first stored its pending token.
|
|
226
|
+
return withTokenRotationLock(options.homeDir, options.lock, false, () => sendCollectorHeartbeatLocked(options));
|
|
227
|
+
}
|
|
228
|
+
async function sendCollectorHeartbeatLocked(options) {
|
|
223
229
|
// BLI-3554. A checkout's heartbeat would move `last_seen_at` on a real fleet
|
|
224
230
|
// device row, so an agent's worktree could keep a genuinely dead machine
|
|
225
231
|
// looking alive to the BLI-3550 reader.
|
|
@@ -282,6 +288,9 @@ export async function sendCollectorHeartbeatBestEffort(options) {
|
|
|
282
288
|
headers: {
|
|
283
289
|
"Content-Type": "application/json",
|
|
284
290
|
Authorization: `Bearer ${session.device_token}`,
|
|
291
|
+
// BLI-17994. "I store a renewed token if you send one": the dashboard
|
|
292
|
+
// only rotates for a collector that says so.
|
|
293
|
+
[TOKEN_ROTATION_HEADER]: "1",
|
|
285
294
|
},
|
|
286
295
|
body: JSON.stringify(heartbeat),
|
|
287
296
|
signal: controller.signal,
|
|
@@ -293,24 +302,33 @@ export async function sendCollectorHeartbeatBestEffort(options) {
|
|
|
293
302
|
// that decides what a failure is — a 401/403 is `auth_failed` and
|
|
294
303
|
// anything else is not, read off the observed status and never off the
|
|
295
304
|
// words in a message (BLI-3551's rule).
|
|
305
|
+
// BLI-17994. The door names why it refused (`device_token_expired`,
|
|
306
|
+
// `device_token_replaced`) in a `reason` field; say it here rather than
|
|
307
|
+
// leaving a bare status.
|
|
308
|
+
const refusal = await Promise.resolve().then(() => response.clone().json()).catch(() => null);
|
|
296
309
|
console.error("[heartbeat] the dashboard refused this tick's heartbeat", JSON.stringify({
|
|
297
310
|
reason: "heartbeat_rejected",
|
|
298
311
|
health_class: classifySyncHealthError({ httpStatus: response.status }),
|
|
299
312
|
http_status: response.status,
|
|
300
313
|
root_count: heartbeat.roots.length,
|
|
314
|
+
...(typeof refusal?.["reason"] === "string" ? { server_reason: refusal["reason"] } : {}),
|
|
301
315
|
...(response.status === 401 || response.status === 403
|
|
302
316
|
? {
|
|
303
|
-
next_action: "
|
|
317
|
+
next_action: "this token expired, was replaced or was revoked: run `cockpit doctor` to pair this machine again",
|
|
304
318
|
}
|
|
305
319
|
: {}),
|
|
306
320
|
}));
|
|
307
321
|
return false;
|
|
308
322
|
}
|
|
309
|
-
//
|
|
310
|
-
//
|
|
311
|
-
//
|
|
323
|
+
// The reply carries `token_expires_at` (and a `pending_device_token` when
|
|
324
|
+
// the token was near its end); the sibling records both, so this machine's
|
|
325
|
+
// own copy follows the server rather than drifting.
|
|
312
326
|
const reply = await response.clone().json().catch(() => null);
|
|
313
327
|
noteDoctorRequest(reply?.device_action);
|
|
328
|
+
// BLI-17994. Store a pending successor if the door handed one out (the
|
|
329
|
+
// date does not move: a pending token has not been used), then the date the
|
|
330
|
+
// door reports, which moves only when this request promoted a token.
|
|
331
|
+
await recordHeartbeatPendingToken(options.homeDir, reply);
|
|
314
332
|
await recordHeartbeatTokenExpiry(options.homeDir, response);
|
|
315
333
|
console.error("[heartbeat] checked in", JSON.stringify({
|
|
316
334
|
reason: "heartbeat_recorded",
|
|
@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
24
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
24
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.219");
|
|
25
25
|
return 0;
|
|
26
26
|
}
|
|
27
27
|
|
|
@@ -28,14 +28,15 @@ const FIXES = {
|
|
|
28
28
|
* the same gap has two different next actions (BLI-4019).
|
|
29
29
|
*
|
|
30
30
|
* `device` says "run `cockpit login`" for every lapse, and for an expired
|
|
31
|
-
* session that
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
31
|
+
* session that can be the wrong instruction: a device token lives 30 days and
|
|
32
|
+
* the heartbeat swaps it for a fresh one when 7 days or fewer are left
|
|
33
|
+
* (BLI-17994), so a machine whose local copy merely lags heals on the next
|
|
34
|
+
* tick. Only a token that stays expired (the machine was offline past its date)
|
|
35
|
+
* needs `cockpit doctor`. Telling somebody to sign in again on a machine that
|
|
36
|
+
* is about to heal itself is how a receipt stops being believed.
|
|
36
37
|
*/
|
|
37
38
|
const REASON_FIXES = {
|
|
38
|
-
session_expired: "Renews on the next sync tick
|
|
39
|
+
session_expired: "Renews on the next sync tick; if it stays expired this machine was offline too long, so run `cockpit doctor` to pair it again.",
|
|
39
40
|
};
|
|
40
41
|
/** Codex hooks a person switched OFF is a decision, not a fault. */
|
|
41
42
|
const UNSUPPORTED_FIX = "Switched off in your own config; nothing to do.";
|
|
@@ -93,8 +93,8 @@ async function healAgentRules(command, paths, now) {
|
|
|
93
93
|
: { step: "agent_rules_install", status: "fail", error_code: "agent_rules_still_missing", error_detail: "before=missing; after=missing" };
|
|
94
94
|
}
|
|
95
95
|
/**
|
|
96
|
-
* Sign-in: the heartbeat (which ran just before this)
|
|
97
|
-
*
|
|
96
|
+
* Sign-in: the heartbeat (which ran just before this) swaps a token near its
|
|
97
|
+
* end for a fresh one without a browser (BLI-17994). If this machine is STILL
|
|
98
98
|
* signed out or the door refused the token (401/403), only a browser can fix
|
|
99
99
|
* it, so record the one action and say it once a day instead of every fifteen
|
|
100
100
|
* minutes. A lapsed local date with no refusal (offline, a checkout build)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bli-cockpit/cli",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.219",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
30
|
"@bli-cockpit/memory-mcp": "0.1.66",
|
|
31
|
-
"@bli-cockpit/mcp": "0.1.
|
|
31
|
+
"@bli-cockpit/mcp": "0.1.126",
|
|
32
32
|
"@bli-cockpit/telemetry-core": "0.1.75"
|
|
33
33
|
},
|
|
34
34
|
"repository": {
|