@bli-cockpit/cli 0.2.173 → 0.2.179
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/backfill-lock-wait.js +4 -1
- package/dist/commands/doctor-backfill.js +160 -0
- package/dist/commands/doctor-budget.js +14 -17
- package/dist/commands/doctor-collection-lock.js +201 -0
- package/dist/commands/doctor-disk-words.js +34 -3
- package/dist/commands/doctor-pipeline-verdicts.js +13 -3
- package/dist/commands/doctor-pipeline.js +22 -149
- package/dist/commands/doctor-progress.js +54 -0
- package/dist/commands/doctor-report.js +2 -1
- package/dist/commands/doctor.js +11 -3
- package/dist/commands/local-args-tower-logs.js +30 -11
- package/dist/commands/local-help-commands-tower-ops.js +11 -3
- package/dist/commands/local-help-commands.js +1 -1
- package/dist/commands/logs-stored.js +56 -0
- package/dist/commands/logs.js +18 -2
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/sync-run.js +3 -0
- package/dist/commands/sync-yield.js +4 -0
- package/dist/commands/sync.js +30 -11
- package/dist/evidence-upload-transport.js +4 -0
- package/dist/state-dir-audit.js +32 -1
- package/dist/sync-lock-holder.js +82 -0
- package/dist/sync-lock.js +111 -13
- package/dist/sync-tick-timing.js +34 -0
- package/package.json +4 -4
- package/dist/commands/doctor-lock-wait.js +0 -55
package/dist/commands/sync.js
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
* check in with the heartbeat door sync-heartbeat.ts
|
|
13
13
|
* say how the tick went reportTickOutcome / reportTickThrew
|
|
14
14
|
* converge the rest of the machine sync-followups.ts, sync-heal.ts
|
|
15
|
+
* time every step above ../sync-tick-timing.ts (BLI-12872)
|
|
15
16
|
*
|
|
16
17
|
* The supporting modules each own one thing those steps share:
|
|
17
18
|
*
|
|
@@ -45,8 +46,26 @@ import { runSyncWithHealthReceipt } from "./sync-receipt.js";
|
|
|
45
46
|
import { runAutostartSelfHealAfterSync, runMemoryInstallAfterSync, runScheduledSelfUpdateAfterSync, runStagingPruneAfterSync, } from "./sync-followups.js";
|
|
46
47
|
import { rotateCollectorLogsBestEffort } from "../log-rotation.js";
|
|
47
48
|
import { stampSyncTick } from "../sync-tick-stamp.js";
|
|
49
|
+
import { startTickTimer } from "../sync-tick-timing.js";
|
|
48
50
|
import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, readLocalCollectorConfig, } from "../local-state.js";
|
|
51
|
+
/**
|
|
52
|
+
* The tick, timed (BLI-12872): ends with one `[sync] tick finished` line that
|
|
53
|
+
* carries its duration and the time spent in each step, win or throw. The
|
|
54
|
+
* steps themselves are `runSyncTick`, below.
|
|
55
|
+
*/
|
|
49
56
|
export async function runSync(command, io) {
|
|
57
|
+
const tick = startTickTimer();
|
|
58
|
+
try {
|
|
59
|
+
const exitCode = await runSyncTick(command, io, tick);
|
|
60
|
+
tick.finish(exitCode);
|
|
61
|
+
return exitCode;
|
|
62
|
+
}
|
|
63
|
+
catch (error) {
|
|
64
|
+
tick.threw(error);
|
|
65
|
+
throw error;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
async function runSyncTick(command, io, tick) {
|
|
50
69
|
const paths = getCollectorRuntimePaths(command.homeDir);
|
|
51
70
|
// Ticket 12831: the memory prompt hook reads this stamp to tell a scheduler
|
|
52
71
|
// that stopped firing from one that is merely quiet.
|
|
@@ -58,7 +77,7 @@ export async function runSync(command, io) {
|
|
|
58
77
|
await rotateCollectorLogsBestEffort(paths);
|
|
59
78
|
const dashboardUrl = await tickDashboardUrl(command, paths);
|
|
60
79
|
const minCliVersionAtStart = await reportTickStarted(command, io, dashboardUrl);
|
|
61
|
-
const updated = await runScheduledSelfUpdateAfterSync(command, io, dashboardUrl, minCliVersionAtStart);
|
|
80
|
+
const updated = await tick.selfUpdate(() => runScheduledSelfUpdateAfterSync(command, io, dashboardUrl, minCliVersionAtStart));
|
|
62
81
|
if (updated && io.exec) {
|
|
63
82
|
const args = ["sync"];
|
|
64
83
|
if (command.homeDir)
|
|
@@ -78,16 +97,16 @@ export async function runSync(command, io) {
|
|
|
78
97
|
return result.code;
|
|
79
98
|
}
|
|
80
99
|
try {
|
|
81
|
-
const result = await runSyncWithHealthReceipt(command, io);
|
|
100
|
+
const result = await tick.collect(() => runSyncWithHealthReceipt(command, io));
|
|
82
101
|
// BLI-3551: every tick checks in, including one that collected nothing.
|
|
83
102
|
// This is the only writer of `last_seen_at` that does not need an envelope,
|
|
84
103
|
// so it is what separates a quiet machine from a dead one.
|
|
85
|
-
await sendSyncHeartbeat(command, io, dashboardUrl, result.heartbeat);
|
|
86
|
-
await reportTickOutcome(command, io, dashboardUrl, result.completion);
|
|
87
|
-
await convergeAfterTick(command, io, dashboardUrl);
|
|
104
|
+
await tick.heartbeat(() => sendSyncHeartbeat(command, io, dashboardUrl, result.heartbeat));
|
|
105
|
+
await tick.outcomeReport(() => reportTickOutcome(command, io, dashboardUrl, result.completion));
|
|
106
|
+
await tick.converge(() => convergeAfterTick(command, io, dashboardUrl));
|
|
88
107
|
// BLI-3619 / BLI-11106: every tick, the staged copies of whatever the
|
|
89
108
|
// server just committed go; a follow-up never blocks or fails collection.
|
|
90
|
-
await runStagingPruneAfterSync(command, io, dashboardUrl);
|
|
109
|
+
await tick.prune(() => runStagingPruneAfterSync(command, io, dashboardUrl));
|
|
91
110
|
return result.exitCode;
|
|
92
111
|
}
|
|
93
112
|
catch (error) {
|
|
@@ -95,13 +114,13 @@ export async function runSync(command, io) {
|
|
|
95
114
|
// A machine whose sync THREW is still alive, and that is worth knowing —
|
|
96
115
|
// a device that stops checking in entirely is a different problem from one
|
|
97
116
|
// checking in with a failure every fifteen minutes.
|
|
98
|
-
await sendSyncHeartbeat(command, io, dashboardUrl, {
|
|
117
|
+
await tick.heartbeat(() => sendSyncHeartbeat(command, io, dashboardUrl, {
|
|
99
118
|
status: "fail",
|
|
100
119
|
reason: errorCode,
|
|
101
|
-
});
|
|
102
|
-
await reportTickThrew(command, io, dashboardUrl, error, errorCode);
|
|
103
|
-
await convergeAfterTick(command, io, dashboardUrl);
|
|
104
|
-
await runStagingPruneAfterSync(command, io, dashboardUrl);
|
|
120
|
+
}));
|
|
121
|
+
await tick.outcomeReport(() => reportTickThrew(command, io, dashboardUrl, error, errorCode));
|
|
122
|
+
await tick.converge(() => convergeAfterTick(command, io, dashboardUrl));
|
|
123
|
+
await tick.prune(() => runStagingPruneAfterSync(command, io, dashboardUrl));
|
|
105
124
|
throw error;
|
|
106
125
|
}
|
|
107
126
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import crypto from "node:crypto";
|
|
2
2
|
import { describeError } from "./health-detail.js";
|
|
3
|
+
import { noteSyncProgress } from "./sync-lock.js";
|
|
3
4
|
/**
|
|
4
5
|
* The `send` stage: one retrying POST to an evidence-upload route, plus the
|
|
5
6
|
* response-body plumbing every later stage reads a reason off of.
|
|
@@ -27,6 +28,9 @@ export async function requestJson(options, routePath, body) {
|
|
|
27
28
|
lastStatus = response.status;
|
|
28
29
|
lastBody = await readResponseJson(response);
|
|
29
30
|
if (response.ok || (response.status < 500 && response.status !== 429)) {
|
|
31
|
+
// BLI-12872: an answered request is a finished unit of work; the lock
|
|
32
|
+
// holder's progress mark is what keeps a slow upload from reading as stuck.
|
|
33
|
+
noteSyncProgress();
|
|
30
34
|
return { ok: response.ok, status: response.status, body: lastBody };
|
|
31
35
|
}
|
|
32
36
|
}
|
package/dist/state-dir-audit.js
CHANGED
|
@@ -16,7 +16,10 @@
|
|
|
16
16
|
* recognised by name, left exactly where they are, and not counted. Anything
|
|
17
17
|
* the table below does not recognise is counted as `other`, so a new
|
|
18
18
|
* collector file that nobody registered still shows up in the footprint
|
|
19
|
-
* rather than hiding.
|
|
19
|
+
* rather than hiding. Since BLI-12872 the biggest of them are also NAMED, in
|
|
20
|
+
* the doctor row and the log: an unrecognised entry has no bound Tower can
|
|
21
|
+
* promise (Tower did not write it), so the rule is that it is never deleted
|
|
22
|
+
* and never anonymous; whoever finds it registers it here with a bound.
|
|
20
23
|
*
|
|
21
24
|
* Every collector entry carries the one-line reason it is bounded; the
|
|
22
25
|
* runbook (`docs/runbooks/cockpit-launchd-sync.md`, "What the state
|
|
@@ -47,6 +50,11 @@ export const STATE_DIR_RULES = [
|
|
|
47
50
|
{ name: "memory-install-receipt.json", match: "exact", class: "state", bound: "one file, overwritten" },
|
|
48
51
|
{ name: "staging-gate.json", match: "exact", class: "state", bound: "one file, overwritten every tick: the staging gate's state and what was last announced (BLI-12832)" },
|
|
49
52
|
{ name: "last-prune.json", match: "exact", class: "state", bound: "one file, overwritten every tick" },
|
|
53
|
+
// BLI-12872: the two writers the audit had never been told about. Both are
|
|
54
|
+
// tiny by construction; naming them keeps the unrecognised bucket meaning
|
|
55
|
+
// "something nobody registered", not "something small we forgot".
|
|
56
|
+
{ name: "session-needs-login.json", match: "exact", class: "state", bound: "one small note, overwritten at most daily and removed when the sign-in is valid again (sync-heal.ts)" },
|
|
57
|
+
{ name: "autostart-uninstalled", match: "exact", class: "state", bound: "a zero-byte marker written by `cockpit autostart uninstall` and removed by install or doctor" },
|
|
50
58
|
{ name: "last-turn-trace.json", match: "exact", class: "state", bound: "one file, overwritten" },
|
|
51
59
|
{ name: "sync.lock", match: "exact", class: "state", bound: "lock file, removed when the tick ends" },
|
|
52
60
|
{ name: "backfill.lock", match: "exact", class: "state", bound: "lock file, removed when the backfill ends" },
|
|
@@ -98,6 +106,8 @@ export function classifyStateEntry(name) {
|
|
|
98
106
|
export function studiesStateDir(paths) {
|
|
99
107
|
return `${paths.state_dir.replace(/[\\/]+$/u, "")}${STUDIES_DIR_SUFFIX}`;
|
|
100
108
|
}
|
|
109
|
+
/** How many unrecognised entries doctor and the log name. */
|
|
110
|
+
export const UNRECOGNISED_NAMED_MAX = 5;
|
|
101
111
|
/**
|
|
102
112
|
* Sizes the `state` and `other` entries of the state directory. Staging,
|
|
103
113
|
* logs and spool are measured by their own readers and skipped here; studies
|
|
@@ -109,8 +119,10 @@ export async function readStateDirFootprint(paths) {
|
|
|
109
119
|
state_bytes: 0,
|
|
110
120
|
other_bytes: 0,
|
|
111
121
|
other_entries: 0,
|
|
122
|
+
other_largest: [],
|
|
112
123
|
study_entries: 0,
|
|
113
124
|
};
|
|
125
|
+
const others = [];
|
|
114
126
|
const entries = await fs
|
|
115
127
|
.readdir(paths.state_dir, { withFileTypes: true })
|
|
116
128
|
.catch(() => []);
|
|
@@ -133,7 +145,26 @@ export async function readStateDirFootprint(paths) {
|
|
|
133
145
|
else {
|
|
134
146
|
footprint.other_bytes += bytes;
|
|
135
147
|
footprint.other_entries += 1;
|
|
148
|
+
others.push({ name: boundedEntryName(entry.name), bytes });
|
|
136
149
|
}
|
|
137
150
|
}
|
|
151
|
+
footprint.other_largest = others
|
|
152
|
+
.sort((a, b) => b.bytes - a.bytes || a.name.localeCompare(b.name))
|
|
153
|
+
.slice(0, UNRECOGNISED_NAMED_MAX);
|
|
154
|
+
if (footprint.other_entries > 0) {
|
|
155
|
+
// Said every time it is true, never only when red: the reference Mac held
|
|
156
|
+
// 108 MB here and nothing said what it was.
|
|
157
|
+
console.error("[state-dir-audit] entries in the state dir that nothing registered; counted, never deleted", JSON.stringify({
|
|
158
|
+
reason: "unrecognised_state_entries",
|
|
159
|
+
entries: footprint.other_entries,
|
|
160
|
+
bytes: footprint.other_bytes,
|
|
161
|
+
largest: footprint.other_largest,
|
|
162
|
+
}));
|
|
163
|
+
}
|
|
138
164
|
return footprint;
|
|
165
|
+
}
|
|
166
|
+
/** A name, not a path: control characters out, 64 characters at most. */
|
|
167
|
+
function boundedEntryName(name) {
|
|
168
|
+
const clean = name.replace(/[\u0000-\u001f\u007f]/gu, "?");
|
|
169
|
+
return clean.length > 64 ? `${clean.slice(0, 61)}...` : clean;
|
|
139
170
|
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { SYNC_LOCK_STALE_TAKEOVER_MS, } from "./sync-lock.js";
|
|
2
|
+
/**
|
|
3
|
+
* What the collection lock's holder is doing, in four answers (BLI-12872).
|
|
4
|
+
*
|
|
5
|
+
* free nobody holds it; whoever wants it just takes it
|
|
6
|
+
* running alive and, as far as the lock can tell, working: leave it alone
|
|
7
|
+
* stale its heartbeat stopped; the process is gone or wedged
|
|
8
|
+
* stuck alive but has finished no work for a long time
|
|
9
|
+
*
|
|
10
|
+
* `cockpit doctor` used to WAIT out a running sync (10 minutes, 40 progress
|
|
11
|
+
* lines, then a row telling the person to run doctor again). A running sync is
|
|
12
|
+
* the machine doing its job, so doctor now asks this module what it is looking
|
|
13
|
+
* at and acts: it reports `running`, repairs `stale` and `stuck`, and never
|
|
14
|
+
* waits. Pure, so every answer is testable off a lock record.
|
|
15
|
+
*
|
|
16
|
+
* What "making progress" means here is only what the lock file can say. The
|
|
17
|
+
* heartbeat proves the process is alive; it fires from a timer whether or not
|
|
18
|
+
* any work happens, so a hung upload keeps it fresh forever. The lock also
|
|
19
|
+
* carries when the holder started and when it last finished a unit of work (a
|
|
20
|
+
* worktree synced, an upload request answered), and that is the progress mark:
|
|
21
|
+
* a sync that keeps finishing work is never called stuck, however long it has
|
|
22
|
+
* been running.
|
|
23
|
+
*/
|
|
24
|
+
/**
|
|
25
|
+
* How long a scheduled sync may go without finishing any work before it is
|
|
26
|
+
* called stuck.
|
|
27
|
+
*
|
|
28
|
+
* Measured, not guessed: a tick on a machine with a backlog runs 10 to 15
|
|
29
|
+
* minutes in total (BLI-11112), and the 2026-10-07 reference-Mac tick held the
|
|
30
|
+
* lock for more than 10 (BLI-12872). A whole tick of that length is slow, not
|
|
31
|
+
* stuck, so the clock here is time since the last finished unit of work, not
|
|
32
|
+
* time since the start: a tick works through its worktrees and uploads one
|
|
33
|
+
* request at a time, and a holder that has finished none of either for a full
|
|
34
|
+
* 15 minutes is hung on something (a network call that never answers, a
|
|
35
|
+
* blocked read), not slow. Fifteen minutes is also the scheduler's own period,
|
|
36
|
+
* so a hung tick costs at most one missed cycle before doctor or the next
|
|
37
|
+
* person clears it. A holder that cannot say when it started (a lock written by
|
|
38
|
+
* an older CLI) and any `backfill` holder are never called stuck: a catch-up
|
|
39
|
+
* scans for minutes between uploads by design.
|
|
40
|
+
*/
|
|
41
|
+
export const SYNC_HOLDER_STUCK_MS = 15 * 60_000;
|
|
42
|
+
export function classifySyncHolder(holder, nowMs) {
|
|
43
|
+
if (!holder)
|
|
44
|
+
return { kind: "free" };
|
|
45
|
+
const startedMs = holder.started_ms;
|
|
46
|
+
const lastWorkMs = startedMs === null ? null : Math.max(startedMs, holder.progress_ms ?? 0);
|
|
47
|
+
const facts = {
|
|
48
|
+
pid: holder.pid > 0 ? holder.pid : null,
|
|
49
|
+
role: holder.role,
|
|
50
|
+
token: holder.token,
|
|
51
|
+
heartbeat_age_ms: Math.max(0, nowMs - holder.heartbeat_ms),
|
|
52
|
+
running_ms: startedMs === null ? null : Math.max(0, nowMs - startedMs),
|
|
53
|
+
idle_ms: lastWorkMs === null ? null : Math.max(0, nowMs - lastWorkMs),
|
|
54
|
+
};
|
|
55
|
+
if (facts.heartbeat_age_ms > SYNC_LOCK_STALE_TAKEOVER_MS) {
|
|
56
|
+
return { kind: "stale", ...facts };
|
|
57
|
+
}
|
|
58
|
+
const stuck = holder.role === "sync" &&
|
|
59
|
+
facts.idle_ms !== null &&
|
|
60
|
+
facts.idle_ms > SYNC_HOLDER_STUCK_MS;
|
|
61
|
+
return { kind: stuck ? "stuck" : "running", ...facts };
|
|
62
|
+
}
|
|
63
|
+
/** "under a minute", "7m", "1h05m": the span a person reads. */
|
|
64
|
+
export function formatSpan(ms) {
|
|
65
|
+
const minutes = Math.floor(ms / 60_000);
|
|
66
|
+
if (minutes < 1)
|
|
67
|
+
return "under a minute";
|
|
68
|
+
if (minutes < 60)
|
|
69
|
+
return `${minutes}m`;
|
|
70
|
+
return `${Math.floor(minutes / 60)}h${String(minutes % 60).padStart(2, "0")}m`;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* The sentence for a healthy holder: "background sync is running (started 7m
|
|
74
|
+
* ago)". A catch-up holder says so instead, and a holder that cannot say when
|
|
75
|
+
* it started leaves the parenthesis off rather than inventing a time.
|
|
76
|
+
*/
|
|
77
|
+
export function runningWords(holder) {
|
|
78
|
+
const started = holder.running_ms === null ? "" : ` (started ${formatSpan(holder.running_ms)} ago)`;
|
|
79
|
+
return holder.role === "backfill"
|
|
80
|
+
? `another catch-up run is in progress${started}`
|
|
81
|
+
: `background sync is running${started}`;
|
|
82
|
+
}
|
package/dist/sync-lock.js
CHANGED
|
@@ -10,8 +10,14 @@ import { describeError } from "./health-detail.js";
|
|
|
10
10
|
* A held lock refreshes a heartbeat every 30 s. Takeover happens only when the
|
|
11
11
|
* heartbeat is more than 2 minutes stale — a fixed short timeout would fire
|
|
12
12
|
* during a legitimate long first sync and recreate the interleaving it
|
|
13
|
-
* prevents (D33). The lock file stays local and stores a pid
|
|
14
|
-
*
|
|
13
|
+
* prevents (D33). The lock file stays local and stores a pid, a heartbeat and
|
|
14
|
+
* (BLI-12872) when the holder started and when it last finished a unit of
|
|
15
|
+
* work; no paths or content.
|
|
16
|
+
*
|
|
17
|
+
* The heartbeat only proves the process is alive. `started_ms` and
|
|
18
|
+
* `progress_ms` are what let `sync-lock-holder.ts` tell a slow tick from a
|
|
19
|
+
* stuck one: a holder that keeps finishing work is never called stuck, however
|
|
20
|
+
* long it has been running.
|
|
15
21
|
*/
|
|
16
22
|
const LOCK_FILENAME = "sync.lock";
|
|
17
23
|
const HEARTBEAT_INTERVAL_MS = 30_000;
|
|
@@ -25,32 +31,61 @@ const HEARTBEAT_INTERVAL_MS = 30_000;
|
|
|
25
31
|
*/
|
|
26
32
|
export const SYNC_LOCK_STALE_TAKEOVER_MS = 2 * 60_000;
|
|
27
33
|
const STALE_TAKEOVER_MS = SYNC_LOCK_STALE_TAKEOVER_MS;
|
|
28
|
-
|
|
34
|
+
/** The locks this process holds right now; `noteSyncProgress` stamps all of them. */
|
|
35
|
+
const heldStates = new Set();
|
|
36
|
+
/**
|
|
37
|
+
* "I just finished a unit of work": a worktree synced, an upload request
|
|
38
|
+
* answered. In memory only; the next 30 s heartbeat writes it down. A no-op
|
|
39
|
+
* when this process holds no lock, so callers never have to ask.
|
|
40
|
+
*/
|
|
41
|
+
export function noteSyncProgress(now = Date.now()) {
|
|
42
|
+
for (const held of heldStates) {
|
|
43
|
+
held.progressMs = now;
|
|
44
|
+
held.marks += 1;
|
|
45
|
+
held.longestGapMs = Math.max(held.longestGapMs, now - held.lastMarkMs);
|
|
46
|
+
held.lastMarkMs = now;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
export async function acquireSyncLock(paths, now = new Date(), options = {}) {
|
|
29
50
|
const lockPath = path.join(paths.state_dir, LOCK_FILENAME);
|
|
30
51
|
await fs.mkdir(paths.state_dir, { recursive: true, mode: 0o700 });
|
|
31
52
|
const token = crypto.randomUUID();
|
|
53
|
+
const held = {
|
|
54
|
+
startedMs: now.getTime(),
|
|
55
|
+
progressMs: null,
|
|
56
|
+
role: options.role ?? "sync",
|
|
57
|
+
marks: 0,
|
|
58
|
+
lastMarkMs: now.getTime(),
|
|
59
|
+
longestGapMs: 0,
|
|
60
|
+
};
|
|
32
61
|
const existing = await readLock(lockPath);
|
|
33
62
|
if (existing && now.getTime() - existing.heartbeat_ms <= STALE_TAKEOVER_MS) {
|
|
63
|
+
logHeldByLiveOwner(existing, now);
|
|
34
64
|
return { acquired: false, held_since: existing.heartbeat_at };
|
|
35
65
|
}
|
|
36
66
|
// Free or stale: try an exclusive create first; if another process created it
|
|
37
67
|
// in the gap, re-check staleness before taking over.
|
|
38
|
-
let
|
|
68
|
+
let takeover = existing ? "stale" : null;
|
|
69
|
+
let owned = await tryExclusiveCreate(lockPath, token, now, held);
|
|
39
70
|
if (!owned) {
|
|
40
71
|
const current = await readLock(lockPath);
|
|
41
72
|
if (current && now.getTime() - current.heartbeat_ms <= STALE_TAKEOVER_MS) {
|
|
73
|
+
logHeldByLiveOwner(current, now);
|
|
42
74
|
return { acquired: false, held_since: current.heartbeat_at };
|
|
43
75
|
}
|
|
44
|
-
|
|
76
|
+
takeover = current ? "stale" : "unreadable";
|
|
77
|
+
await writeLock(lockPath, token, now, held); // take over the stale lock
|
|
45
78
|
owned = true;
|
|
46
79
|
}
|
|
80
|
+
logAcquired(held, takeover, existing, now);
|
|
81
|
+
heldStates.add(held);
|
|
47
82
|
const timer = setInterval(() => {
|
|
48
83
|
// Refresh only while the lock is still ours; if we were taken over, stop
|
|
49
84
|
// touching the file so we cannot steal it back from the new owner.
|
|
50
85
|
void (async () => {
|
|
51
86
|
const current = await readLock(lockPath);
|
|
52
87
|
if (current && current.token === token) {
|
|
53
|
-
await writeLock(lockPath, token, new Date()).catch(() => undefined);
|
|
88
|
+
await writeLock(lockPath, token, new Date(), held).catch(() => undefined);
|
|
54
89
|
}
|
|
55
90
|
else {
|
|
56
91
|
clearInterval(timer);
|
|
@@ -64,6 +99,8 @@ export async function acquireSyncLock(paths, now = new Date()) {
|
|
|
64
99
|
handle: {
|
|
65
100
|
release: async () => {
|
|
66
101
|
clearInterval(timer);
|
|
102
|
+
heldStates.delete(held);
|
|
103
|
+
logReleased(held);
|
|
67
104
|
// Ownership-safe: only remove the lock if it is still ours. A stale
|
|
68
105
|
// owner that resumes after takeover must not delete the new owner's lock.
|
|
69
106
|
const current = await readLock(lockPath);
|
|
@@ -74,10 +111,46 @@ export async function acquireSyncLock(paths, now = new Date()) {
|
|
|
74
111
|
},
|
|
75
112
|
};
|
|
76
113
|
}
|
|
77
|
-
|
|
114
|
+
/**
|
|
115
|
+
* How long this holder kept the lock and how it spent it: the duration that
|
|
116
|
+
* makes a 10-minute tick measurable (BLI-12872). The longest gap between two
|
|
117
|
+
* finished units of work is what a slow tick and a stuck one differ by.
|
|
118
|
+
*/
|
|
119
|
+
function logReleased(held) {
|
|
120
|
+
const nowMs = Date.now();
|
|
121
|
+
console.error("[sync-lock] lock released", JSON.stringify({
|
|
122
|
+
reason: "holder_finished",
|
|
123
|
+
role: held.role,
|
|
124
|
+
held_ms: Math.max(0, nowMs - held.startedMs),
|
|
125
|
+
progress_marks: held.marks,
|
|
126
|
+
longest_gap_ms: Math.max(held.longestGapMs, nowMs - held.lastMarkMs),
|
|
127
|
+
}));
|
|
128
|
+
}
|
|
129
|
+
/** The ordinary contended case, named so a long hold is measurable (BLI-12872). */
|
|
130
|
+
function logHeldByLiveOwner(owner, now) {
|
|
131
|
+
console.error("[sync-lock] lock is held by a live owner; standing aside", JSON.stringify({
|
|
132
|
+
reason: "owner_heartbeat_fresh",
|
|
133
|
+
owner_pid: owner.pid > 0 ? owner.pid : null,
|
|
134
|
+
owner_role: owner.role,
|
|
135
|
+
heartbeat_age_ms: Math.max(0, now.getTime() - owner.heartbeat_ms),
|
|
136
|
+
held_ms: owner.started_ms === null ? null : Math.max(0, now.getTime() - owner.started_ms),
|
|
137
|
+
}));
|
|
138
|
+
}
|
|
139
|
+
/** A takeover says why it happened; an ordinary free acquisition says so too. */
|
|
140
|
+
function logAcquired(held, takeover, previous, now) {
|
|
141
|
+
console.error(takeover === null
|
|
142
|
+
? "[sync-lock] lock acquired"
|
|
143
|
+
: "[sync-lock] took over a lock whose owner stopped answering", JSON.stringify({
|
|
144
|
+
role: held.role,
|
|
145
|
+
reason: takeover === "stale" ? "heartbeat_stale" : takeover === "unreadable" ? "lock_file_unreadable" : "lock_free",
|
|
146
|
+
previous_pid: previous && previous.pid > 0 ? previous.pid : null,
|
|
147
|
+
previous_heartbeat_age_ms: previous ? Math.max(0, now.getTime() - previous.heartbeat_ms) : null,
|
|
148
|
+
}));
|
|
149
|
+
}
|
|
150
|
+
async function tryExclusiveCreate(lockPath, token, now, held) {
|
|
78
151
|
try {
|
|
79
152
|
const handle = await fs.open(lockPath, "wx", 0o600);
|
|
80
|
-
await handle.writeFile(serializeLock(token, now));
|
|
153
|
+
await handle.writeFile(serializeLock(token, now, held));
|
|
81
154
|
await handle.close();
|
|
82
155
|
return true;
|
|
83
156
|
}
|
|
@@ -102,15 +175,18 @@ function isLockHeldError(error) {
|
|
|
102
175
|
error !== null &&
|
|
103
176
|
error.code === "EEXIST");
|
|
104
177
|
}
|
|
105
|
-
async function writeLock(lockPath, token, now) {
|
|
106
|
-
await fs.writeFile(lockPath, serializeLock(token, now), { mode: 0o600 });
|
|
178
|
+
async function writeLock(lockPath, token, now, held) {
|
|
179
|
+
await fs.writeFile(lockPath, serializeLock(token, now, held), { mode: 0o600 });
|
|
107
180
|
}
|
|
108
|
-
function serializeLock(token, now) {
|
|
181
|
+
function serializeLock(token, now, held) {
|
|
109
182
|
const record = {
|
|
110
183
|
pid: process.pid,
|
|
111
184
|
token,
|
|
112
185
|
heartbeat_at: now.toISOString(),
|
|
113
186
|
heartbeat_ms: now.getTime(),
|
|
187
|
+
started_ms: held.startedMs,
|
|
188
|
+
progress_ms: held.progressMs,
|
|
189
|
+
role: held.role,
|
|
114
190
|
};
|
|
115
191
|
return `${JSON.stringify(record)}\n`;
|
|
116
192
|
}
|
|
@@ -132,6 +208,11 @@ async function readLock(lockPath) {
|
|
|
132
208
|
? heartbeatAt
|
|
133
209
|
: new Date(heartbeatMs).toISOString(),
|
|
134
210
|
heartbeat_ms: heartbeatMs,
|
|
211
|
+
started_ms: finiteOrNull(record["started_ms"]),
|
|
212
|
+
progress_ms: finiteOrNull(record["progress_ms"]),
|
|
213
|
+
role: record["role"] === "sync" || record["role"] === "backfill"
|
|
214
|
+
? record["role"]
|
|
215
|
+
: null,
|
|
135
216
|
};
|
|
136
217
|
}
|
|
137
218
|
catch {
|
|
@@ -143,8 +224,25 @@ async function readLock(lockPath) {
|
|
|
143
224
|
return null;
|
|
144
225
|
}
|
|
145
226
|
}
|
|
227
|
+
function finiteOrNull(value) {
|
|
228
|
+
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
229
|
+
}
|
|
146
230
|
/** Read owner metadata without acquiring or disturbing the collection lock. */
|
|
147
|
-
export async function inspectSyncLock(paths) {
|
|
231
|
+
export async function inspectSyncLock(paths, now = Date.now()) {
|
|
148
232
|
const record = await readLock(path.join(paths.state_dir, LOCK_FILENAME));
|
|
149
|
-
|
|
233
|
+
if (!record)
|
|
234
|
+
return null;
|
|
235
|
+
return { ...record, held: now - record.heartbeat_ms <= STALE_TAKEOVER_MS };
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* Remove the lock file only if it still carries `token`: the holder this caller
|
|
239
|
+
* inspected and ended. A newer owner's lock is never touched. Returns whether
|
|
240
|
+
* a file was removed.
|
|
241
|
+
*/
|
|
242
|
+
export async function removeSyncLockIfOwnedBy(paths, token) {
|
|
243
|
+
const lockPath = path.join(paths.state_dir, LOCK_FILENAME);
|
|
244
|
+
const current = await readLock(lockPath);
|
|
245
|
+
if (!current || current.token !== token)
|
|
246
|
+
return false;
|
|
247
|
+
return fs.rm(lockPath, { force: true }).then(() => true, () => false);
|
|
150
248
|
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
export function startTickTimer(now = () => Date.now()) {
|
|
2
|
+
const started = now();
|
|
3
|
+
const stages = {};
|
|
4
|
+
const wrap = (name) => async (run) => {
|
|
5
|
+
const stageStarted = now();
|
|
6
|
+
try {
|
|
7
|
+
return await run();
|
|
8
|
+
}
|
|
9
|
+
finally {
|
|
10
|
+
stages[name] = (stages[name] ?? 0) + (now() - stageStarted);
|
|
11
|
+
}
|
|
12
|
+
};
|
|
13
|
+
const report = (outcome, exitCode, errorName) => {
|
|
14
|
+
const longest = Object.entries(stages).sort((a, b) => b[1] - a[1])[0];
|
|
15
|
+
console.error("[sync] tick finished", JSON.stringify({
|
|
16
|
+
outcome,
|
|
17
|
+
exit_code: exitCode,
|
|
18
|
+
duration_ms: now() - started,
|
|
19
|
+
stages,
|
|
20
|
+
longest_stage: longest?.[0] ?? null,
|
|
21
|
+
...(errorName ? { error_name: errorName } : {}),
|
|
22
|
+
}));
|
|
23
|
+
};
|
|
24
|
+
return {
|
|
25
|
+
selfUpdate: wrap("self_update"),
|
|
26
|
+
collect: wrap("collect"),
|
|
27
|
+
heartbeat: wrap("heartbeat"),
|
|
28
|
+
outcomeReport: wrap("outcome_report"),
|
|
29
|
+
converge: wrap("converge"),
|
|
30
|
+
prune: wrap("prune"),
|
|
31
|
+
finish: (exitCode) => report(exitCode === 0 ? "ok" : "failed", exitCode, null),
|
|
32
|
+
threw: (error) => report("threw", null, error instanceof Error ? error.name : "unknown"),
|
|
33
|
+
};
|
|
34
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bli-cockpit/cli",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.179",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -27,9 +27,9 @@
|
|
|
27
27
|
"test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-verb-help.mjs && node ../../scripts/assert-public-cli-surface-doors.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-exit-contract.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-cli-doctor-json.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
|
-
"@bli-cockpit/memory-mcp": "0.1.
|
|
31
|
-
"@bli-cockpit/mcp": "0.1.
|
|
32
|
-
"@bli-cockpit/telemetry-core": "0.1.
|
|
30
|
+
"@bli-cockpit/memory-mcp": "0.1.52",
|
|
31
|
+
"@bli-cockpit/mcp": "0.1.94",
|
|
32
|
+
"@bli-cockpit/telemetry-core": "0.1.64"
|
|
33
33
|
},
|
|
34
34
|
"repository": {
|
|
35
35
|
"type": "git",
|
|
@@ -1,55 +0,0 @@
|
|
|
1
|
-
import { inspectBackfillLock } from "../backfill-lock.js";
|
|
2
|
-
import { getCollectorRuntimePaths } from "../local-state.js";
|
|
3
|
-
import { inspectSyncLock } from "../sync-lock.js";
|
|
4
|
-
import { doctorLockWaitSeconds } from "./doctor-budget.js";
|
|
5
|
-
const BUSY = new Set(["sync_already_running", "sync_lock_held", "backfill_already_running", "live_sync_paused_during_backfill"]);
|
|
6
|
-
/** Retry the operation itself: its existing exclusive lock acquisition owns the
|
|
7
|
-
* handoff. Merely observing a free lock never grants permission to collect. */
|
|
8
|
-
export async function withDoctorLockWait(context, run, timing = { now: () => Date.now(), sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)) }) {
|
|
9
|
-
const started = timing.now();
|
|
10
|
-
const limit = doctorLockWaitSeconds(context) * 1000;
|
|
11
|
-
let nextProgress = 0;
|
|
12
|
-
let row = await run();
|
|
13
|
-
while (BUSY.has(row.code)) {
|
|
14
|
-
const paths = getCollectorRuntimePaths(context.command.homeDir);
|
|
15
|
-
const sync = await inspectSyncLock(paths);
|
|
16
|
-
const backfill = await inspectBackfillLock(paths);
|
|
17
|
-
const owner = backfill.held
|
|
18
|
-
? { pid: backfill.pid, heartbeat_at: backfill.held_since }
|
|
19
|
-
: sync;
|
|
20
|
-
const elapsed = timing.now() - started;
|
|
21
|
-
if (elapsed >= limit) {
|
|
22
|
-
const pid = owner?.pid;
|
|
23
|
-
let name = "unknown process";
|
|
24
|
-
if (pid && pid > 0 && context.io.exec) {
|
|
25
|
-
const result = process.platform === "win32"
|
|
26
|
-
? await context.io.exec("tasklist", ["/FI", `PID eq ${pid}`, "/FO", "CSV", "/NH"])
|
|
27
|
-
: await context.io.exec("ps", ["-p", String(pid), "-o", "comm="]);
|
|
28
|
-
if (result.code === 0 && result.stdout.trim())
|
|
29
|
-
name = result.stdout.trim().split("\n")[0] ?? name;
|
|
30
|
-
}
|
|
31
|
-
// BLI-11101: the scheduled sync holding the lock is not a fault; it is
|
|
32
|
-
// doing the upload this row would prove. Say so by name.
|
|
33
|
-
const syncOwns = !backfill.held && Boolean(sync?.held);
|
|
34
|
-
const code = syncOwns ? "sync_running" : "lock_wait_timeout";
|
|
35
|
-
console.error(`[doctor] ${row.id} lock wait ended`, JSON.stringify({ reason: code, owner_pid: pid ?? null, waited_seconds: Math.round(elapsed / 1000), limit_seconds: Math.round(limit / 1000) }));
|
|
36
|
-
const message = syncOwns
|
|
37
|
-
? `The background sync (pid ${pid ?? "unknown"}, ${name}) is running and holds the collection lock; waited ${Math.round(elapsed / 1000)}s. Run \`cockpit doctor\` after it finishes, or \`cockpit doctor --lock-wait 600\` to wait for it.`
|
|
38
|
-
: `Waited ${Math.round(elapsed / 1000)}s for the collection lock; owner pid ${pid ?? "unknown"}, ${name}. Run \`cockpit doctor --lock-wait 600\`.`;
|
|
39
|
-
return { ...row, status: "needs_fix", code, nextAction: "cockpit doctor --lock-wait 600", message };
|
|
40
|
-
}
|
|
41
|
-
if (elapsed >= nextProgress) {
|
|
42
|
-
const seconds = Math.floor(elapsed / 1000);
|
|
43
|
-
console.error(`[doctor] ${row.id} waiting`, JSON.stringify({ reason: row.code, owner_pid: owner?.pid ?? null, heartbeat_at: owner?.heartbeat_at ?? null, elapsed_seconds: seconds }));
|
|
44
|
-
context.io.stderr.write(`waiting for the background sync to finish, ${Math.floor(seconds / 60)}m${seconds % 60}s\n`);
|
|
45
|
-
nextProgress = elapsed + 30_000;
|
|
46
|
-
}
|
|
47
|
-
await timing.sleep(Math.min(1000, limit - elapsed));
|
|
48
|
-
// A live heartbeat means there is no point spawning a collector yet.
|
|
49
|
-
const currentSync = await inspectSyncLock(paths);
|
|
50
|
-
const currentBackfill = await inspectBackfillLock(paths);
|
|
51
|
-
if (!currentSync?.held && !currentBackfill.held)
|
|
52
|
-
row = await run();
|
|
53
|
-
}
|
|
54
|
-
return row;
|
|
55
|
-
}
|