@bli-cockpit/cli 0.2.174 → 0.2.179

package/dist/sync-lock-holder.js

@@ -0,0 +1,82 @@
+import { SYNC_LOCK_STALE_TAKEOVER_MS, } from "./sync-lock.js";
+/**
+ * What the collection lock's holder is doing, in four answers (BLI-12872).
+ *
+ *   free      nobody holds it; whoever wants it just takes it
+ *   running   alive and, as far as the lock can tell, working: leave it alone
+ *   stale     its heartbeat stopped; the process is gone or wedged
+ *   stuck     alive but has finished no work for a long time
+ *
+ * `cockpit doctor` used to WAIT out a running sync (10 minutes, 40 progress
+ * lines, then a row telling the person to run doctor again). A running sync is
+ * the machine doing its job, so doctor now asks this module what it is looking
+ * at and acts: it reports `running`, repairs `stale` and `stuck`, and never
+ * waits. Pure, so every answer is testable off a lock record.
+ *
+ * What "making progress" means here is only what the lock file can say. The
+ * heartbeat proves the process is alive; it fires from a timer whether or not
+ * any work happens, so a hung upload keeps it fresh forever. The lock also
+ * carries when the holder started and when it last finished a unit of work (a
+ * worktree synced, an upload request answered), and that is the progress mark:
+ * a sync that keeps finishing work is never called stuck, however long it has
+ * been running.
+ */
+/**
+ * How long a scheduled sync may go without finishing any work before it is
+ * called stuck.
+ *
+ * Measured, not guessed: a tick on a machine with a backlog runs 10 to 15
+ * minutes in total (BLI-11112), and the 2026-10-07 reference-Mac tick held the
+ * lock for more than 10 (BLI-12872). A whole tick of that length is slow, not
+ * stuck, so the clock here is time since the last finished unit of work, not
+ * time since the start: a tick works through its worktrees and uploads one
+ * request at a time, and a holder that has finished none of either for a full
+ * 15 minutes is hung on something (a network call that never answers, a
+ * blocked read), not slow. Fifteen minutes is also the scheduler's own period,
+ * so a hung tick costs at most one missed cycle before doctor or the next
+ * person clears it. A holder that cannot say when it started (a lock written by
+ * an older CLI) and any `backfill` holder are never called stuck: a catch-up
+ * scans for minutes between uploads by design.
+ */
+export const SYNC_HOLDER_STUCK_MS = 15 * 60_000;
+export function classifySyncHolder(holder, nowMs) {
+    if (!holder)
+        return { kind: "free" };
+    const startedMs = holder.started_ms;
+    const lastWorkMs = startedMs === null ? null : Math.max(startedMs, holder.progress_ms ?? 0);
+    const facts = {
+        pid: holder.pid > 0 ? holder.pid : null,
+        role: holder.role,
+        token: holder.token,
+        heartbeat_age_ms: Math.max(0, nowMs - holder.heartbeat_ms),
+        running_ms: startedMs === null ? null : Math.max(0, nowMs - startedMs),
+        idle_ms: lastWorkMs === null ? null : Math.max(0, nowMs - lastWorkMs),
+    };
+    if (facts.heartbeat_age_ms > SYNC_LOCK_STALE_TAKEOVER_MS) {
+        return { kind: "stale", ...facts };
+    }
+    const stuck = holder.role === "sync" &&
+        facts.idle_ms !== null &&
+        facts.idle_ms > SYNC_HOLDER_STUCK_MS;
+    return { kind: stuck ? "stuck" : "running", ...facts };
+}
+/** "under a minute", "7m", "1h05m": the span a person reads. */
+export function formatSpan(ms) {
+    const minutes = Math.floor(ms / 60_000);
+    if (minutes < 1)
+        return "under a minute";
+    if (minutes < 60)
+        return `${minutes}m`;
+    return `${Math.floor(minutes / 60)}h${String(minutes % 60).padStart(2, "0")}m`;
+}
+/**
+ * The sentence for a healthy holder: "background sync is running (started 7m
+ * ago)". A catch-up holder says so instead, and a holder that cannot say when
+ * it started leaves the parenthesis off rather than inventing a time.
+ */
+export function runningWords(holder) {
+    const started = holder.running_ms === null ? "" : ` (started ${formatSpan(holder.running_ms)} ago)`;
+    return holder.role === "backfill"
+        ? `another catch-up run is in progress${started}`
+        : `background sync is running${started}`;
+}

package/dist/sync-lock.js

@@ -10,8 +10,14 @@ import { describeError } from "./health-detail.js";
  * A held lock refreshes a heartbeat every 30 s. Takeover happens only when the
  * heartbeat is more than 2 minutes stale — a fixed short timeout would fire
  * during a legitimate long first sync and recreate the interleaving it
- * prevents (D33). The lock file stays local and stores a pid + heartbeat only;
- * no paths or content.
+ * prevents (D33). The lock file stays local and stores a pid, a heartbeat and
+ * (BLI-12872) when the holder started and when it last finished a unit of
+ * work; no paths or content.
+ *
+ * The heartbeat only proves the process is alive. `started_ms` and
+ * `progress_ms` are what let `sync-lock-holder.ts` tell a slow tick from a
+ * stuck one: a holder that keeps finishing work is never called stuck, however
+ * long it has been running.
  */
 const LOCK_FILENAME = "sync.lock";
 const HEARTBEAT_INTERVAL_MS = 30_000;
@@ -25,32 +31,61 @@ const HEARTBEAT_INTERVAL_MS = 30_000;
  */
 export const SYNC_LOCK_STALE_TAKEOVER_MS = 2 * 60_000;
 const STALE_TAKEOVER_MS = SYNC_LOCK_STALE_TAKEOVER_MS;
-export async function acquireSyncLock(paths, now = new Date()) {
+/** The locks this process holds right now; `noteSyncProgress` stamps all of them. */
+const heldStates = new Set();
+/**
+ * "I just finished a unit of work": a worktree synced, an upload request
+ * answered. In memory only; the next 30 s heartbeat writes it down. A no-op
+ * when this process holds no lock, so callers never have to ask.
+ */
+export function noteSyncProgress(now = Date.now()) {
+    for (const held of heldStates) {
+        held.progressMs = now;
+        held.marks += 1;
+        held.longestGapMs = Math.max(held.longestGapMs, now - held.lastMarkMs);
+        held.lastMarkMs = now;
+    }
+}
+export async function acquireSyncLock(paths, now = new Date(), options = {}) {
     const lockPath = path.join(paths.state_dir, LOCK_FILENAME);
     await fs.mkdir(paths.state_dir, { recursive: true, mode: 0o700 });
     const token = crypto.randomUUID();
+    const held = {
+        startedMs: now.getTime(),
+        progressMs: null,
+        role: options.role ?? "sync",
+        marks: 0,
+        lastMarkMs: now.getTime(),
+        longestGapMs: 0,
+    };
     const existing = await readLock(lockPath);
     if (existing && now.getTime() - existing.heartbeat_ms <= STALE_TAKEOVER_MS) {
+        logHeldByLiveOwner(existing, now);
         return { acquired: false, held_since: existing.heartbeat_at };
     }
     // Free or stale: try an exclusive create first; if another process created it
     // in the gap, re-check staleness before taking over.
-    let owned = await tryExclusiveCreate(lockPath, token, now);
+    let takeover = existing ? "stale" : null;
+    let owned = await tryExclusiveCreate(lockPath, token, now, held);
     if (!owned) {
         const current = await readLock(lockPath);
         if (current && now.getTime() - current.heartbeat_ms <= STALE_TAKEOVER_MS) {
+            logHeldByLiveOwner(current, now);
             return { acquired: false, held_since: current.heartbeat_at };
         }
-        await writeLock(lockPath, token, now); // take over the stale lock
+        takeover = current ? "stale" : "unreadable";
+        await writeLock(lockPath, token, now, held); // take over the stale lock
         owned = true;
     }
+    logAcquired(held, takeover, existing, now);
+    heldStates.add(held);
     const timer = setInterval(() => {
         // Refresh only while the lock is still ours; if we were taken over, stop
         // touching the file so we cannot steal it back from the new owner.
         void (async () => {
             const current = await readLock(lockPath);
             if (current && current.token === token) {
-                await writeLock(lockPath, token, new Date()).catch(() => undefined);
+                await writeLock(lockPath, token, new Date(), held).catch(() => undefined);
             }
             else {
                 clearInterval(timer);
@@ -64,6 +99,8 @@ export async function acquireSyncLock(paths, now = new Date()) {
         handle: {
             release: async () => {
                 clearInterval(timer);
+                heldStates.delete(held);
+                logReleased(held);
                 // Ownership-safe: only remove the lock if it is still ours. A stale
                 // owner that resumes after takeover must not delete the new owner's lock.
                 const current = await readLock(lockPath);
@@ -74,10 +111,46 @@ export async function acquireSyncLock(paths, now = new Date()) {
         },
     };
 }
-async function tryExclusiveCreate(lockPath, token, now) {
+/**
+ * How long this holder kept the lock and how it spent it: the duration that
+ * makes a 10-minute tick measurable (BLI-12872). The longest gap between two
+ * finished units of work is what a slow tick and a stuck one differ by.
+ */
+function logReleased(held) {
+    const nowMs = Date.now();
+    console.error("[sync-lock] lock released", JSON.stringify({
+        reason: "holder_finished",
+        role: held.role,
+        held_ms: Math.max(0, nowMs - held.startedMs),
+        progress_marks: held.marks,
+        longest_gap_ms: Math.max(held.longestGapMs, nowMs - held.lastMarkMs),
+    }));
+}
+/** The ordinary contended case, named so a long hold is measurable (BLI-12872). */
+function logHeldByLiveOwner(owner, now) {
+    console.error("[sync-lock] lock is held by a live owner; standing aside", JSON.stringify({
+        reason: "owner_heartbeat_fresh",
+        owner_pid: owner.pid > 0 ? owner.pid : null,
+        owner_role: owner.role,
+        heartbeat_age_ms: Math.max(0, now.getTime() - owner.heartbeat_ms),
+        held_ms: owner.started_ms === null ? null : Math.max(0, now.getTime() - owner.started_ms),
+    }));
+}
+/** A takeover says why it happened; an ordinary free acquisition says so too. */
+function logAcquired(held, takeover, previous, now) {
+    console.error(takeover === null
+        ? "[sync-lock] lock acquired"
+        : "[sync-lock] took over a lock whose owner stopped answering", JSON.stringify({
+        role: held.role,
+        reason: takeover === "stale" ? "heartbeat_stale" : takeover === "unreadable" ? "lock_file_unreadable" : "lock_free",
+        previous_pid: previous && previous.pid > 0 ? previous.pid : null,
+        previous_heartbeat_age_ms: previous ? Math.max(0, now.getTime() - previous.heartbeat_ms) : null,
+    }));
+}
+async function tryExclusiveCreate(lockPath, token, now, held) {
     try {
         const handle = await fs.open(lockPath, "wx", 0o600);
-        await handle.writeFile(serializeLock(token, now));
+        await handle.writeFile(serializeLock(token, now, held));
         await handle.close();
         return true;
     }
@@ -102,15 +175,18 @@ function isLockHeldError(error) {
         error !== null &&
         error.code === "EEXIST");
 }
-async function writeLock(lockPath, token, now) {
-    await fs.writeFile(lockPath, serializeLock(token, now), { mode: 0o600 });
+async function writeLock(lockPath, token, now, held) {
+    await fs.writeFile(lockPath, serializeLock(token, now, held), { mode: 0o600 });
 }
-function serializeLock(token, now) {
+function serializeLock(token, now, held) {
     const record = {
         pid: process.pid,
         token,
         heartbeat_at: now.toISOString(),
         heartbeat_ms: now.getTime(),
+        started_ms: held.startedMs,
+        progress_ms: held.progressMs,
+        role: held.role,
     };
     return `${JSON.stringify(record)}\n`;
 }
@@ -132,6 +208,11 @@ async function readLock(lockPath) {
                 ? heartbeatAt
                 : new Date(heartbeatMs).toISOString(),
             heartbeat_ms: heartbeatMs,
+            started_ms: finiteOrNull(record["started_ms"]),
+            progress_ms: finiteOrNull(record["progress_ms"]),
+            role: record["role"] === "sync" || record["role"] === "backfill"
+                ? record["role"]
+                : null,
         };
     }
     catch {
@@ -143,8 +224,25 @@ async function readLock(lockPath) {
         return null;
     }
 }
+function finiteOrNull(value) {
+    return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
 /** Read owner metadata without acquiring or disturbing the collection lock. */
-export async function inspectSyncLock(paths) {
+export async function inspectSyncLock(paths, now = Date.now()) {
     const record = await readLock(path.join(paths.state_dir, LOCK_FILENAME));
-    return record ? { pid: record.pid, heartbeat_at: record.heartbeat_at, held: Date.now() - record.heartbeat_ms <= STALE_TAKEOVER_MS } : null;
+    if (!record)
+        return null;
+    return { ...record, held: now - record.heartbeat_ms <= STALE_TAKEOVER_MS };
+}
+/**
+ * Remove the lock file only if it still carries `token`: the holder this caller
+ * inspected and ended. A newer owner's lock is never touched. Returns whether
+ * a file was removed.
+ */
+export async function removeSyncLockIfOwnedBy(paths, token) {
+    const lockPath = path.join(paths.state_dir, LOCK_FILENAME);
+    const current = await readLock(lockPath);
+    if (!current || current.token !== token)
+        return false;
+    return fs.rm(lockPath, { force: true }).then(() => true, () => false);
 }

package/dist/sync-tick-timing.js

@@ -0,0 +1,34 @@
+export function startTickTimer(now = () => Date.now()) {
+    const started = now();
+    const stages = {};
+    const wrap = (name) => async (run) => {
+        const stageStarted = now();
+        try {
+            return await run();
+        }
+        finally {
+            stages[name] = (stages[name] ?? 0) + (now() - stageStarted);
+        }
+    };
+    const report = (outcome, exitCode, errorName) => {
+        const longest = Object.entries(stages).sort((a, b) => b[1] - a[1])[0];
+        console.error("[sync] tick finished", JSON.stringify({
+            outcome,
+            exit_code: exitCode,
+            duration_ms: now() - started,
+            stages,
+            longest_stage: longest?.[0] ?? null,
+            ...(errorName ? { error_name: errorName } : {}),
+        }));
+    };
+    return {
+        selfUpdate: wrap("self_update"),
+        collect: wrap("collect"),
+        heartbeat: wrap("heartbeat"),
+        outcomeReport: wrap("outcome_report"),
+        converge: wrap("converge"),
+        prune: wrap("prune"),
+        finish: (exitCode) => report(exitCode === 0 ? "ok" : "failed", exitCode, null),
+        threw: (error) => report("threw", null, error instanceof Error ? error.name : "unknown"),
+    };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bli-cockpit/cli",
-  "version": "0.2.174",
+  "version": "0.2.179",
   "private": false,
   "type": "module",
   "bin": {
@@ -27,9 +27,9 @@
     "test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-verb-help.mjs && node ../../scripts/assert-public-cli-surface-doors.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-exit-contract.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-cli-doctor-json.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
   },
   "dependencies": {
-    "@bli-cockpit/memory-mcp": "0.1.48",
-    "@bli-cockpit/mcp": "0.1.90",
-    "@bli-cockpit/telemetry-core": "0.1.60"
+    "@bli-cockpit/memory-mcp": "0.1.52",
+    "@bli-cockpit/mcp": "0.1.94",
+    "@bli-cockpit/telemetry-core": "0.1.64"
   },
   "repository": {
     "type": "git",

package/dist/commands/doctor-lock-wait.js

@@ -1,55 +0,0 @@
-import { inspectBackfillLock } from "../backfill-lock.js";
-import { getCollectorRuntimePaths } from "../local-state.js";
-import { inspectSyncLock } from "../sync-lock.js";
-import { doctorLockWaitSeconds } from "./doctor-budget.js";
-const BUSY = new Set(["sync_already_running", "sync_lock_held", "backfill_already_running", "live_sync_paused_during_backfill"]);
-/** Retry the operation itself: its existing exclusive lock acquisition owns the
- * handoff. Merely observing a free lock never grants permission to collect. */
-export async function withDoctorLockWait(context, run, timing = { now: () => Date.now(), sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)) }) {
-    const started = timing.now();
-    const limit = doctorLockWaitSeconds(context) * 1000;
-    let nextProgress = 0;
-    let row = await run();
-    while (BUSY.has(row.code)) {
-        const paths = getCollectorRuntimePaths(context.command.homeDir);
-        const sync = await inspectSyncLock(paths);
-        const backfill = await inspectBackfillLock(paths);
-        const owner = backfill.held
-            ? { pid: backfill.pid, heartbeat_at: backfill.held_since }
-            : sync;
-        const elapsed = timing.now() - started;
-        if (elapsed >= limit) {
-            const pid = owner?.pid;
-            let name = "unknown process";
-            if (pid && pid > 0 && context.io.exec) {
-                const result = process.platform === "win32"
-                    ? await context.io.exec("tasklist", ["/FI", `PID eq ${pid}`, "/FO", "CSV", "/NH"])
-                    : await context.io.exec("ps", ["-p", String(pid), "-o", "comm="]);
-                if (result.code === 0 && result.stdout.trim())
-                    name = result.stdout.trim().split("\n")[0] ?? name;
-            }
-            // BLI-11101: the scheduled sync holding the lock is not a fault; it is
-            // doing the upload this row would prove. Say so by name.
-            const syncOwns = !backfill.held && Boolean(sync?.held);
-            const code = syncOwns ? "sync_running" : "lock_wait_timeout";
-            console.error(`[doctor] ${row.id} lock wait ended`, JSON.stringify({ reason: code, owner_pid: pid ?? null, waited_seconds: Math.round(elapsed / 1000), limit_seconds: Math.round(limit / 1000) }));
-            const message = syncOwns
-                ? `The background sync (pid ${pid ?? "unknown"}, ${name}) is running and holds the collection lock; waited ${Math.round(elapsed / 1000)}s. Run \`cockpit doctor\` after it finishes, or \`cockpit doctor --lock-wait 600\` to wait for it.`
-                : `Waited ${Math.round(elapsed / 1000)}s for the collection lock; owner pid ${pid ?? "unknown"}, ${name}. Run \`cockpit doctor --lock-wait 600\`.`;
-            return { ...row, status: "needs_fix", code, nextAction: "cockpit doctor --lock-wait 600", message };
-        }
-        if (elapsed >= nextProgress) {
-            const seconds = Math.floor(elapsed / 1000);
-            console.error(`[doctor] ${row.id} waiting`, JSON.stringify({ reason: row.code, owner_pid: owner?.pid ?? null, heartbeat_at: owner?.heartbeat_at ?? null, elapsed_seconds: seconds }));
-            context.io.stderr.write(`waiting for the background sync to finish, ${Math.floor(seconds / 60)}m${seconds % 60}s\n`);
-            nextProgress = elapsed + 30_000;
-        }
-        await timing.sleep(Math.min(1000, limit - elapsed));
-        // A live heartbeat means there is no point spawning a collector yet.
-        const currentSync = await inspectSyncLock(paths);
-        const currentBackfill = await inspectBackfillLock(paths);
-        if (!currentSync?.held && !currentBackfill.held)
-            row = await run();
-    }
-    return row;
-}