@bli-cockpit/cli 0.2.174 → 0.2.180

package/dist/commands/sync.js

@@ -12,6 +12,7 @@
  *   check in with the heartbeat door  sync-heartbeat.ts
  *   say how the tick went             reportTickOutcome / reportTickThrew
  *   converge the rest of the machine  sync-followups.ts, sync-heal.ts
+ *   time every step above             ../sync-tick-timing.ts (BLI-12872)
  *
  * The supporting modules each own one thing those steps share:
  *
@@ -45,8 +46,26 @@ import { runSyncWithHealthReceipt } from "./sync-receipt.js";
 import { runAutostartSelfHealAfterSync, runMemoryInstallAfterSync, runScheduledSelfUpdateAfterSync, runStagingPruneAfterSync, } from "./sync-followups.js";
 import { rotateCollectorLogsBestEffort } from "../log-rotation.js";
 import { stampSyncTick } from "../sync-tick-stamp.js";
+import { startTickTimer } from "../sync-tick-timing.js";
 import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, readLocalCollectorConfig, } from "../local-state.js";
+/**
+ * The tick, timed (BLI-12872): ends with one `[sync] tick finished` line that
+ * carries its duration and the time spent in each step, win or throw. The
+ * steps themselves are `runSyncTick`, below.
+ */
 export async function runSync(command, io) {
+    const tick = startTickTimer();
+    try {
+        const exitCode = await runSyncTick(command, io, tick);
+        tick.finish(exitCode);
+        return exitCode;
+    }
+    catch (error) {
+        tick.threw(error);
+        throw error;
+    }
+}
+async function runSyncTick(command, io, tick) {
     const paths = getCollectorRuntimePaths(command.homeDir);
     // Ticket 12831: the memory prompt hook reads this stamp to tell a scheduler
     // that stopped firing from one that is merely quiet.
@@ -58,7 +77,7 @@ export async function runSync(command, io) {
     await rotateCollectorLogsBestEffort(paths);
     const dashboardUrl = await tickDashboardUrl(command, paths);
     const minCliVersionAtStart = await reportTickStarted(command, io, dashboardUrl);
-    const updated = await runScheduledSelfUpdateAfterSync(command, io, dashboardUrl, minCliVersionAtStart);
+    const updated = await tick.selfUpdate(() => runScheduledSelfUpdateAfterSync(command, io, dashboardUrl, minCliVersionAtStart));
     if (updated && io.exec) {
         const args = ["sync"];
         if (command.homeDir)
@@ -78,16 +97,16 @@ export async function runSync(command, io) {
         return result.code;
     }
     try {
-        const result = await runSyncWithHealthReceipt(command, io);
+        const result = await tick.collect(() => runSyncWithHealthReceipt(command, io));
         // BLI-3551: every tick checks in, including one that collected nothing.
         // This is the only writer of `last_seen_at` that does not need an envelope,
         // so it is what separates a quiet machine from a dead one.
-        await sendSyncHeartbeat(command, io, dashboardUrl, result.heartbeat);
-        await reportTickOutcome(command, io, dashboardUrl, result.completion);
-        await convergeAfterTick(command, io, dashboardUrl);
+        await tick.heartbeat(() => sendSyncHeartbeat(command, io, dashboardUrl, result.heartbeat));
+        await tick.outcomeReport(() => reportTickOutcome(command, io, dashboardUrl, result.completion));
+        await tick.converge(() => convergeAfterTick(command, io, dashboardUrl));
         // BLI-3619 / BLI-11106: every tick, the staged copies of whatever the
         // server just committed go; a follow-up never blocks or fails collection.
-        await runStagingPruneAfterSync(command, io, dashboardUrl);
+        await tick.prune(() => runStagingPruneAfterSync(command, io, dashboardUrl));
         return result.exitCode;
     }
     catch (error) {
@@ -95,13 +114,13 @@ export async function runSync(command, io) {
         // A machine whose sync THREW is still alive, and that is worth knowing —
         // a device that stops checking in entirely is a different problem from one
         // checking in with a failure every fifteen minutes.
-        await sendSyncHeartbeat(command, io, dashboardUrl, {
+        await tick.heartbeat(() => sendSyncHeartbeat(command, io, dashboardUrl, {
             status: "fail",
             reason: errorCode,
-        });
-        await reportTickThrew(command, io, dashboardUrl, error, errorCode);
-        await convergeAfterTick(command, io, dashboardUrl);
-        await runStagingPruneAfterSync(command, io, dashboardUrl);
+        }));
+        await tick.outcomeReport(() => reportTickThrew(command, io, dashboardUrl, error, errorCode));
+        await tick.converge(() => convergeAfterTick(command, io, dashboardUrl));
+        await tick.prune(() => runStagingPruneAfterSync(command, io, dashboardUrl));
         throw error;
     }
 }

package/dist/evidence-upload-transport.js

@@ -1,5 +1,6 @@
 import crypto from "node:crypto";
 import { describeError } from "./health-detail.js";
+import { noteSyncProgress } from "./sync-lock.js";
 /**
  * The `send` stage: one retrying POST to an evidence-upload route, plus the
  * response-body plumbing every later stage reads a reason off of.
@@ -27,6 +28,9 @@ export async function requestJson(options, routePath, body) {
             lastStatus = response.status;
             lastBody = await readResponseJson(response);
             if (response.ok || (response.status < 500 && response.status !== 429)) {
+                // BLI-12872: an answered request is a finished unit of work; the lock
+                // holder's progress mark is what keeps a slow upload from reading as stuck.
+                noteSyncProgress();
                 return { ok: response.ok, status: response.status, body: lastBody };
             }
         }

package/dist/local-state-status.js

@@ -53,6 +53,7 @@ export async function inspectLocalCollectorStatus(options = {}) {
         oldest_pending_health_receipt_at: healthOutbox.oldest_created_at,
         last_health_receipt_failure_reason: healthOutbox.last_failure_reason,
         stuck_evidence_object_count: stuckEvidence.stuck_object_count,
+        stuck_evidence_ghost_record_count: stuckEvidence.ghost_record_count,
         stuck_evidence_held_count: stuckEvidence.held_object_count,
         stuck_evidence_max_attempts: stuckEvidence.max_attempts,
         stuck_evidence_oldest_failure_at: stuckEvidence.oldest_first_failed_at,
@@ -148,7 +149,7 @@ function describeCollectorStatus(reading) {
         details.push(`Last upload failure: ${uploadSpool.last_upload_failure_reason}.`);
     }
     if (uploadSpool.pending_upload_count > 0) {
-        details.push(`Upload retry pending: ${uploadSpool.pending_upload_count} safe metadata record(s) spooled. Run \`${uploadSpool.retry_command ?? "cockpit sync"}\` to retry.`);
+        details.push(`Upload retry pending: ${uploadSpool.pending_upload_count} safe metadata record(s) are saved on this machine and go out on the next background sync.`);
     }
     if (healthOutbox.pending_count > 0) {
         details.push(`Collector health retry pending: ${healthOutbox.pending_count} sanitized receipt(s) queued since ${healthOutbox.oldest_created_at ?? "unknown"}.`);
@@ -168,6 +169,9 @@ function describeCollectorStatus(reading) {
     if (stuckEvidence.stuck_object_count > 0) {
         details.push(`Raw evidence stuck: ${stuckEvidence.stuck_object_count} object(s) have never been accepted (${stuckEvidence.held_object_count} waiting on backoff, worst ${stuckEvidence.max_attempts} attempt(s) since ${stuckEvidence.oldest_first_failed_at ?? "unknown"}) — reasons: ${stuckEvidence.reasons.join(", ") || "unknown"}.`);
     }
+    if (stuckEvidence.ghost_record_count > 0) {
+        details.push(`${stuckEvidence.ghost_record_count} old delivery record(s) have nothing left to send; no action needed.`);
+    }
     return details;
 }
 /**

package/dist/raw-evidence-stuck-summary.js

@@ -20,7 +20,26 @@ import { deliveryRetryAtMs, terminalDeliveryReason } from "./raw-evidence-stagin
  * read green while an object has been failing for nine days.
  */
 export function summarizeStuckEvidence(state, now) {
-    const entries = Object.values(state.delivery_attempts).filter(entry => !terminalDeliveryReason(entry.last_reason));
+    const entries = [];
+    let ghostCount = 0;
+    for (const [hash, entry] of Object.entries(state.delivery_attempts)) {
+        if (terminalDeliveryReason(entry.last_reason))
+            continue;
+        // A row whose object left staging (pruned after commit, or reconciled by a
+        // committed 409) has nothing to offer; counting it as stuck was BLI-12910.
+        if (!state.staged[hash]) {
+            ghostCount += 1;
+            continue;
+        }
+        entries.push(entry);
+    }
+    if (ghostCount > 0) {
+        console.error("[raw-evidence-stuck] delivery records with no staged object left out of the stuck count", JSON.stringify({
+            ghost_record_count: ghostCount,
+            stuck_object_count: entries.length,
+            reason: "object_no_longer_staged",
+        }));
+    }
     const reasons = new Set();
     let heldCount = 0;
     let maxAttempts = 0;
@@ -52,6 +71,7 @@ export function summarizeStuckEvidence(state, now) {
     return {
         stuck_object_count: entries.length,
         held_object_count: heldCount,
+        ghost_record_count: ghostCount,
         max_attempts: maxAttempts,
         oldest_first_failed_at: oldest,
         reasons: [...reasons].sort(),

package/dist/state-dir-audit.js

@@ -16,7 +16,10 @@
  * recognised by name, left exactly where they are, and not counted. Anything
  * the table below does not recognise is counted as `other`, so a new
  * collector file that nobody registered still shows up in the footprint
- * rather than hiding.
+ * rather than hiding. Since BLI-12872 the biggest of them are also NAMED, in
+ * the doctor row and the log: an unrecognised entry has no bound Tower can
+ * promise (Tower did not write it), so the rule is that it is never deleted
+ * and never anonymous; whoever finds it registers it here with a bound.
  *
  * Every collector entry carries the one-line reason it is bounded; the
  * runbook (`docs/runbooks/cockpit-launchd-sync.md`, "What the state
@@ -47,6 +50,11 @@ export const STATE_DIR_RULES = [
     { name: "memory-install-receipt.json", match: "exact", class: "state", bound: "one file, overwritten" },
     { name: "staging-gate.json", match: "exact", class: "state", bound: "one file, overwritten every tick: the staging gate's state and what was last announced (BLI-12832)" },
     { name: "last-prune.json", match: "exact", class: "state", bound: "one file, overwritten every tick" },
+    // BLI-12872: the two writers the audit had never been told about. Both are
+    // tiny by construction; naming them keeps the unrecognised bucket meaning
+    // "something nobody registered", not "something small we forgot".
+    { name: "session-needs-login.json", match: "exact", class: "state", bound: "one small note, overwritten at most daily and removed when the sign-in is valid again (sync-heal.ts)" },
+    { name: "autostart-uninstalled", match: "exact", class: "state", bound: "a zero-byte marker written by `cockpit autostart uninstall` and removed by install or doctor" },
     { name: "last-turn-trace.json", match: "exact", class: "state", bound: "one file, overwritten" },
     { name: "sync.lock", match: "exact", class: "state", bound: "lock file, removed when the tick ends" },
     { name: "backfill.lock", match: "exact", class: "state", bound: "lock file, removed when the backfill ends" },
@@ -98,6 +106,8 @@ export function classifyStateEntry(name) {
 export function studiesStateDir(paths) {
     return `${paths.state_dir.replace(/[\\/]+$/u, "")}${STUDIES_DIR_SUFFIX}`;
 }
+/** How many unrecognised entries doctor and the log name. */
+export const UNRECOGNISED_NAMED_MAX = 5;
 /**
  * Sizes the `state` and `other` entries of the state directory. Staging,
  * logs and spool are measured by their own readers and skipped here; studies
@@ -109,8 +119,10 @@ export async function readStateDirFootprint(paths) {
         state_bytes: 0,
         other_bytes: 0,
         other_entries: 0,
+        other_largest: [],
         study_entries: 0,
     };
+    const others = [];
     const entries = await fs
         .readdir(paths.state_dir, { withFileTypes: true })
         .catch(() => []);
@@ -133,7 +145,26 @@ export async function readStateDirFootprint(paths) {
         else {
             footprint.other_bytes += bytes;
             footprint.other_entries += 1;
+            others.push({ name: boundedEntryName(entry.name), bytes });
         }
     }
+    footprint.other_largest = others
+        .sort((a, b) => b.bytes - a.bytes || a.name.localeCompare(b.name))
+        .slice(0, UNRECOGNISED_NAMED_MAX);
+    if (footprint.other_entries > 0) {
+        // Said every time it is true, never only when red: the reference Mac held
+        // 108 MB here and nothing said what it was.
+        console.error("[state-dir-audit] entries in the state dir that nothing registered; counted, never deleted", JSON.stringify({
+            reason: "unrecognised_state_entries",
+            entries: footprint.other_entries,
+            bytes: footprint.other_bytes,
+            largest: footprint.other_largest,
+        }));
+    }
     return footprint;
+}
+/** A name, not a path: control characters out, 64 characters at most. */
+function boundedEntryName(name) {
+    const clean = name.replace(/[\u0000-\u001f\u007f]/gu, "?");
+    return clean.length > 64 ? `${clean.slice(0, 61)}...` : clean;
 }

package/dist/sync-lock-holder.js

@@ -0,0 +1,82 @@
+import { SYNC_LOCK_STALE_TAKEOVER_MS, } from "./sync-lock.js";
+/**
+ * What the collection lock's holder is doing, in four answers (BLI-12872).
+ *
+ *   free      nobody holds it; whoever wants it just takes it
+ *   running   alive and, as far as the lock can tell, working: leave it alone
+ *   stale     its heartbeat stopped; the process is gone or wedged
+ *   stuck     alive but has finished no work for a long time
+ *
+ * `cockpit doctor` used to WAIT out a running sync (10 minutes, 40 progress
+ * lines, then a row telling the person to run doctor again). A running sync is
+ * the machine doing its job, so doctor now asks this module what it is looking
+ * at and acts: it reports `running`, repairs `stale` and `stuck`, and never
+ * waits. Pure, so every answer is testable off a lock record.
+ *
+ * What "making progress" means here is only what the lock file can say. The
+ * heartbeat proves the process is alive; it fires from a timer whether or not
+ * any work happens, so a hung upload keeps it fresh forever. The lock also
+ * carries when the holder started and when it last finished a unit of work (a
+ * worktree synced, an upload request answered), and that is the progress mark:
+ * a sync that keeps finishing work is never called stuck, however long it has
+ * been running.
+ */
+/**
+ * How long a scheduled sync may go without finishing any work before it is
+ * called stuck.
+ *
+ * Measured, not guessed: a tick on a machine with a backlog runs 10 to 15
+ * minutes in total (BLI-11112), and the 2026-10-07 reference-Mac tick held the
+ * lock for more than 10 (BLI-12872). A whole tick of that length is slow, not
+ * stuck, so the clock here is time since the last finished unit of work, not
+ * time since the start: a tick works through its worktrees and uploads one
+ * request at a time, and a holder that has finished none of either for a full
+ * 15 minutes is hung on something (a network call that never answers, a
+ * blocked read), not slow. Fifteen minutes is also the scheduler's own period,
+ * so a hung tick costs at most one missed cycle before doctor or the next
+ * person clears it. A holder that cannot say when it started (a lock written by
+ * an older CLI) and any `backfill` holder are never called stuck: a catch-up
+ * scans for minutes between uploads by design.
+ */
+export const SYNC_HOLDER_STUCK_MS = 15 * 60_000;
+export function classifySyncHolder(holder, nowMs) {
+    if (!holder)
+        return { kind: "free" };
+    const startedMs = holder.started_ms;
+    const lastWorkMs = startedMs === null ? null : Math.max(startedMs, holder.progress_ms ?? 0);
+    const facts = {
+        pid: holder.pid > 0 ? holder.pid : null,
+        role: holder.role,
+        token: holder.token,
+        heartbeat_age_ms: Math.max(0, nowMs - holder.heartbeat_ms),
+        running_ms: startedMs === null ? null : Math.max(0, nowMs - startedMs),
+        idle_ms: lastWorkMs === null ? null : Math.max(0, nowMs - lastWorkMs),
+    };
+    if (facts.heartbeat_age_ms > SYNC_LOCK_STALE_TAKEOVER_MS) {
+        return { kind: "stale", ...facts };
+    }
+    const stuck = holder.role === "sync" &&
+        facts.idle_ms !== null &&
+        facts.idle_ms > SYNC_HOLDER_STUCK_MS;
+    return { kind: stuck ? "stuck" : "running", ...facts };
+}
+/** "under a minute", "7m", "1h05m": the span a person reads. */
+export function formatSpan(ms) {
+    const minutes = Math.floor(ms / 60_000);
+    if (minutes < 1)
+        return "under a minute";
+    if (minutes < 60)
+        return `${minutes}m`;
+    return `${Math.floor(minutes / 60)}h${String(minutes % 60).padStart(2, "0")}m`;
+}
+/**
+ * The sentence for a healthy holder: "background sync is running (started 7m
+ * ago)". A catch-up holder says so instead, and a holder that cannot say when
+ * it started leaves the parenthesis off rather than inventing a time.
+ */
+export function runningWords(holder) {
+    const started = holder.running_ms === null ? "" : ` (started ${formatSpan(holder.running_ms)} ago)`;
+    return holder.role === "backfill"
+        ? `another catch-up run is in progress${started}`
+        : `background sync is running${started}`;
+}

package/dist/sync-lock.js

@@ -10,8 +10,14 @@ import { describeError } from "./health-detail.js";
  * A held lock refreshes a heartbeat every 30 s. Takeover happens only when the
  * heartbeat is more than 2 minutes stale — a fixed short timeout would fire
  * during a legitimate long first sync and recreate the interleaving it
- * prevents (D33). The lock file stays local and stores a pid + heartbeat only;
- * no paths or content.
+ * prevents (D33). The lock file stays local and stores a pid, a heartbeat and
+ * (BLI-12872) when the holder started and when it last finished a unit of
+ * work; no paths or content.
+ *
+ * The heartbeat only proves the process is alive. `started_ms` and
+ * `progress_ms` are what let `sync-lock-holder.ts` tell a slow tick from a
+ * stuck one: a holder that keeps finishing work is never called stuck, however
+ * long it has been running.
  */
 const LOCK_FILENAME = "sync.lock";
 const HEARTBEAT_INTERVAL_MS = 30_000;
@@ -25,32 +31,61 @@ const HEARTBEAT_INTERVAL_MS = 30_000;
  */
 export const SYNC_LOCK_STALE_TAKEOVER_MS = 2 * 60_000;
 const STALE_TAKEOVER_MS = SYNC_LOCK_STALE_TAKEOVER_MS;
-export async function acquireSyncLock(paths, now = new Date()) {
+/** The locks this process holds right now; `noteSyncProgress` stamps all of them. */
+const heldStates = new Set();
+/**
+ * "I just finished a unit of work": a worktree synced, an upload request
+ * answered. In memory only; the next 30 s heartbeat writes it down. A no-op
+ * when this process holds no lock, so callers never have to ask.
+ */
+export function noteSyncProgress(now = Date.now()) {
+    for (const held of heldStates) {
+        held.progressMs = now;
+        held.marks += 1;
+        held.longestGapMs = Math.max(held.longestGapMs, now - held.lastMarkMs);
+        held.lastMarkMs = now;
+    }
+}
+export async function acquireSyncLock(paths, now = new Date(), options = {}) {
     const lockPath = path.join(paths.state_dir, LOCK_FILENAME);
     await fs.mkdir(paths.state_dir, { recursive: true, mode: 0o700 });
     const token = crypto.randomUUID();
+    const held = {
+        startedMs: now.getTime(),
+        progressMs: null,
+        role: options.role ?? "sync",
+        marks: 0,
+        lastMarkMs: now.getTime(),
+        longestGapMs: 0,
+    };
     const existing = await readLock(lockPath);
     if (existing && now.getTime() - existing.heartbeat_ms <= STALE_TAKEOVER_MS) {
+        logHeldByLiveOwner(existing, now);
         return { acquired: false, held_since: existing.heartbeat_at };
     }
     // Free or stale: try an exclusive create first; if another process created it
     // in the gap, re-check staleness before taking over.
-    let owned = await tryExclusiveCreate(lockPath, token, now);
+    let takeover = existing ? "stale" : null;
+    let owned = await tryExclusiveCreate(lockPath, token, now, held);
     if (!owned) {
         const current = await readLock(lockPath);
         if (current && now.getTime() - current.heartbeat_ms <= STALE_TAKEOVER_MS) {
+            logHeldByLiveOwner(current, now);
             return { acquired: false, held_since: current.heartbeat_at };
         }
-        await writeLock(lockPath, token, now); // take over the stale lock
+        takeover = current ? "stale" : "unreadable";
+        await writeLock(lockPath, token, now, held); // take over the stale lock
         owned = true;
     }
+    logAcquired(held, takeover, existing, now);
+    heldStates.add(held);
     const timer = setInterval(() => {
         // Refresh only while the lock is still ours; if we were taken over, stop
         // touching the file so we cannot steal it back from the new owner.
         void (async () => {
             const current = await readLock(lockPath);
             if (current && current.token === token) {
-                await writeLock(lockPath, token, new Date()).catch(() => undefined);
+                await writeLock(lockPath, token, new Date(), held).catch(() => undefined);
             }
             else {
                 clearInterval(timer);
@@ -64,6 +99,8 @@ export async function acquireSyncLock(paths, now = new Date()) {
         handle: {
             release: async () => {
                 clearInterval(timer);
+                heldStates.delete(held);
+                logReleased(held);
                 // Ownership-safe: only remove the lock if it is still ours. A stale
                 // owner that resumes after takeover must not delete the new owner's lock.
                 const current = await readLock(lockPath);
@@ -74,10 +111,46 @@ export async function acquireSyncLock(paths, now = new Date()) {
         },
     };
 }
-async function tryExclusiveCreate(lockPath, token, now) {
+/**
+ * How long this holder kept the lock and how it spent it: the duration that
+ * makes a 10-minute tick measurable (BLI-12872). The longest gap between two
+ * finished units of work is what a slow tick and a stuck one differ by.
+ */
+function logReleased(held) {
+    const nowMs = Date.now();
+    console.error("[sync-lock] lock released", JSON.stringify({
+        reason: "holder_finished",
+        role: held.role,
+        held_ms: Math.max(0, nowMs - held.startedMs),
+        progress_marks: held.marks,
+        longest_gap_ms: Math.max(held.longestGapMs, nowMs - held.lastMarkMs),
+    }));
+}
+/** The ordinary contended case, named so a long hold is measurable (BLI-12872). */
+function logHeldByLiveOwner(owner, now) {
+    console.error("[sync-lock] lock is held by a live owner; standing aside", JSON.stringify({
+        reason: "owner_heartbeat_fresh",
+        owner_pid: owner.pid > 0 ? owner.pid : null,
+        owner_role: owner.role,
+        heartbeat_age_ms: Math.max(0, now.getTime() - owner.heartbeat_ms),
+        held_ms: owner.started_ms === null ? null : Math.max(0, now.getTime() - owner.started_ms),
+    }));
+}
+/** A takeover says why it happened; an ordinary free acquisition says so too. */
+function logAcquired(held, takeover, previous, now) {
+    console.error(takeover === null
+        ? "[sync-lock] lock acquired"
+        : "[sync-lock] took over a lock whose owner stopped answering", JSON.stringify({
+        role: held.role,
+        reason: takeover === "stale" ? "heartbeat_stale" : takeover === "unreadable" ? "lock_file_unreadable" : "lock_free",
+        previous_pid: previous && previous.pid > 0 ? previous.pid : null,
+        previous_heartbeat_age_ms: previous ? Math.max(0, now.getTime() - previous.heartbeat_ms) : null,
+    }));
+}
+async function tryExclusiveCreate(lockPath, token, now, held) {
     try {
         const handle = await fs.open(lockPath, "wx", 0o600);
-        await handle.writeFile(serializeLock(token, now));
+        await handle.writeFile(serializeLock(token, now, held));
         await handle.close();
         return true;
     }
@@ -102,15 +175,18 @@ function isLockHeldError(error) {
         error !== null &&
         error.code === "EEXIST");
 }
-async function writeLock(lockPath, token, now) {
-    await fs.writeFile(lockPath, serializeLock(token, now), { mode: 0o600 });
+async function writeLock(lockPath, token, now, held) {
+    await fs.writeFile(lockPath, serializeLock(token, now, held), { mode: 0o600 });
 }
-function serializeLock(token, now) {
+function serializeLock(token, now, held) {
     const record = {
         pid: process.pid,
         token,
         heartbeat_at: now.toISOString(),
         heartbeat_ms: now.getTime(),
+        started_ms: held.startedMs,
+        progress_ms: held.progressMs,
+        role: held.role,
     };
     return `${JSON.stringify(record)}\n`;
 }
@@ -132,6 +208,11 @@ async function readLock(lockPath) {
                 ? heartbeatAt
                 : new Date(heartbeatMs).toISOString(),
             heartbeat_ms: heartbeatMs,
+            started_ms: finiteOrNull(record["started_ms"]),
+            progress_ms: finiteOrNull(record["progress_ms"]),
+            role: record["role"] === "sync" || record["role"] === "backfill"
+                ? record["role"]
+                : null,
         };
     }
     catch {
@@ -143,8 +224,25 @@ async function readLock(lockPath) {
         return null;
     }
 }
+function finiteOrNull(value) {
+    return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
 /** Read owner metadata without acquiring or disturbing the collection lock. */
-export async function inspectSyncLock(paths) {
+export async function inspectSyncLock(paths, now = Date.now()) {
     const record = await readLock(path.join(paths.state_dir, LOCK_FILENAME));
-    return record ? { pid: record.pid, heartbeat_at: record.heartbeat_at, held: Date.now() - record.heartbeat_ms <= STALE_TAKEOVER_MS } : null;
+    if (!record)
+        return null;
+    return { ...record, held: now - record.heartbeat_ms <= STALE_TAKEOVER_MS };
+}
+/**
+ * Remove the lock file only if it still carries `token`: the holder this caller
+ * inspected and ended. A newer owner's lock is never touched. Returns whether
+ * a file was removed.
+ */
+export async function removeSyncLockIfOwnedBy(paths, token) {
+    const lockPath = path.join(paths.state_dir, LOCK_FILENAME);
+    const current = await readLock(lockPath);
+    if (!current || current.token !== token)
+        return false;
+    return fs.rm(lockPath, { force: true }).then(() => true, () => false);
 }

package/dist/sync-tick-timing.js

@@ -0,0 +1,34 @@
+export function startTickTimer(now = () => Date.now()) {
+    const started = now();
+    const stages = {};
+    const wrap = (name) => async (run) => {
+        const stageStarted = now();
+        try {
+            return await run();
+        }
+        finally {
+            stages[name] = (stages[name] ?? 0) + (now() - stageStarted);
+        }
+    };
+    const report = (outcome, exitCode, errorName) => {
+        const longest = Object.entries(stages).sort((a, b) => b[1] - a[1])[0];
+        console.error("[sync] tick finished", JSON.stringify({
+            outcome,
+            exit_code: exitCode,
+            duration_ms: now() - started,
+            stages,
+            longest_stage: longest?.[0] ?? null,
+            ...(errorName ? { error_name: errorName } : {}),
+        }));
+    };
+    return {
+        selfUpdate: wrap("self_update"),
+        collect: wrap("collect"),
+        heartbeat: wrap("heartbeat"),
+        outcomeReport: wrap("outcome_report"),
+        converge: wrap("converge"),
+        prune: wrap("prune"),
+        finish: (exitCode) => report(exitCode === 0 ? "ok" : "failed", exitCode, null),
+        threw: (error) => report("threw", null, error instanceof Error ? error.name : "unknown"),
+    };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bli-cockpit/cli",
-  "version": "0.2.174",
+  "version": "0.2.180",
   "private": false,
   "type": "module",
   "bin": {
@@ -27,9 +27,9 @@
     "test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-verb-help.mjs && node ../../scripts/assert-public-cli-surface-doors.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-exit-contract.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-cli-doctor-json.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
   },
   "dependencies": {
-    "@bli-cockpit/memory-mcp": "0.1.48",
-    "@bli-cockpit/mcp": "0.1.90",
-    "@bli-cockpit/telemetry-core": "0.1.60"
+    "@bli-cockpit/memory-mcp": "0.1.53",
+    "@bli-cockpit/mcp": "0.1.95",
+    "@bli-cockpit/telemetry-core": "0.1.65"
   },
   "repository": {
     "type": "git",

package/dist/commands/doctor-lock-wait.js

@@ -1,55 +0,0 @@
-import { inspectBackfillLock } from "../backfill-lock.js";
-import { getCollectorRuntimePaths } from "../local-state.js";
-import { inspectSyncLock } from "../sync-lock.js";
-import { doctorLockWaitSeconds } from "./doctor-budget.js";
-const BUSY = new Set(["sync_already_running", "sync_lock_held", "backfill_already_running", "live_sync_paused_during_backfill"]);
-/** Retry the operation itself: its existing exclusive lock acquisition owns the
- * handoff. Merely observing a free lock never grants permission to collect. */
-export async function withDoctorLockWait(context, run, timing = { now: () => Date.now(), sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)) }) {
-    const started = timing.now();
-    const limit = doctorLockWaitSeconds(context) * 1000;
-    let nextProgress = 0;
-    let row = await run();
-    while (BUSY.has(row.code)) {
-        const paths = getCollectorRuntimePaths(context.command.homeDir);
-        const sync = await inspectSyncLock(paths);
-        const backfill = await inspectBackfillLock(paths);
-        const owner = backfill.held
-            ? { pid: backfill.pid, heartbeat_at: backfill.held_since }
-            : sync;
-        const elapsed = timing.now() - started;
-        if (elapsed >= limit) {
-            const pid = owner?.pid;
-            let name = "unknown process";
-            if (pid && pid > 0 && context.io.exec) {
-                const result = process.platform === "win32"
-                    ? await context.io.exec("tasklist", ["/FI", `PID eq ${pid}`, "/FO", "CSV", "/NH"])
-                    : await context.io.exec("ps", ["-p", String(pid), "-o", "comm="]);
-                if (result.code === 0 && result.stdout.trim())
-                    name = result.stdout.trim().split("\n")[0] ?? name;
-            }
-            // BLI-11101: the scheduled sync holding the lock is not a fault; it is
-            // doing the upload this row would prove. Say so by name.
-            const syncOwns = !backfill.held && Boolean(sync?.held);
-            const code = syncOwns ? "sync_running" : "lock_wait_timeout";
-            console.error(`[doctor] ${row.id} lock wait ended`, JSON.stringify({ reason: code, owner_pid: pid ?? null, waited_seconds: Math.round(elapsed / 1000), limit_seconds: Math.round(limit / 1000) }));
-            const message = syncOwns
-                ? `The background sync (pid ${pid ?? "unknown"}, ${name}) is running and holds the collection lock; waited ${Math.round(elapsed / 1000)}s. Run \`cockpit doctor\` after it finishes, or \`cockpit doctor --lock-wait 600\` to wait for it.`
-                : `Waited ${Math.round(elapsed / 1000)}s for the collection lock; owner pid ${pid ?? "unknown"}, ${name}. Run \`cockpit doctor --lock-wait 600\`.`;
-            return { ...row, status: "needs_fix", code, nextAction: "cockpit doctor --lock-wait 600", message };
-        }
-        if (elapsed >= nextProgress) {
-            const seconds = Math.floor(elapsed / 1000);
-            console.error(`[doctor] ${row.id} waiting`, JSON.stringify({ reason: row.code, owner_pid: owner?.pid ?? null, heartbeat_at: owner?.heartbeat_at ?? null, elapsed_seconds: seconds }));
-            context.io.stderr.write(`waiting for the background sync to finish, ${Math.floor(seconds / 60)}m${seconds % 60}s\n`);
-            nextProgress = elapsed + 30_000;
-        }
-        await timing.sleep(Math.min(1000, limit - elapsed));
-        // A live heartbeat means there is no point spawning a collector yet.
-        const currentSync = await inspectSyncLock(paths);
-        const currentBackfill = await inspectBackfillLock(paths);
-        if (!currentSync?.held && !currentBackfill.held)
-            row = await run();
-    }
-    return row;
-}