@bli-cockpit/cli 0.2.46 → 0.2.48

package/dist/adapters/raw-evidence-git-diff.js

@@ -11,8 +11,18 @@
  *
  * Either cap yields a *truncated* result rather than a failure: the bytes that
  * were captured are real evidence, and the truncation reason travels with them.
+ *
+ * One thing that is NOT a broken git (BLI-3551): a repo root that is no longer
+ * on disk. `spawn` reports a missing `cwd` as `ENOENT` with syscall `spawn git`
+ * — indistinguishable, by the error alone, from git not being installed — so a
+ * pruned worktree was logged as `git diff failed` twice per session per tick.
+ * One operator's `sync.err.log` held 30,842 such lines against 7,380 real
+ * ingests. The root is now checked before the spawn and the outcome names
+ * itself: `repo_root_missing`, a fact about the folder rather than a fault in
+ * git.
  */
 import { spawn } from "node:child_process";
+import fs from "node:fs/promises";
 export const MAX_GIT_DIFF_BYTES = 2 * 1024 * 1024;
 export const GIT_DIFF_TIMEOUT_MS = 3_000;
 /** Paths Cockpit never reads, excluded at the git level rather than after. */
@@ -28,7 +38,47 @@ const SECRET_EXCLUDING_PATHSPEC = [
     ":(exclude)**/*.pem",
     ":(exclude)**/*.key",
 ];
+/**
+ * The repo root a diff was asked for is not on disk any more.
+ *
+ * A named, typed outcome rather than a bare `ENOENT`: the caller reports it as
+ * its own skip reason and logs it once per root, and nothing downstream has to
+ * read an errno to tell a deleted worktree from a broken git install.
+ */
+export const REPO_ROOT_MISSING_REASON = "repo_root_missing";
+export class RepoRootMissingError extends Error {
+    reason = REPO_ROOT_MISSING_REASON;
+    constructor() {
+        // No path in the message: this string reaches receipts and logs, and a
+        // local path is exactly what must not travel (see health-detail.ts).
+        super("git diff skipped: the repository root is no longer on disk");
+        this.name = "RepoRootMissingError";
+    }
+}
+/**
+ * Is this repo root still a directory on disk?
+ *
+ * Answered with one `stat` rather than by attempting the spawn and reading the
+ * errno, so the same answer holds on both host families: Windows reports a
+ * missing `cwd` through the same `ENOENT`/`spawn git` shape macOS does, and a
+ * path that exists but is a file is equally not a repo root.
+ */
+export async function repoRootExists(repoRoot) {
+    try {
+        const stats = await fs.stat(repoRoot);
+        return stats.isDirectory();
+    }
+    catch {
+        return false;
+    }
+}
 export async function runGitDiff(args, repoRoot) {
+    // Checked before the spawn, not after: `spawn` collapses "no such cwd" and
+    // "no such program" into one `ENOENT`, and only one of those is worth an
+    // operator's attention.
+    if (!(await repoRootExists(repoRoot))) {
+        throw new RepoRootMissingError();
+    }
     return new Promise((resolve, reject) => {
         const child = spawn("git", [...args, ...SECRET_EXCLUDING_PATHSPEC], {
             cwd: repoRoot,

package/dist/adapters/raw-evidence-keys.js

@@ -8,7 +8,10 @@
  *    already staged, so they must never depend on a clock or a path.
  * 2. **Remote object keys** — the readable `operators/…/repos/…/sessions/…`
  *    namespace an operator reads during an incident, followed by an immutable
- *    content address (or a pack-relative manifest path).
+ *    content address. Every raw-evidence object carries its own content hash in
+ *    its key, the manifest included since BLI-3552; the pack-relative fallback
+ *    below survives only for a caller that has no content address to give, and
+ *    a key that does not name its own bytes can collide with different bytes.
  *
  * Everything here is pure: same input, same name, on every machine and every
  * platform. Nothing in this file reads a file, spawns a process or logs.

package/dist/adapters/raw-evidence-pack-store.js

@@ -97,10 +97,19 @@ async function countPriorPacks(rawEvidenceRoot, workContextId, packId) {
 /**
  * Write the manifest, or keep the one already in a reused pack.
  *
- * Byte-stability matters here: the manifest's object key is the only key that
- * embeds the pack id, so rewriting it with a fresh `created_at` every sync
- * would push different bytes at the same content-addressed key forever. A
- * reused pack whose manifest already describes exactly these files keeps it.
+ * Byte-stability matters here: rewriting the manifest with a fresh `created_at`
+ * on every sync means a new remote object per sync for a pack whose files have
+ * not changed. A reused pack whose manifest already describes exactly these
+ * files keeps it.
+ *
+ * It used to matter for a second and much sharper reason. The manifest's object
+ * key embedded the PACK id — a function of the other files' hashes, not of the
+ * manifest's own bytes — so a rewrite pushed different bytes at the identical
+ * key, and once one of them was durable `begin` answered
+ * `hash_mismatch_committed_object` forever. Reuse was the only thing standing
+ * in the way, and it does not hold when the pack directory is refilled, pruned
+ * or absent. BLI-3552 moved the manifest onto its own content address, so this
+ * is now a cost saving rather than the load-bearing invariant.
  */
 export async function stageManifest(options) {
     if (options.reusePack) {

package/dist/adapters/raw-evidence.js

@@ -14,7 +14,7 @@ import { sanitizeTextEvidenceForUpload } from "./raw-evidence-sanitize.js";
 import { countEvidenceEntries, makeEvidenceCompleteness, markBudgetCapApplied, markCapApplied, recordScanned, recordSkipCount, recordTruncationCount, } from "./raw-evidence-completeness.js";
 import { evidenceEntry, pointerFromEntry, RAW_EVIDENCE_BUCKET, } from "./raw-evidence-manifest.js";
 import { chmodPrivate, ensurePrivateDir, persistStagingState, promoteStagedPack, stageManifest, } from "./raw-evidence-pack-store.js";
-import { GIT_DIFF_TIMEOUT_MS, MAX_GIT_DIFF_BYTES, runGitDiff, } from "./raw-evidence-git-diff.js";
+import { GIT_DIFF_TIMEOUT_MS, MAX_GIT_DIFF_BYTES, REPO_ROOT_MISSING_REASON, RepoRootMissingError, repoRootExists, runGitDiff, } from "./raw-evidence-git-diff.js";
 // Re-exported so every consumer keeps importing from `adapters/raw-evidence`.
 export { RAW_EVIDENCE_BUCKET, RAW_EVIDENCE_RETENTION_MODE, } from "./raw-evidence-manifest.js";
 export { sanitizeTextEvidenceForUpload } from "./raw-evidence-sanitize.js";
@@ -247,6 +247,19 @@ async function finishWithPromotedPack(collection, options, run) {
         mediaType: "application/json",
         redactedSummary: "Local raw evidence pack manifest.",
         bytes: manifestBytes,
+        // The manifest carries its own content hash in its key, exactly like
+        // every other object in the pack (BLI-3552). Without it the key was
+        // `…/<packId>/manifest.json`, and a pack id is a function of the OTHER
+        // files' hashes — not of the manifest's bytes, which also carry
+        // `created_at`, the ordering-dependent `files[].relative_path`, `branch`,
+        // and the skipped/redacted/reused ledgers. So the same key named
+        // different bytes whenever the pack directory was not adopted verbatim
+        // (a refill, a pruned or wiped state dir, a machine that had never seen
+        // the pack). `begin` then answered `hash_mismatch_committed_object` on
+        // every sync forever, because the new manifest is stable and the old one
+        // is durable. `stageManifest`'s byte-stability trick still stands; it is
+        // now a nice-to-have rather than the only thing between us and a loop.
+        contentAddress: `manifest/${sha256(manifestBytes).slice(0, 16)}.json`,
     }));
     await persistStagingState(options.stateDir, collection.staging, context.now.toISOString());
     console.error("[raw-evidence] pack staged", JSON.stringify({
@@ -1011,6 +1024,25 @@ const GIT_DIFF_TARGETS = [
     { label: "staged", args: ["diff", "--cached", "--no-ext-diff", "--"] },
 ];
 async function collectGitDiffFiles(collection, repoRoot) {
+    // BLI-3551: asked once per root, before either target, so a pruned worktree
+    // costs ONE line instead of one per diff target per session per tick — and
+    // the line says the folder is gone rather than accusing git of failing.
+    if (!(await repoRootExists(repoRoot))) {
+        console.error("[raw-evidence] git diff skipped, repository root is no longer on disk", JSON.stringify({
+            reason: REPO_ROOT_MISSING_REASON,
+            diff_targets_skipped: GIT_DIFF_TARGETS.length,
+            next_action: "nothing to do; the diff returns when the worktree is restored or the session ages out",
+        }));
+        for (const target of GIT_DIFF_TARGETS) {
+            recordScanned(collection, "git_diff");
+            collection.skipped.push({
+                kind: "git_diff",
+                label: target.label,
+                reason: REPO_ROOT_MISSING_REASON,
+            });
+        }
+        return;
+    }
     for (const target of GIT_DIFF_TARGETS) {
         recordScanned(collection, "git_diff");
         let diff;
@@ -1018,6 +1050,16 @@ async function collectGitDiffFiles(collection, repoRoot) {
             diff = await runGitDiff(target.args, repoRoot);
         }
         catch (error) {
+            // The root was there a moment ago and is not now (or a second collector
+            // pruned it mid-tick). Same named outcome, still not a git failure.
+            if (error instanceof RepoRootMissingError) {
+                collection.skipped.push({
+                    kind: "git_diff",
+                    label: target.label,
+                    reason: REPO_ROOT_MISSING_REASON,
+                });
+                continue;
+            }
             // `git_diff_failed` is the skip label and stays. It covers git not being
             // installed, the folder not being a repo, a locked index and a diff that
             // exceeded the child-process buffer — and the diff is half the evidence

package/dist/autostart-node-path.js

@@ -0,0 +1,141 @@
+import path from "node:path";
+import { realpath, stat } from "node:fs/promises";
+/**
+ * BLI-3553. The scheduler's action string names an absolute node binary — it
+ * has to, because launchd starts the tick with a bare
+ * `PATH=/usr/bin:/bin:/usr/sbin:/sbin` (see scheduled-self-update.ts) and Task
+ * Scheduler is no friendlier. The hazard is that `process.execPath`, which is
+ * what we render, points INTO a version-pinned directory on both of the
+ * layouts this fleet actually runs:
+ *
+ * - Homebrew: `/opt/homebrew/Cellar/node/26.0.0/bin/node`. `brew upgrade node`
+ *   deletes that whole directory, so every tick afterwards exits 127 forever.
+ *   Observed verbatim in `launchctl print gui/501/com.bli.cockpit.sync` on the
+ *   reference Mac, 2026-09-04.
+ * - nvm-windows: `%APPDATA%\nvm\v22.11.0\node.exe`. `nvm use <other>` leaves
+ *   the old directory in place but the operator's node moves; `nvm uninstall`
+ *   deletes it outright.
+ *
+ * Both layouts publish a STABLE alias beside the versioned one — Homebrew's
+ * `<prefix>/opt/<formula>/bin/node` symlink, nvm-windows' `NVM_SYMLINK`
+ * junction (`C:\Program Files\nodejs` by default) — and that alias is exactly
+ * what survives the upgrade. So: when the running node is version-pinned, and
+ * the stable alias exists AND resolves to the same major version, render the
+ * alias. Otherwise render the exact path we were given and let the caller's
+ * runtime fallback (renderDarwinSyncCommand) find a node another way.
+ *
+ * "Verify against what the platform returns, not what you passed it"
+ * (BLI-2541): the check reads the symlink back off the filesystem and compares
+ * the version segment it actually lands on. A guessed alias that happens not to
+ * exist, or that a `brew link` left pointing at node@20 while the collector
+ * runs on 26, is rejected by name rather than written into the plist.
+ */
+/** The zsh the launchd plist runs; also the login shell the fallback probes. */
+export const DARWIN_LOGIN_SHELL = "/bin/zsh";
+const defaultProbe = {
+    realpath: (candidate) => realpath(candidate),
+    exists: (candidate) => stat(candidate).then(() => true, () => false),
+};
+/** `/opt/homebrew/Cellar/node@22/22.11.0/bin/node` → prefix, formula, version. */
+const HOMEBREW_CELLAR_NODE = /^(?<prefix>.*)\/Cellar\/(?<formula>node(?:@[0-9]+)?)\/(?<version>[^/]+)\/bin\/node$/u;
+/** `C:\Users\x\AppData\Roaming\nvm\v22.11.0\node.exe` → version. */
+const NVM_WINDOWS_NODE = /[\\/]nvm[\\/]v(?<version>[0-9]+(?:\.[0-9]+)*)[\\/]node\.exe$/iu;
+const NVM_WINDOWS_DEFAULT_SYMLINK = "C:\\Program Files\\nodejs";
+/**
+ * The node path the scheduler should name, given the one this process is
+ * running under. Never throws: every failure is a reason label and the
+ * original path.
+ */
+export async function resolveStableNodeExecutable(nodeExecutable, options = {}) {
+    const platform = options.platform ?? process.platform;
+    const probe = {
+        ...defaultProbe,
+        env: options.probe?.env ?? process.env,
+        ...options.probe,
+    };
+    if (platform === "darwin") {
+        return resolveHomebrewStableNode(nodeExecutable, probe);
+    }
+    if (platform === "win32") {
+        return resolveNvmWindowsStableNode(nodeExecutable, probe);
+    }
+    return { path: nodeExecutable, reason: "not_version_pinned", changed: false };
+}
+async function resolveHomebrewStableNode(nodeExecutable, probe) {
+    const match = HOMEBREW_CELLAR_NODE.exec(nodeExecutable);
+    const prefix = match?.groups?.["prefix"];
+    const formula = match?.groups?.["formula"];
+    const pinnedVersion = match?.groups?.["version"];
+    if (!match || !prefix || !formula || !pinnedVersion) {
+        return unchanged(nodeExecutable, "not_version_pinned");
+    }
+    // Homebrew's opt-prefix: the one path that does not move across upgrades.
+    const candidate = path.posix.join(prefix, "opt", formula, "bin", "node");
+    return verifyStableAlias({
+        original: nodeExecutable,
+        candidate,
+        pinnedVersion,
+        readVersion: (resolved) => HOMEBREW_CELLAR_NODE.exec(resolved)?.groups?.["version"] ?? null,
+        probe,
+    });
+}
+async function resolveNvmWindowsStableNode(nodeExecutable, probe) {
+    const pinnedVersion = NVM_WINDOWS_NODE.exec(nodeExecutable)?.groups?.["version"];
+    if (!pinnedVersion)
+        return unchanged(nodeExecutable, "not_version_pinned");
+    // nvm-windows points NVM_SYMLINK at whichever version is active; the
+    // installer's default is C:\Program Files\nodejs and that is what every
+    // machine-wide PATH entry names.
+    const symlinkDir = (probe.env?.["NVM_SYMLINK"] ?? "").trim() || NVM_WINDOWS_DEFAULT_SYMLINK;
+    const candidate = path.win32.join(symlinkDir, "node.exe");
+    if (path.win32.resolve(candidate).toLowerCase() ===
+        path.win32.resolve(nodeExecutable).toLowerCase()) {
+        return unchanged(nodeExecutable, "not_version_pinned");
+    }
+    return verifyStableAlias({
+        original: nodeExecutable,
+        candidate,
+        pinnedVersion,
+        readVersion: (resolved) => NVM_WINDOWS_NODE.exec(resolved)?.groups?.["version"] ?? null,
+        probe,
+    });
+}
+/**
+ * The BLI-2541 half: read the alias back off the filesystem and judge what it
+ * actually lands on, rather than trusting the string we composed.
+ */
+async function verifyStableAlias(options) {
+    const { original, candidate, pinnedVersion, readVersion, probe } = options;
+    if (!(await probe.exists(candidate))) {
+        return unchanged(original, "stable_alias_missing");
+    }
+    const resolved = await probe.realpath(candidate).catch(() => null);
+    if (!resolved)
+        return unchanged(original, "stable_alias_unresolvable");
+    // Exactly the binary we are running: nothing left to check.
+    const originalResolved = await probe.realpath(original).catch(() => original);
+    if (samePath(resolved, originalResolved)) {
+        return { path: candidate, reason: "pinned_to_stable_alias", changed: true };
+    }
+    const resolvedVersion = readVersion(resolved);
+    if (!resolvedVersion)
+        return unchanged(original, "stable_alias_unresolvable");
+    const resolvedMajor = majorOf(resolvedVersion);
+    const pinnedMajor = majorOf(pinnedVersion);
+    // An unparseable version on either side is a mismatch, not a pass: two
+    // nulls comparing equal would silently accept an alias we cannot judge.
+    if (!resolvedMajor || !pinnedMajor || resolvedMajor !== pinnedMajor) {
+        return unchanged(original, "stable_alias_major_mismatch");
+    }
+    return { path: candidate, reason: "pinned_to_stable_alias", changed: true };
+}
+function unchanged(nodeExecutable, reason) {
+    return { path: nodeExecutable, reason, changed: false };
+}
+function samePath(left, right) {
+    return left.replace(/[\\/]+$/u, "").toLowerCase() ===
+        right.replace(/[\\/]+$/u, "").toLowerCase();
+}
+function majorOf(version) {
+    return /^([0-9]+)/u.exec(version)?.[1] ?? null;
+}

package/dist/autostart-self-heal.js

@@ -1,12 +1,14 @@
 import fs from "node:fs/promises";
 import path from "node:path";
-import { autostartStatus, installAutostartAgent, } from "./autostart.js";
+import { spawn } from "node:child_process";
+import { autostartStatus, installAutostartAgent, registeredRuntimePathProblems, } from "./autostart.js";
+import { detectSecondCockpitInstall, secondInstallReason, } from "./second-install.js";
 export const AUTOSTART_REPAIR_THROTTLE_MARKER = ".last-autostart-repair";
 const AUTOSTART_REPAIR_MIN_INTERVAL_MS = 24 * 60 * 60 * 1000;
 const DETAIL_MAX_CHARS = 300;
 export async function runAutostartSelfHeal(paths, options) {
     const platform = options.platform ?? process.platform;
-    if (platform !== "win32")
+    if (platform !== "win32" && platform !== "darwin")
         return null;
     if (options.repoRoots.length === 0)
         return null;
@@ -18,14 +20,49 @@ export async function runAutostartSelfHeal(paths, options) {
         exec: options.exec,
         platform,
     });
-    // Healthy is the steady state and stays silent; absent means the operator
-    // (or onboarding) owns the decision, not this tick.
-    if (status.status !== "not_loaded")
+    // Absent means the operator (or onboarding) owns the decision, not this tick.
+    if (status.status === "absent")
         return null;
-    // Attempts are throttled like the self-update's (marker mtime, written for
-    // the attempt not the outcome) so a persistently failing repair cannot spawn
-    // a registration every 15 minutes. The status probe above still runs every
-    // tick — it is one schtasks query.
+    // A registration can read as "loaded" and still name a binary that no longer
+    // exists — that is precisely what `brew upgrade node` leaves behind. Ask the
+    // filesystem about the paths the SCHEDULER holds, not the ones this process
+    // is running under (BLI-3553).
+    const runtimeProblems = status.status === "loaded"
+        ? await registeredRuntimePathProblems({
+            homeDir: options.homeDir,
+            exec: options.exec,
+            platform,
+        })
+        : [];
+    if (status.status !== "not_loaded" && runtimeProblems.length === 0) {
+        // Healthy is the steady state and stays silent.
+        return null;
+    }
+    const problem = [
+        ...(status.message ? [status.message] : []),
+        ...runtimeProblems,
+    ]
+        .join("; ")
+        .slice(0, DETAIL_MAX_CHARS) || "registration not loaded";
+    // Two installs fighting over one registration re-register each other every
+    // tick. Naming that is the repair; rewriting the plist would BE the flapping.
+    const secondInstall = await detectSecondCockpitInstall({
+        exec: options.exec,
+        platform,
+        cliEntryPoint: options.cliEntryPoint,
+    });
+    if (secondInstall.detected) {
+        return {
+            status: "skipped",
+            reason: secondInstallReason(secondInstall),
+            detail: `${problem}; refusing to re-register while ${secondInstall.install_count} Tower installs are on PATH`.slice(0, DETAIL_MAX_CHARS),
+            step: platform === "darwin" ? "autostart_heal" : "autostart_repair",
+        };
+    }
+    // Attempts are throttled (marker mtime, written for the attempt not the
+    // outcome) so a persistently failing repair cannot run every 15 minutes. The
+    // status probe above still runs every tick — it is one launchctl/schtasks
+    // query.
     const now = options.now ?? new Date();
     const marker = path.join(paths.state_dir, AUTOSTART_REPAIR_THROTTLE_MARKER);
     const lastAttempt = await fs.stat(marker).catch(() => null);
@@ -35,7 +72,8 @@ export async function runAutostartSelfHeal(paths, options) {
     }
     await fs.mkdir(paths.state_dir, { recursive: true }).catch(() => undefined);
     await fs.writeFile(marker, now.toISOString()).catch(() => undefined);
-    const problem = (status.message ?? "task not loaded").slice(0, DETAIL_MAX_CHARS);
+    if (platform === "darwin")
+        return scheduleDarwinHeal(options, problem);
     const repaired = await installAutostartAgent({
         homeDir: options.homeDir,
         repoRoot: options.repoRoots[0],
@@ -45,11 +83,77 @@ export async function runAutostartSelfHeal(paths, options) {
         platform,
     });
     if (repaired.loaded) {
-        return { status: "ok", reason: "autostart_repaired", detail: problem };
+        return {
+            status: "ok",
+            reason: "autostart_repaired",
+            detail: problem,
+            step: "autostart_repair",
+        };
     }
     return {
         status: "fail",
         reason: "autostart_repair_failed",
         detail: (repaired.message ?? problem).slice(0, DETAIL_MAX_CHARS),
+        step: "autostart_repair",
     };
+}
+/**
+ * Hands the repair to a process launchd is not about to kill.
+ *
+ * `detached: true` puts the child in its own session, so it survives the
+ * launchd job's process group going away; `stdio: "ignore"` means it holds no
+ * descriptor on the job's log files (which the tick may be rotating); `unref()`
+ * lets this process exit immediately. The child's first act is to WAIT for
+ * this pid to disappear — see commands/autostart-heal.ts.
+ */
+async function scheduleDarwinHeal(options, problem) {
+    const nodeExecutable = options.nodeExecutable ?? process.execPath;
+    const cliEntryPoint = options.cliEntryPoint ?? process.argv[1] ?? "";
+    if (!cliEntryPoint) {
+        return {
+            status: "fail",
+            reason: "heal_entry_point_unknown",
+            detail: problem,
+            step: "autostart_heal",
+        };
+    }
+    const args = [
+        cliEntryPoint,
+        "autostart",
+        "heal-detached",
+        "--parent-pid",
+        String(process.pid),
+        ...(options.homeDir ? ["--home", options.homeDir] : []),
+        ...(options.dashboardUrl ? ["--dashboard-url", options.dashboardUrl] : []),
+    ];
+    const spawner = options.spawnDetached ?? defaultSpawnDetached;
+    let child = null;
+    try {
+        child = spawner(nodeExecutable, args);
+    }
+    catch {
+        child = null;
+    }
+    if (!child || child.pid === null) {
+        return {
+            status: "fail",
+            reason: "heal_spawn_failed",
+            detail: problem,
+            step: "autostart_heal",
+        };
+    }
+    return {
+        status: "skipped",
+        reason: "heal_scheduled",
+        detail: problem,
+        step: "autostart_heal",
+    };
+}
+function defaultSpawnDetached(command, args) {
+    const child = spawn(command, args, {
+        detached: true,
+        stdio: "ignore",
+    });
+    child.unref();
+    return { pid: child.pid ?? null };
 }