@indigoai-us/hq-cli 5.345.202 → 5.345.203

package/dist/lib/daemon/selfcheck.js

@@ -0,0 +1,150 @@
+/**
+ * The daemon's sync-engine self-check (`hq daemon service sync-core-selfcheck`).
+ *
+ * It reads the supervisor's state and holds and finds engine units
+ * (`sync-core:<vault>`) that have been held, or failing without a heartbeat,
+ * for more than ATTENTION_AFTER_MS. When there are any, it sends one DM to the
+ * person signed in on this machine, at most once a day per machine whichever
+ * units are affected, naming every affected vault in that one DM. The 16-hour outage of 2026-11-03 was
+ * found by hand; this makes the next one reach the owner.
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { syncCoreUnitId } from "../sync-core/engine-home.js";
+import { ensurePrivateDir } from "./paths.js";
+/** An engine unit that has been held or failing this long needs a person. */
+export const ATTENTION_AFTER_MS = 30 * 60_000;
+/** This machine sends at most one self-check DM per this long. */
+export const REPORT_INTERVAL_MS = 24 * 60 * 60_000;
+const ENGINE_UNIT_PREFIX = syncCoreUnitId("");
+/** Engine units held, or failing without a heartbeat, for more than ATTENTION_AFTER_MS at `now`. */
+export function engineUnitsNeedingAttention(state, holds, now, 
+/** False for a vault that is skipped on this machine or not on the rollout. */
+wanted = () => true) {
+    const problems = [];
+    const units = state?.units ?? [];
+    const seen = new Set();
+    for (const unit of units) {
+        if (!unit.id.startsWith(ENGINE_UNIT_PREFIX))
+            continue;
+        if (!wanted(unit.id.slice(ENGINE_UNIT_PREFIX.length)))
+            continue;
+        seen.add(unit.id);
+        const problem = unitProblem(unit, holds[unit.id], state, now);
+        if (problem && now - problem.since > ATTENTION_AFTER_MS)
+            problems.push(problem);
+    }
+    // With no daemon state, the persisted holds are all there is to go on. With
+    // state, a hold for a unit the daemon does not supervise is left over from a
+    // vault that is no longer on the engine: it stops nothing, so it is not reported.
+    if (state)
+        return problems.sort((a, b) => a.unitId.localeCompare(b.unitId));
+    for (const [unitId, hold] of Object.entries(holds)) {
+        if (seen.has(unitId) || !unitId.startsWith(ENGINE_UNIT_PREFIX) || hold.retryStopped === true)
+            continue;
+        if (!wanted(unitId.slice(ENGINE_UNIT_PREFIX.length)))
+            continue;
+        if (now - hold.since > ATTENTION_AFTER_MS) {
+            problems.push({ unitId, since: hold.since, what: `held since ${iso(hold.since)}: ${hold.detail}` });
+        }
+    }
+    return problems.sort((a, b) => a.unitId.localeCompare(b.unitId));
+}
+function unitProblem(unit, hold, state, now) {
+    // Stopped on request (`hq daemon stop`, or a stopped recovery probe) is not failing.
+    if (unit.hold?.retryStopped === true || hold?.retryStopped === true)
+        return null;
+    if (unit.status === "held") {
+        const since = unit.hold?.since ?? hold?.since ?? now;
+        const detail = unit.hold?.detail ?? hold?.detail ?? unit.reason ?? "held";
+        return { unitId: unit.id, since, what: `held since ${iso(since)}: ${detail}` };
+    }
+    // Paused on purpose (sync paused, waiting for the legacy sync to hand over) is not failing.
+    if (unit.status !== "backoff" && unit.status !== "exited" && unit.status !== "running")
+        return null;
+    const last = unit.lastHeartbeatAt ? Date.parse(unit.lastHeartbeatAt) : Number.NaN;
+    // No heartbeat yet: count from when the daemon started supervising it.
+    const since = Number.isFinite(last) ? last : state.startedAt;
+    if (unit.status === "running" && unit.restarts === 0)
+        return null;
+    if (unit.status === "running" && Number.isFinite(last) && now - last <= ATTENTION_AFTER_MS)
+        return null;
+    const exit = unit.lastExit ? ` (last exit code ${unit.lastExit.code ?? "none"}${unit.lastExit.signal ? `, signal ${unit.lastExit.signal}` : ""})` : "";
+    return {
+        unitId: unit.id,
+        since,
+        what: `${unit.status === "running" ? "restarting" : "failing"} with no heartbeat since ${iso(since)}, restarted ${unit.restarts} times${exit}`,
+    };
+}
+function iso(ms) {
+    return new Date(ms).toISOString().replace(/\.\d{3}Z$/, "Z");
+}
+export function readSelfCheckRecord(file) {
+    try {
+        const parsed = JSON.parse(fs.readFileSync(file, "utf8"));
+        if (typeof parsed.lastSentAt !== "number" || !Array.isArray(parsed.units))
+            return null;
+        return { lastSentAt: parsed.lastSentAt, units: parsed.units.filter((u) => typeof u === "string") };
+    }
+    catch {
+        return null;
+    }
+}
+export function writeSelfCheckRecord(file, record) {
+    ensurePrivateDir(path.dirname(file));
+    const tmp = `${file}.${process.pid}.tmp`;
+    fs.writeFileSync(tmp, `${JSON.stringify(record, null, 2)}\n`, { mode: 0o600 });
+    fs.renameSync(tmp, file);
+}
+/**
+ * Send now: something is wrong and this machine has not sent a self-check DM
+ * in the last day. Which units are affected does not matter, so units that
+ * take turns failing cannot send a DM on every change.
+ */
+export function shouldReport(problems, last, now) {
+    if (problems.length === 0)
+        return false;
+    if (!last)
+        return true;
+    return now - last.lastSentAt >= REPORT_INTERVAL_MS;
+}
+/** The DM text for the owner. Plain, with the commands that fix each case. */
+export function selfCheckMessage(problems, machine, 
+/** A name for a vault (company slug, "personal"), when known. */
+vaultName = () => undefined) {
+    const named = (p) => {
+        const uid = p.unitId.slice(ENGINE_UNIT_PREFIX.length);
+        const name = vaultName(uid);
+        return name ? `${name} (${uid})` : uid;
+    };
+    const names = problems.map(named).join(", ");
+    const body = `HQ sync on ${machine}: the sync engine for ${names} has not synced for over ${ATTENTION_AFTER_MS / 60_000} minutes. ` +
+        "Run `hq daemon status` on that machine.";
+    const lines = problems.map((p) => `- ${named(p)}: ${p.what}. Retry with: hq daemon restart ${p.unitId}`);
+    lines.push("", "Logs: hq daemon logs <unit>. This check runs in hq daemon and sends at most one DM a day from this machine.");
+    return { body, details: lines.join("\n") };
+}
+export async function runSelfCheck(deps) {
+    const now = deps.now();
+    const problems = engineUnitsNeedingAttention(deps.readState(), deps.readHolds(), now, deps.wanted);
+    if (problems.length === 0) {
+        deps.log("sync engine self-check: every engine unit is healthy");
+        return "healthy";
+    }
+    for (const p of problems)
+        deps.log(`sync engine self-check: ${p.unitId} ${p.what}`);
+    const last = readSelfCheckRecord(deps.recordPath);
+    if (!shouldReport(problems, last, now)) {
+        deps.log(`sync engine self-check: already reported at ${iso(last.lastSentAt)}; next report after ${iso(last.lastSentAt + REPORT_INTERVAL_MS)}`);
+        return "already-reported";
+    }
+    const to = await deps.sendToOwner(selfCheckMessage(problems, deps.machine, deps.vaultName));
+    if (to === null) {
+        deps.log("sync engine self-check: this machine is signed in as an agent, so there is no owner to DM; nothing was sent");
+        return "no-owner";
+    }
+    writeSelfCheckRecord(deps.recordPath, { lastSentAt: now, units: problems.map((p) => p.unitId) });
+    deps.log(`sync engine self-check: sent a DM to ${to}`);
+    return "reported";
+}
+//# sourceMappingURL=selfcheck.js.map

package/dist/lib/daemon/supervisor.d.ts

@@ -46,6 +46,30 @@ export declare const CLI_UPDATE_SYNC_IDLE_KILL_GRACE_MS = 60000;
 export declare const HANDOFF_STAGGER_MS = 15000;
 /** How long a draining unit may take to finish its work before it gets SIGTERM. */
 export declare const HANDOFF_DRAIN_MAX_MS: number;
+/** hq-syncd's exit code when another engine holds its engine lease or the HQ operation lock. */
+export declare const SYNC_CORE_LOCK_HELD_EXIT = 17;
+/**
+ * Heartbeat-timeout restarts of an engine that reported growing progress are
+ * not counted toward the hold, up to this many per failure episode. After
+ * that they count, so an engine that moves a few bytes each run and then hangs
+ * is still held (and reported by the self-check) instead of restarting forever.
+ */
+export declare const SYNC_CORE_PROGRESS_EXEMPT_MAX = 6;
+/** How often a unit held for exit 17 checks whether the lock holder is still alive. */
+export declare const LOCK_HOLDER_POLL_MS = 5000;
+/**
+ * The pid hq-syncd names when it exits 17: "local sync engine is already
+ * running (pid N)" or "HQ operation lock for ... is held by "cmd" (pid N)".
+ * Null when the output names no live-looking pid.
+ */
+export declare function lockHolderPidFromOutput(output: string | undefined): number | null;
+/**
+ * Bytes the engine reports having moved, from a protocol heartbeat that
+ * carries `progress` (see the engine contract in the hq-sync-core 0.1.24
+ * notes): `{"type":"heartbeat","progress":{"bytesReceived":N,"bytesSent":M}}`,
+ * both cumulative over the engine process. Null for a heartbeat without it.
+ */
+export declare function heartbeatProgressBytes(event: Record<string, unknown>): number | null;
 export interface SupervisedChild {
     pid?: number;
     stdout?: {
@@ -84,6 +108,11 @@ export interface UnitHold {
     probing?: boolean;
     /** Explicit stop of a recovering unit; only an explicit restart clears it. */
     retryStopped?: boolean;
+    /**
+     * Exit 17 holds: the pid hq-syncd said holds its lock. The hold is released
+     * as soon as that process is gone instead of waiting out the cooldown.
+     */
+    lockHolderPid?: number;
 }
 /** The holds file: unit id to hold. */
 export type PersistedHolds = Record<string, UnitHold>;
@@ -172,6 +201,8 @@ export interface SupervisorDeps {
         read(): PersistedHolds;
         write(holds: PersistedHolds): void;
     };
+    /** Whether a process is alive (signal 0). Enables releasing exit-17 holds when the lock holder exits. */
+    pidAlive?: (pid: number) => boolean;
 }
 export interface Supervisor {
     start(): void;

package/dist/lib/daemon/supervisor.js

@@ -51,6 +51,47 @@ const MESH_OWNER_LOG_INTERVAL_MS = 10 * 60_000;
 export const HANDOFF_STAGGER_MS = 15_000;
 /** How long a draining unit may take to finish its work before it gets SIGTERM. */
 export const HANDOFF_DRAIN_MAX_MS = 15 * 60_000;
+/** hq-syncd's exit code when another engine holds its engine lease or the HQ operation lock. */
+export const SYNC_CORE_LOCK_HELD_EXIT = 17;
+/**
+ * Heartbeat-timeout restarts of an engine that reported growing progress are
+ * not counted toward the hold, up to this many per failure episode. After
+ * that they count, so an engine that moves a few bytes each run and then hangs
+ * is still held (and reported by the self-check) instead of restarting forever.
+ */
+export const SYNC_CORE_PROGRESS_EXEMPT_MAX = 6;
+/** How often a unit held for exit 17 checks whether the lock holder is still alive. */
+export const LOCK_HOLDER_POLL_MS = 5_000;
+/**
+ * The pid hq-syncd names when it exits 17: "local sync engine is already
+ * running (pid N)" or "HQ operation lock for ... is held by "cmd" (pid N)".
+ * Null when the output names no live-looking pid.
+ */
+export function lockHolderPidFromOutput(output) {
+    if (!output)
+        return null;
+    const matches = [...output.matchAll(/\(pid (\d+)\)/g)];
+    const pid = Number(matches.at(-1)?.[1]);
+    return Number.isSafeInteger(pid) && pid > 1 ? pid : null;
+}
+/**
+ * Bytes the engine reports having moved, from a protocol heartbeat that
+ * carries `progress` (see the engine contract in the hq-sync-core 0.1.24
+ * notes): `{"type":"heartbeat","progress":{"bytesReceived":N,"bytesSent":M}}`,
+ * both cumulative over the engine process. Null for a heartbeat without it.
+ */
+export function heartbeatProgressBytes(event) {
+    if (event.type !== "heartbeat")
+        return null;
+    const progress = event.progress;
+    if (!progress || typeof progress !== "object" || Array.isArray(progress))
+        return null;
+    const { bytesReceived, bytesSent } = progress;
+    const count = (v) => (typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : 0);
+    if (typeof bytesReceived !== "number" && typeof bytesSent !== "number")
+        return null;
+    return count(bytesReceived) + count(bytesSent);
+}
 const PASS_REQUEST_DIR_ENV = "HQ_SYNC_PASS_REQUEST_DIR";
 function passRequestDirOf(env) {
     const dir = env?.[PASS_REQUEST_DIR_ENV];
@@ -182,6 +223,9 @@ export function createSupervisor(deps) {
         if (slot.recoveryHealthyTimer !== undefined)
             clearT(slot.recoveryHealthyTimer);
         slot.recoveryHealthyTimer = undefined;
+        if (slot.lockHolderTimer !== undefined)
+            clearT(slot.lockHolderTimer);
+        slot.lockHolderTimer = undefined;
     };
     const clearTimer = (slot) => {
         clearRecoveryWindow(slot);
@@ -363,16 +407,60 @@ export function createSupervisor(deps) {
         if (deps.holds && recoverableHold(slot.plan, slot.state.hold)) {
             slot.state.hold.retryAt = now() + EPISODE_HEALTHY_RESET_MS;
         }
+        if (kind === "exhausted" && slot.plan.service === "sync-core"
+            && slot.state.lastExit?.code === SYNC_CORE_LOCK_HELD_EXIT && slot.lastLockHolderPid !== undefined) {
+            slot.state.hold.lockHolderPid = slot.lastLockHolderPid;
+            slot.state.hold.detail += `; the lock is held by pid ${slot.lastLockHolderPid}, released when it exits`;
+        }
         persistedHolds[slot.plan.id] = slot.state.hold;
         showHeld(slot);
         deps.log("error", `holding ${slot.plan.id}: ${slot.state.reason}`);
         writeHolds();
         armRecoveryProbe(slot);
+        armLockHolderWatch(slot);
+    };
+    /**
+     * A unit held because its lock was taken (exit 17) starts again as soon as
+     * the process holding the lock is gone, instead of after the cooldown.
+     */
+    const armLockHolderWatch = (slot) => {
+        const hold = slot.state.hold;
+        const pid = hold?.lockHolderPid;
+        if (!hold || pid === undefined || !deps.pidAlive || hold.retryStopped === true || stopped || slot.child)
+            return;
+        if (slot.lockHolderTimer !== undefined)
+            clearT(slot.lockHolderTimer);
+        const check = () => {
+            slot.lockHolderTimer = undefined;
+            if (stopped || slots.get(slot.plan.id) !== slot || slot.child || slot.state.hold !== hold || hold.retryStopped === true)
+                return;
+            let alive;
+            try {
+                alive = deps.pidAlive(pid);
+            }
+            catch {
+                alive = true;
+            }
+            if (alive) {
+                slot.lockHolderTimer = setT(check, LOCK_HOLDER_POLL_MS);
+                return;
+            }
+            deps.log("info", `releasing the hold on ${slot.plan.id}: pid ${pid}, which held its lock, has exited`);
+            clearTimer(slot);
+            clearHold(slot);
+            if (slot.plan.blocked)
+                setBlocked(slot);
+            else
+                startLong(slot);
+            emit();
+        };
+        slot.lockHolderTimer = setT(check, 0);
     };
     const clearHold = (slot) => {
         clearRecoveryWindow(slot);
         slot.attempts = 0;
         slot.failures = 0;
+        slot.progressExemptRestarts = 0;
         if (!slot.state.hold && !persistedHolds[slot.plan.id])
             return;
         delete slot.state.hold;
@@ -466,10 +554,11 @@ export function createSupervisor(deps) {
         escalateToKill();
         signalChild(slot, child, "SIGTERM");
     };
-    const scheduleRestart = (slot) => {
+    const scheduleRestart = (slot, countFailure = true) => {
         if (stopped)
             return;
-        slot.failures += 1;
+        if (countFailure)
+            slot.failures += 1;
         if (slot.failures >= EPISODE_MAX_FAILURES) {
             holdUnit(slot, "exhausted", "transient", `${slot.failures} failures without staying up for ${EPISODE_HEALTHY_RESET_MS / 60_000} min`);
             return;
@@ -502,8 +591,20 @@ export function createSupervisor(deps) {
                 slot.heartbeatTimer = undefined;
                 if (stopped || slot.child !== child || slot.expectingExit)
                     return;
-                deps.log("warn", `${slot.plan.id} emitted no sync protocol heartbeat for ${timeoutMs}ms; restarting it`);
-                terminate(slot, () => scheduleRestart(slot));
+                // An engine that reported bytes moving during this run is slow, not
+                // hung: restart it, but do not count it toward the exhausted hold.
+                // Bounded per episode: past SYNC_CORE_PROGRESS_EXEMPT_MAX these count again.
+                const grew = slot.plan.service === "sync-core" && slot.progressedThisRun === true;
+                const progressed = grew && (slot.progressExemptRestarts ?? 0) < SYNC_CORE_PROGRESS_EXEMPT_MAX;
+                if (progressed)
+                    slot.progressExemptRestarts = (slot.progressExemptRestarts ?? 0) + 1;
+                deps.log("warn", `${slot.plan.id} emitted no sync protocol heartbeat for ${timeoutMs}ms; restarting it` +
+                    (progressed
+                        ? ` (it reported transfer progress this run, so this does not count toward holding it; ${slot.progressExemptRestarts}/${SYNC_CORE_PROGRESS_EXEMPT_MAX})`
+                        : grew
+                            ? ` (it reported progress, but ${SYNC_CORE_PROGRESS_EXEMPT_MAX} such restarts already did not count, so this one does)`
+                            : ""));
+                terminate(slot, () => scheduleRestart(slot, !progressed));
                 emit();
             }, timeoutMs);
         };
@@ -543,6 +644,16 @@ export function createSupervisor(deps) {
                         runDeferredCliRestart();
                 }
             }
+            if (slot.plan.service === "sync-core") {
+                const bytes = heartbeatProgressBytes(event);
+                if (bytes !== null) {
+                    // The first progress heartbeat of a run is the baseline: counters are
+                    // cumulative per process, and a handshake alone moves some bytes.
+                    if (slot.progressBytes !== undefined && bytes > slot.progressBytes)
+                        slot.progressedThisRun = true;
+                    slot.progressBytes = Math.max(bytes, slot.progressBytes ?? 0);
+                }
+            }
             const at = now();
             slot.state.lastHeartbeatAt = new Date(at).toISOString();
             const pass = syncPassResult(event, slot.state.lastHeartbeatAt);
@@ -610,8 +721,10 @@ export function createSupervisor(deps) {
                 // run starts a new failure episode, as an unexpected exit would.
                 if (ranFor >= HEALTHY_RUN_MS)
                     slot.attempts = 0;
-                if (ranFor >= EPISODE_HEALTHY_RESET_MS)
+                if (ranFor >= EPISODE_HEALTHY_RESET_MS) {
                     slot.failures = 0;
+                    slot.progressExemptRestarts = 0;
+                }
                 const then = slot.afterExit;
                 slot.afterExit = undefined;
                 then?.();
@@ -688,8 +801,10 @@ export function createSupervisor(deps) {
             deps.log("warn", `${slot.plan.id} exited (code=${code ?? "null"} signal=${signal ?? "none"})`);
             if (ranFor >= HEALTHY_RUN_MS)
                 slot.attempts = 0;
-            if (ranFor >= EPISODE_HEALTHY_RESET_MS)
+            if (ranFor >= EPISODE_HEALTHY_RESET_MS) {
                 slot.failures = 0;
+                slot.progressExemptRestarts = 0;
+            }
             const stoppedFromOutside = signal === "SIGTERM" || signal === "SIGINT";
             if (slot.plan.restart === "on-failure" && (code === 0 || stoppedFromOutside)) {
                 slot.state.status = "exited";
@@ -702,6 +817,12 @@ export function createSupervisor(deps) {
                 const spawnError = slot.lastSpawnError;
                 delete slot.lastSpawnError;
                 const outputTail = spawnError === undefined ? readExitOutputTail(slot) : undefined;
+                delete slot.lastLockHolderPid;
+                if (slot.plan.service === "sync-core" && code === SYNC_CORE_LOCK_HELD_EXIT) {
+                    const holder = lockHolderPidFromOutput(outputTail);
+                    if (holder !== null)
+                        slot.lastLockHolderPid = holder;
+                }
                 if (!holdIfPermanent(slot, { code, signal, spawnError, outputTail }))
                     scheduleRestart(slot);
             }
@@ -740,6 +861,8 @@ export function createSupervisor(deps) {
         delete slot.lastProtocolLine;
         delete slot.lastProtocolStatus;
         delete slot.syncPassActive;
+        delete slot.progressBytes;
+        delete slot.progressedThisRun;
         let child;
         try {
             child = deps.spawn(slot.plan);
@@ -923,6 +1046,7 @@ export function createSupervisor(deps) {
         if (slot.state.hold) {
             showHeld(slot);
             armRecoveryProbe(slot);
+            armLockHolderWatch(slot);
             return;
         }
         startLong(slot);

package/dist/lib/daemon/unit-holds.js

@@ -20,7 +20,8 @@ function isHold(value) {
         && typeof h.detail === "string"
         && typeof h.since === "number"
         && typeof h.failures === "number"
-        && typeof h.keyHash === "string");
+        && typeof h.keyHash === "string"
+        && (h.lockHolderPid === undefined || (typeof h.lockHolderPid === "number" && Number.isSafeInteger(h.lockHolderPid) && h.lockHolderPid > 1)));
 }
 export function readUnitHolds(file) {
     let raw;

package/dist/lib/service-manager/launchd.d.ts

@@ -114,6 +114,31 @@ export declare function activateLaunchd(plistPath: string, run: RunCommand, uid?
     sleep?: SleepFn;
     unloadDeadlineMs?: number;
 }): void;
+export interface LaunchdBootstrapResult {
+    ok: boolean;
+    attempts: number;
+    error?: string;
+}
+/**
+ * Bootstrap a plist that is not loaded, retrying launchd's transient EIO with
+ * backoff. A label that is already loaded counts as success. Unlike
+ * reloadLaunchd this never boots anything out, so it is safe to call on a
+ * running job.
+ */
+export declare function bootstrapLaunchd(plistPath: string, run: RunCommand, uid?: number, opts?: {
+    sleep?: SleepFn;
+}): LaunchdBootstrapResult;
+export interface LaunchdJobInfo {
+    loaded: boolean;
+    /** launchd's `state = ...` line (running, not running, spawn scheduled, ...). */
+    state?: string;
+    pid?: number;
+    /** The loaded job's program arguments (`arguments = { ... }`), when printed. */
+    arguments?: string[];
+}
+/** What `launchctl print gui/<uid>/<label>` says about one job. */
+export declare function launchdJobInfo(label: string, run: RunCommand, uid?: number): LaunchdJobInfo;
+export declare function parseLaunchctlPrintJob(stdout: string): LaunchdJobInfo;
 export declare function deactivateLaunchd(plistPath: string, run: RunCommand, uid?: number): void;
 export declare function launchdIsLoaded(label: string, run: RunCommand, uid?: number): boolean;
 //# sourceMappingURL=launchd.d.ts.map

package/dist/lib/service-manager/launchd.js

@@ -58,9 +58,11 @@ export function launchdPlistPath(home, spec) {
     return path.join(launchAgentsDir(home), `${spec.label}.plist`);
 }
 export function renderLaunchdPlist(spec, host) {
-    const discovered = discoverLaunchdIdentity(host.home);
-    const associatedBundleIdentifier = host.associatedBundleIdentifier ?? discovered.associatedBundleIdentifier;
-    const launcherBinary = host.launchdLauncherBinary ?? discovered.launcherBinary;
+    const discovered = spec.independentOfApp ? {} : discoverLaunchdIdentity(host.home);
+    const associatedBundleIdentifier = spec.independentOfApp
+        ? undefined
+        : host.associatedBundleIdentifier ?? discovered.associatedBundleIdentifier;
+    const launcherBinary = spec.independentOfApp ? undefined : host.launchdLauncherBinary ?? discovered.launcherBinary;
     const args = launchdProgramArguments({ ...host, launchdLauncherBinary: launcherBinary }, spec.args)
         .map((a) => `    <string>${escapeXml(a)}</string>`)
         .join("\n");
@@ -303,6 +305,52 @@ export function activateLaunchd(plistPath, run, uid, opts) {
     }
     throw new Error(`${detail}. ${target} is NOT loaded and will not run until it is. Load it with: launchctl bootstrap ${domain} ${plistPath}`);
 }
+/**
+ * Bootstrap a plist that is not loaded, retrying launchd's transient EIO with
+ * backoff. A label that is already loaded counts as success. Unlike
+ * reloadLaunchd this never boots anything out, so it is safe to call on a
+ * running job.
+ */
+export function bootstrapLaunchd(plistPath, run, uid, opts) {
+    const domain = guiDomain(uid);
+    const target = launchdServiceTarget(plistPath, uid);
+    const sleep = opts?.sleep ?? blockingSleep;
+    if (run("launchctl", ["print", target]).status === 0)
+        return { ok: true, attempts: 0 };
+    let last = run("launchctl", ["bootstrap", domain, plistPath]);
+    let attempts = 1;
+    for (const backoff of LAUNCHD_BOOTSTRAP_BACKOFF_MS) {
+        if (last.status === 0 || !isRetryableLaunchdBootstrap(last))
+            break;
+        sleep(backoff);
+        last = run("launchctl", ["bootstrap", domain, plistPath]);
+        attempts++;
+    }
+    // "already bootstrapped" races (another caller loaded it) are success.
+    if (last.status === 0 || run("launchctl", ["print", target]).status === 0)
+        return { ok: true, attempts };
+    return { ok: false, attempts, error: last.stderr.trim() || last.stdout.trim() || `exit ${last.status}` };
+}
+/** What `launchctl print gui/<uid>/<label>` says about one job. */
+export function launchdJobInfo(label, run, uid) {
+    const r = run("launchctl", ["print", `${guiDomain(uid)}/${label}`]);
+    if (r.status !== 0)
+        return { loaded: false };
+    return parseLaunchctlPrintJob(r.stdout);
+}
+export function parseLaunchctlPrintJob(stdout) {
+    const state = /^\s*state\s*=\s*(.+?)\s*$/m.exec(stdout)?.[1];
+    const pidText = /^\s*pid\s*=\s*(\d+)\s*$/m.exec(stdout)?.[1];
+    const pid = pidText ? Number(pidText) : undefined;
+    const argsBlock = /^\s*arguments\s*=\s*\{\s*\n([\s\S]*?)\n\s*\}/m.exec(stdout)?.[1];
+    const args = argsBlock?.split("\n").map((line) => line.trim()).filter((line) => line.length > 0);
+    return {
+        loaded: true,
+        ...(state ? { state } : {}),
+        ...(pid && pid > 0 ? { pid } : {}),
+        ...(args && args.length > 0 ? { arguments: args } : {}),
+    };
+}
 export function deactivateLaunchd(plistPath, run, uid) {
     run("launchctl", ["bootout", guiDomain(uid), plistPath]);
 }

package/dist/lib/service-manager/migration.d.ts

@@ -1,5 +1,11 @@
 import * as fs from "node:fs";
 import { type RunCommand, type SleepFn } from "./launchd.js";
+/**
+ * Agents that must load at login without the app, so they are never wrapped
+ * in the HQ.app launcher or attributed to its bundle. Kept in step with
+ * DAEMON_LAUNCHD_LABEL in daemon/install.ts.
+ */
+export declare const APP_INDEPENDENT_LAUNCHD_LABELS: ReadonlySet<string>;
 export interface LaunchdMigrationOptions {
     home: string;
     run: RunCommand;
@@ -19,4 +25,12 @@ export interface LaunchdMigrationResult {
 }
 /** First-CLI-run cleanup for HQ's old Node-first LaunchAgents and backup plists. */
 export declare function migrateLegacyNodeLaunchAgents(options: LaunchdMigrationOptions): LaunchdMigrationResult;
+/**
+ * The app-independent form of an HQ LaunchAgent plist written by an older
+ * install: the HQ.app launcher (`hq-sync-menubar --hq-launchd-task`) is
+ * dropped from ProgramArguments and the AssociatedBundleIdentifiers entry is
+ * removed. Everything else (environment, logs, KeepAlive) is kept as written.
+ * Returns null when the plist is already app-independent.
+ */
+export declare function appIndependentLaunchdPlist(source: string): string | null;
 //# sourceMappingURL=migration.d.ts.map

package/dist/lib/service-manager/migration.js

@@ -5,6 +5,12 @@ import { isVersionedNodePath, preferStableNodeBinary } from "../node-binary.js";
 import { preferStableHqBinary } from "../mesh/live/daemon/install.js";
 import { launchdMigrationMarkerPath, shouldRunLaunchdMigration, writeLaunchdMigrationMarker } from "./migration-state.js";
 const HQ_LABEL_PREFIXES = ["ai.indigo.hq-", "ai.getindigo.hq-"];
+/**
+ * Agents that must load at login without the app, so they are never wrapped
+ * in the HQ.app launcher or attributed to its bundle. Kept in step with
+ * DAEMON_LAUNCHD_LABEL in daemon/install.ts.
+ */
+export const APP_INDEPENDENT_LAUNCHD_LABELS = new Set(["ai.getindigo.hq-daemon"]);
 function xml(value) {
     return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
 }
@@ -36,6 +42,8 @@ export function migrateLegacyNodeLaunchAgents(options) {
         }
         if (!name.endsWith(".plist") || !HQ_LABEL_PREFIXES.some((prefix) => name.startsWith(prefix)))
             continue;
+        if (APP_INDEPENDENT_LAUNCHD_LABELS.has(name.slice(0, -".plist".length)))
+            continue;
         const source = files.readFileSync(fullPath, "utf8");
         const args = launchdPlistProgramArguments(source);
         if (args.length < 2 || !isNodeProgram(args[0]))
@@ -82,4 +90,30 @@ export function migrateLegacyNodeLaunchAgents(options) {
     writeLaunchdMigrationMarker(options.home, deferred);
     return { migrated, backupsRemoved, deferred, alreadyDone: false };
 }
+/**
+ * The app-independent form of an HQ LaunchAgent plist written by an older
+ * install: the HQ.app launcher (`hq-sync-menubar --hq-launchd-task`) is
+ * dropped from ProgramArguments and the AssociatedBundleIdentifiers entry is
+ * removed. Everything else (environment, logs, KeepAlive) is kept as written.
+ * Returns null when the plist is already app-independent.
+ */
+export function appIndependentLaunchdPlist(source) {
+    const args = launchdPlistProgramArguments(source);
+    const launcherIndex = args.indexOf("--hq-launchd-task");
+    const wrapped = launcherIndex === 1 && args.length > 3;
+    const associated = /<key>AssociatedBundleIdentifiers<\/key>/.test(source);
+    if (!wrapped && !associated)
+        return null;
+    let updated = source;
+    if (wrapped) {
+        updated = updated.replace(/<key>ProgramArguments<\/key>\s*<array>[\s\S]*?<\/array>/, plistArgs(args.slice(2)).trimStart());
+    }
+    if (associated) {
+        updated = updated.replace(/[ \t]*<key>AssociatedBundleIdentifiers<\/key>\s*(?:<array>[\s\S]*?<\/array>|<string>[^<]*<\/string>)[ \t]*\n?/, "");
+    }
+    if (updated === source || /<key>AssociatedBundleIdentifiers<\/key>/.test(updated) || launchdPlistProgramArguments(updated).includes("--hq-launchd-task")) {
+        throw new Error("Could not safely rewrite the LaunchAgent to load without HQ.app");
+    }
+    return updated;
+}
 //# sourceMappingURL=migration.js.map

package/dist/lib/service-manager/systemd.d.ts

@@ -16,4 +16,17 @@ export declare function renderSystemdUserUnit(spec: ServiceSpec, host: ServiceHo
 export declare function activateSystemd(unitName: string, run: RunCommand): void;
 export declare function deactivateSystemd(unitName: string, run: RunCommand): void;
 export declare function systemdIsActive(unitName: string, run: RunCommand): boolean;
+export interface SystemdUnitInfo {
+    /** `systemctl is-enabled` printed enabled (or enabled-runtime / linked / alias). */
+    enabled: boolean;
+    /** `systemctl is-active` printed active (or activating / reloading). */
+    active: boolean;
+    /** The raw `is-enabled` / `is-active` answers, for status output. */
+    enabledState: string;
+    activeState: string;
+    /** MainPID when systemd reports one. */
+    pid?: number;
+}
+/** Ask systemd about one unit; `scope: "user"` adds `--user`. */
+export declare function systemdUnitInfo(unitName: string, run: RunCommand, scope?: "user" | "system"): SystemdUnitInfo;
 //# sourceMappingURL=systemd.d.ts.map

package/dist/lib/service-manager/systemd.js

@@ -69,4 +69,23 @@ export function systemdIsActive(unitName, run) {
     const r = run("systemctl", ["--user", "is-active", "--quiet", unitName]);
     return r.status === 0;
 }
+/** Ask systemd about one unit; `scope: "user"` adds `--user`. */
+export function systemdUnitInfo(unitName, run, scope = "user") {
+    const prefix = scope === "user" ? ["--user"] : [];
+    const word = (args) => {
+        const r = run("systemctl", [...prefix, ...args, unitName]);
+        return (r.stdout.trim().split("\n")[0] ?? "").trim() || "unknown";
+    };
+    const enabledState = word(["is-enabled"]);
+    const activeState = word(["is-active"]);
+    const show = run("systemctl", [...prefix, "show", "-p", "MainPID", "--value", unitName]);
+    const pid = Number(show.stdout.trim());
+    return {
+        enabled: /^(enabled|enabled-runtime|linked|linked-runtime|alias|static|indirect|generated)$/.test(enabledState),
+        active: /^(active|activating|reloading)$/.test(activeState),
+        enabledState,
+        activeState,
+        ...(show.status === 0 && Number.isInteger(pid) && pid > 0 ? { pid } : {}),
+    };
+}
 //# sourceMappingURL=systemd.js.map