@indigoai-us/hq-cli 5.276.0 → 5.278.0

package/dist/lib/daemon/run.js

@@ -17,7 +17,7 @@ import { agentKitPaths } from "../agent-kit/paths.js";
 import { readBotConfig } from "../bot/config.js";
 import { botsRoot } from "../bot/paths.js";
 import { CLAUDE_RUNNER_SCRIPT, CODEX_RUNNER_SCRIPT, enabledServices, planUnits } from "./catalog.js";
-import { readDaemonConfig } from "./config.js";
+import { effectiveSyncOwner, readDaemonConfig } from "./config.js";
 import { SUPERVISOR_HANDOFF_FLAG, readDaemonFlag } from "./flag.js";
 import { findLegacyUnits, legacyOwnerLookup } from "./legacy.js";
 import { readLegacySyncEnv } from "./legacy-sync-env.js";
@@ -29,6 +29,7 @@ import { takeControlRequests } from "./requests.js";
 import { currentHqCloud, hqCloudRunners } from "./hq-cloud.js";
 import { runnerSessionName } from "./remote-control.js";
 import { adoptChild, createChildTracker, finishSuccession, handoffEligible, readProcStart, stopOrphans, successionPaths, systemdKillMode, takeSuccession, writeHandoff, } from "./succession.js";
+import { listSyncRunnerProcesses, readWatchRunnerOwnerStatus, syncHeartbeatTimeoutMs, syncRunnerNodeOptions, watchRunnerOwnerStatusPath, } from "./sync-runner.js";
 import { STOP_ESCALATION_MS, createSupervisor, } from "./supervisor.js";
 import { userToolDirs } from "./tools.js";
 export const LOG_ROTATE_BYTES = 5 * 1024 * 1024;
@@ -218,7 +219,10 @@ function makeLogger(paths, foreground) {
  * an on-failure unit down. The daemon's updater handles CLI updates.
  */
 export function unitProcessEnv(base, unit) {
-    return { ...base, ...unit.env, HQ_DAEMON_UNIT: unit.id, HQ_NO_UPDATE_CHECK: "1" };
+    const env = { ...base, ...unit.env, HQ_DAEMON_UNIT: unit.id, HQ_NO_UPDATE_CHECK: "1" };
+    if (unit.service === "sync")
+        env.NODE_OPTIONS = syncRunnerNodeOptions(env);
+    return env;
 }
 export async function runDaemon(opts) {
     const env = opts.env ?? process.env;
@@ -289,10 +293,11 @@ export async function runDaemon(opts) {
         return legacyCache.units;
     };
     let warnedMissingPin = null;
+    const daemonSyncRunnerPids = new Set();
     const plan = () => {
         const config = readDaemonConfig(paths.configPath);
         const legacy = legacyUnits();
-        const configured = config.hqRoot ?? env.HQ_ROOT?.trim() ?? undefined;
+        const configured = config.hqRoot ?? (env.HQ_ROOT?.trim() || undefined);
         const hqRoot = configured && fs.existsSync(configured) ? configured : undefined;
         // Re-picked every reconcile, so a newer hq-cloud from the updater is used as soon as it lands.
         const hqCloud = currentHqCloud(paths.root);
@@ -304,7 +309,33 @@ export async function runDaemon(opts) {
             }
         }
         const runners = hqCloud ? hqCloudRunners(hqCloud.root) : { syncRunner: null, heartbeat: null, sessionHost: null };
-        return planUnits(enabledServices(machineType, config), {
+        const services = enabledServices(machineType, config);
+        const isSignedIn = signedIn(env, home, log);
+        const syncOwner = effectiveSyncOwner(config, machineType);
+        let syncRunnerProcesses = [];
+        let syncRunnerOwnerPid = null;
+        if (machineType === "desktop" &&
+            syncOwner === "daemon" &&
+            services.includes("sync") &&
+            !config.syncPaused &&
+            hqRoot &&
+            isSignedIn) {
+            const processList = listSyncRunnerProcesses(platform);
+            if (!processList.ok) {
+                syncRunnerProcesses = null;
+                log("warn", `${processList.error}; sync will wait before taking ownership`);
+            }
+            else {
+                syncRunnerProcesses = processList.processes;
+                try {
+                    syncRunnerOwnerPid = readWatchRunnerOwnerStatus(watchRunnerOwnerStatusPath(hqRoot, home, env))?.pid ?? null;
+                }
+                catch (error) {
+                    log("warn", `could not verify the sync watch-owner status: ${error instanceof Error ? error.message : String(error)}`);
+                }
+            }
+        }
+        return planUnits(services, {
             machineType,
             platform,
             home,
@@ -313,14 +344,20 @@ export async function runDaemon(opts) {
             heartbeatRunnerPath: runners.heartbeat,
             sessionHostRunnerPath: runners.sessionHost,
             legacySyncEnv: readLegacySyncEnv(platform),
-            signedIn: signedIn(env, home, log),
+            signedIn: isSignedIn,
             kitConfigured: fs.existsSync(agentKitPaths(home, env).kitConfigPath),
+            syncPaused: config.syncPaused === true,
+            syncOwner,
             desktopAppManagesSync: desktopAppManagesSync({
                 machineType,
                 host: opts.host,
                 legacy,
                 appRunning: () => desktopAppRunning(platform),
             }),
+            syncRunnerProcesses,
+            syncRunnerOwnerPid,
+            daemonSyncRunnerPids,
+            syncHeartbeatTimeoutMs: syncHeartbeatTimeoutMs(env),
             bots: listLocalBots(),
             legacyOwner: legacyOwnerLookup(legacy),
             cliSelfUpdatable: updatable,
@@ -342,9 +379,37 @@ export async function runDaemon(opts) {
             : command.type === "node"
                 ? [opts.nodeBinary, [command.script, ...command.args]]
                 : [command.file, command.args];
-        const fd = openPrivateAppend(paths.unitLogPath(unit.id));
+        const logPath = paths.unitLogPath(unit.id);
+        const cwd = unit.env.HQ_ROOT && fs.existsSync(unit.env.HQ_ROOT) ? unit.env.HQ_ROOT : home;
+        if (unit.heartbeatTimeoutMs) {
+            // The supervisor reads this unit's protocol heartbeats from stdout, so its
+            // output goes through the daemon. A piped child cannot outlive the daemon,
+            // so it is never handed over: a handoff stops it and the next daemon
+            // starts it again.
+            const logFd = openPrivateAppend(logPath);
+            const logStream = fs.createWriteStream(logPath, { fd: logFd, autoClose: true });
+            logStream.on("error", (err) => log("error", `could not write ${unit.id} log ${logPath}: ${err.message}`));
+            const child = spawn(file, args, {
+                cwd,
+                env: unitProcessEnv(env, unit),
+                stdio: ["ignore", "pipe", "pipe"],
+                windowsHide: true,
+            });
+            spawned.add(child);
+            if (unit.service === "sync" && child.pid !== undefined)
+                daemonSyncRunnerPids.add(child.pid);
+            child.stdout?.pipe(logStream, { end: false });
+            child.stderr?.pipe(logStream, { end: false });
+            child.on("close", () => {
+                spawned.delete(child);
+                if (child.pid !== undefined)
+                    daemonSyncRunnerPids.delete(child.pid);
+                logStream.end();
+            });
+            return child;
+        }
+        const fd = openPrivateAppend(logPath);
         try {
-            const cwd = unit.env.HQ_ROOT && fs.existsSync(unit.env.HQ_ROOT) ? unit.env.HQ_ROOT : home;
             const child = spawn(file, args, {
                 cwd,
                 env: unitProcessEnv(env, unit),
@@ -352,7 +417,13 @@ export async function runDaemon(opts) {
                 windowsHide: true,
             });
             spawned.add(child);
-            child.once("exit", () => spawned.delete(child));
+            if (unit.service === "sync" && child.pid !== undefined)
+                daemonSyncRunnerPids.add(child.pid);
+            child.once("exit", () => {
+                spawned.delete(child);
+                if (child.pid !== undefined)
+                    daemonSyncRunnerPids.delete(child.pid);
+            });
             // Only long-running services are handed over; a scheduled run left behind finishes on its own.
             if (unit.kind === "long-running")
                 tracker?.track(unit.id, child);
@@ -382,6 +453,21 @@ export async function runDaemon(opts) {
         restartDaemon: () => restartDaemon(),
         writeState: (state) => writeJsonAtomic(paths.statePath, state),
         log,
+        syncOwnerBlockReason: () => {
+            const config = readDaemonConfig(paths.configPath);
+            const hqRoot = config.hqRoot ?? env.HQ_ROOT?.trim();
+            if (!hqRoot)
+                return undefined;
+            const statusPath = watchRunnerOwnerStatusPath(hqRoot, home, env);
+            try {
+                const owner = readWatchRunnerOwnerStatus(statusPath);
+                return owner ? `sync owned by ${owner.owner} pid ${owner.pid}` : undefined;
+            }
+            catch (err) {
+                log("error", `could not read sync owner status at ${statusPath}: ${err instanceof Error ? err.message : String(err)}`);
+                return undefined;
+            }
+        },
         cliVersion: CLI_VERSION,
         adopt,
     });

package/dist/lib/daemon/supervisor.d.ts

@@ -24,6 +24,7 @@
 import type { UnitPlan } from "./catalog.js";
 import type { MachineType } from "./machine-type.js";
 import type { ControlRequest } from "./requests.js";
+import { syncPassResult } from "./sync-runner.js";
 export declare const BACKOFF_BASE_MS = 1000;
 export declare const BACKOFF_MAX_MS = 60000;
 export declare const HEALTHY_RUN_MS = 60000;
@@ -34,6 +35,9 @@ export declare const HANDOFF_STAGGER_MS = 15000;
 export declare const HANDOFF_DRAIN_MAX_MS: number;
 export interface SupervisedChild {
     pid?: number;
+    stdout?: {
+        on(event: "data" | "end", handler: (chunk?: Buffer | string) => void): unknown;
+    };
     on(event: "exit", handler: (code: number | null, signal: string | null) => void): unknown;
     on(event: "error", handler: (err: Error) => void): unknown;
     kill(signal?: NodeJS.Signals | number): boolean;
@@ -57,6 +61,8 @@ export interface UnitState {
     lastRunAt?: number;
     nextRunAt?: number;
     lastResult?: "ok" | "failed";
+    lastHeartbeatAt?: string;
+    lastPassResult?: ReturnType<typeof syncPassResult>;
 }
 /** An app that runs the daemon as its own child process instead of an OS unit. */
 export type HostApp = "desktop";
@@ -99,6 +105,8 @@ export interface SupervisorDeps {
     clearTimeout?: (handle: unknown) => void;
     reconcileMs?: number;
     requestPollMs?: number;
+    /** Read hq-cloud's lease status after it exits with the lease-conflict code. */
+    syncOwnerBlockReason?: () => string | undefined;
     /** The running CLI's version; part of the key of units that run `hq` commands. */
     cliVersion?: string;
     /** Services to take over from the previous daemon instead of starting them. */

package/dist/lib/daemon/supervisor.js

@@ -21,6 +21,8 @@
  *
  * Processes, time, and state output are injected; `run.ts` wires the real ones.
  */
+import { StringDecoder } from "node:string_decoder";
+import { SYNC_RUNNER_LEASE_CONFLICT_EXIT, SYNC_RUNNER_LEASE_RETRY_MS, parseSyncRunnerProtocolLine, syncPassResult, } from "./sync-runner.js";
 export const BACKOFF_BASE_MS = 1_000;
 export const BACKOFF_MAX_MS = 60_000;
 export const HEALTHY_RUN_MS = 60_000;
@@ -39,7 +41,15 @@ export function backoffMs(attempt) {
  * `keepAcrossCliUpdates` running.
  */
 export function planKey(plan, cliVersion) {
-    const fields = [plan.kind, plan.restart, plan.intervalSec, plan.initialDelaySec, plan.command, plan.env];
+    const fields = [
+        plan.kind,
+        plan.restart,
+        plan.intervalSec,
+        plan.initialDelaySec,
+        plan.heartbeatTimeoutMs,
+        plan.command,
+        plan.env,
+    ];
     if (cliVersion !== undefined && plan.command?.type === "hq" && !plan.keepAcrossCliUpdates)
         fields.push(cliVersion);
     return JSON.stringify(fields);
@@ -89,6 +99,11 @@ export function createSupervisor(deps) {
             clearT(slot.timer);
         slot.timer = undefined;
     };
+    const clearHeartbeatTimer = (slot) => {
+        if (slot.heartbeatTimer !== undefined)
+            clearT(slot.heartbeatTimer);
+        slot.heartbeatTimer = undefined;
+    };
     const setBlocked = (slot) => {
         const block = slot.plan.blocked;
         slot.state.status = block.state;
@@ -111,6 +126,7 @@ export function createSupervisor(deps) {
      */
     const terminate = (slot, then, drain = false) => {
         clearTimer(slot);
+        clearHeartbeatTimer(slot);
         const child = slot.child;
         if (!child) {
             then?.();
@@ -163,6 +179,69 @@ export function createSupervisor(deps) {
     const watch = (slot, child) => {
         live.add(child);
         let ended = false;
+        const timeoutMs = slot.plan.heartbeatTimeoutMs;
+        const decoder = timeoutMs ? new StringDecoder("utf8") : null;
+        let stdoutRemainder = "";
+        let lastHeartbeatPersistedAt = now() - 15_000;
+        const armHeartbeatWatchdog = () => {
+            if (!timeoutMs)
+                return;
+            clearHeartbeatTimer(slot);
+            slot.heartbeatTimer = setT(() => {
+                slot.heartbeatTimer = undefined;
+                if (stopped || slot.child !== child || slot.expectingExit)
+                    return;
+                deps.log("warn", `${slot.plan.id} emitted no sync protocol heartbeat for ${timeoutMs}ms; restarting it`);
+                terminate(slot, () => scheduleRestart(slot));
+                emit();
+            }, timeoutMs);
+        };
+        const acceptProtocolLine = (line) => {
+            let event;
+            try {
+                event = parseSyncRunnerProtocolLine(line);
+            }
+            catch (err) {
+                deps.log("error", `${slot.plan.id} emitted invalid sync protocol output (${Buffer.byteLength(line)} bytes): ${errorMessage(err)}`);
+                return;
+            }
+            if (!event)
+                return;
+            const at = now();
+            slot.state.lastHeartbeatAt = new Date(at).toISOString();
+            const pass = syncPassResult(event, slot.state.lastHeartbeatAt);
+            if (pass)
+                slot.state.lastPassResult = pass;
+            armHeartbeatWatchdog();
+            if (pass || at - lastHeartbeatPersistedAt >= 15_000) {
+                lastHeartbeatPersistedAt = at;
+                emit();
+            }
+        };
+        if (decoder && child.stdout) {
+            child.stdout.on("data", (chunk) => {
+                if (chunk === undefined)
+                    return;
+                stdoutRemainder += decoder.write(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+                let newline = stdoutRemainder.indexOf("\n");
+                while (newline >= 0) {
+                    acceptProtocolLine(stdoutRemainder.slice(0, newline).trim());
+                    stdoutRemainder = stdoutRemainder.slice(newline + 1);
+                    newline = stdoutRemainder.indexOf("\n");
+                }
+                // A malformed or unexpectedly large line is not a heartbeat. Bound
+                // retained data so a broken child cannot grow daemon memory forever.
+                if (Buffer.byteLength(stdoutRemainder) > 1024 * 1024)
+                    stdoutRemainder = "";
+            });
+            child.stdout.on("end", () => {
+                stdoutRemainder += decoder.end();
+                if (stdoutRemainder)
+                    acceptProtocolLine(stdoutRemainder.trim());
+                stdoutRemainder = "";
+            });
+        }
+        armHeartbeatWatchdog();
         const onEnd = (code, signal) => {
             if (ended)
                 return;
@@ -173,6 +252,7 @@ export function createSupervisor(deps) {
             if (slot.child !== child)
                 return;
             slot.child = undefined;
+            clearHeartbeatTimer(slot);
             if (slot.killTimer !== undefined) {
                 clearT(slot.killTimer);
                 slot.killTimer = undefined;
@@ -200,6 +280,20 @@ export function createSupervisor(deps) {
                 emit();
                 return;
             }
+            if (slot.plan.service === "sync" && code === SYNC_RUNNER_LEASE_CONFLICT_EXIT) {
+                const reason = deps.syncOwnerBlockReason?.() ?? "sync owned by another runner";
+                slot.state.status = "waiting";
+                slot.state.reason = reason;
+                slot.state.nextRunAt = now() + SYNC_RUNNER_LEASE_RETRY_MS;
+                deps.log("info", `${reason}; retrying in ${SYNC_RUNNER_LEASE_RETRY_MS}ms`);
+                slot.timer = setT(() => {
+                    slot.timer = undefined;
+                    startLong(slot);
+                    emit();
+                }, SYNC_RUNNER_LEASE_RETRY_MS);
+                emit();
+                return;
+            }
             deps.log("warn", `${slot.plan.id} exited (code=${code ?? "null"} signal=${signal ?? "none"})`);
             if (ranFor >= HEALTHY_RUN_MS)
                 slot.attempts = 0;
@@ -505,6 +599,11 @@ export function createSupervisor(deps) {
         }
     };
     const handleRequest = (req) => {
+        if (req.action === "reconcile") {
+            deps.log("info", "reconciling daemon configuration (requested)");
+            reconcile();
+            return;
+        }
         if (req.action === "restart-daemon") {
             deps.log("info", "restarting the daemon (requested)");
             if (deps.restartDaemon)
@@ -614,6 +713,7 @@ export function createSupervisor(deps) {
                 clearT(handoffTimer);
             for (const slot of slots.values()) {
                 clearTimer(slot);
+                clearHeartbeatTimer(slot);
                 slot.child = undefined;
                 if (slot.killTimer !== undefined)
                     clearT(slot.killTimer);

package/dist/lib/daemon/sync-runner.d.ts

@@ -0,0 +1,58 @@
+/** Local status and resource controls shared by the daemon and its sync runner. */
+export declare const SYNC_RUNNER_OWNER = "hq-daemon";
+export declare const SYNC_RUNNER_LEASE_CONFLICT_EXIT = 20;
+export declare const SYNC_RUNNER_LEASE_RETRY_MS = 15000;
+export declare const DEFAULT_SYNC_HEARTBEAT_TIMEOUT_MS = 300000;
+export declare const DEFAULT_SYNC_RUNNER_HEAP_MB = 3584;
+export declare const SYNC_HEARTBEAT_TIMEOUT_ENV = "HQ_DAEMON_SYNC_HEARTBEAT_TIMEOUT_SECONDS";
+export declare const SYNC_RUNNER_HEAP_ENV = "HQ_SYNC_RUNNER_MAX_OLD_SPACE_MB";
+export interface WatchRunnerOwnerStatus {
+    owner: string;
+    pid: number;
+    version: string;
+    startedAt: string;
+    heartbeatAt: string;
+}
+export interface SyncRunnerProcess {
+    pid: number;
+    commandLine: string;
+}
+export type SyncRunnerProcessList = {
+    ok: true;
+    processes: SyncRunnerProcess[];
+} | {
+    ok: false;
+    error: string;
+};
+export interface ProcessListCommandResult {
+    status: number | null;
+    stdout: string;
+    stderr: string;
+    error?: Error;
+}
+export type ProcessListRunner = (command: string, args: string[]) => ProcessListCommandResult;
+export interface SyncPassResult {
+    status: "ok" | "partial" | "failed";
+    completedAt: string;
+    companiesAttempted?: number;
+    errors: number;
+    transient: number;
+}
+/** The status file path used by hq-cloud's watch-owner lease. */
+export declare function watchRunnerOwnerStatusPath(hqRoot: string, home?: string, env?: NodeJS.ProcessEnv): string;
+/** Read the watch-owner status JSON without taking or changing its lease. */
+export declare function readWatchRunnerOwnerStatus(statusPath: string): WatchRunnerOwnerStatus | null;
+/** Match a live process command line to the hq-cloud watch runner executable. */
+export declare function isSyncRunnerCommandLine(commandLine: string): boolean;
+/** Return a clear wait reason when an unleased runner already targets this HQ root. */
+export declare function syncRunnerUnleasedBlockReason(hqRoot: string, processes: readonly SyncRunnerProcess[], ownerPid: number | null, platform?: NodeJS.Platform, daemonRunnerPids?: ReadonlySet<number>): string | undefined;
+/** Read complete process command lines so root matching can include trailing arguments. */
+export declare function listSyncRunnerProcesses(platform?: NodeJS.Platform, run?: ProcessListRunner): SyncRunnerProcessList;
+/** The watchdog follows the runner's NDJSON protocol, the same liveness signal used by the desktop app. */
+export declare function parseSyncRunnerProtocolLine(line: string): Record<string, unknown> | null;
+/** Summarize the runner's all-complete event for daemon status. */
+export declare function syncPassResult(event: Record<string, unknown>, completedAt?: string): SyncPassResult | null;
+export declare function syncHeartbeatTimeoutMs(env: NodeJS.ProcessEnv): number;
+/** Preserve an explicit NODE_OPTIONS heap setting; otherwise apply the desktop runner's default/override. */
+export declare function syncRunnerNodeOptions(env: NodeJS.ProcessEnv): string | undefined;
+//# sourceMappingURL=sync-runner.d.ts.map