@indigoai-us/hq-cli 5.272.0 → 5.273.0-alpha.0

package/dist/lib/daemon/config.d.ts

@@ -12,11 +12,17 @@ export interface DaemonConfig {
     hqRoot?: string;
     /** Exact hq-cloud package version preferred by this daemon, including prereleases. */
     hqCloudVersion?: string;
+    /** Persisted machine-local sync pause. */
+    syncPaused?: boolean;
+    /** Explicit desktop handoff; absent values preserve the legacy per-machine owner. */
+    syncOwner?: SyncOwner;
     /** Services turned on here beyond the machine type's defaults. */
     enable: string[];
     /** Default services turned off here. */
     disable: string[];
 }
+export type SyncOwner = "daemon" | "desktop-app";
+export declare function effectiveSyncOwner(config: Pick<DaemonConfig, "syncOwner">, machineType: MachineType): SyncOwner;
 export declare function emptyDaemonConfig(): DaemonConfig;
 export declare function readDaemonConfig(file: string): DaemonConfig;
 export declare function writeDaemonConfig(file: string, config: DaemonConfig): void;

package/dist/lib/daemon/config.js

@@ -10,8 +10,11 @@ import { valid as validSemver } from "semver";
 import { isServiceName } from "./catalog.js";
 import { isMachineType } from "./machine-type.js";
 import { ensurePrivateDir } from "./paths.js";
+export function effectiveSyncOwner(config, machineType) {
+    return config.syncOwner ?? (machineType === "desktop" ? "desktop-app" : "daemon");
+}
 export function emptyDaemonConfig() {
-    return { version: 1, enable: [], disable: [] };
+    return { version: 1, syncPaused: false, enable: [], disable: [] };
 }
 function stringList(value) {
     return Array.isArray(value) ? value.filter((v) => typeof v === "string") : [];
@@ -36,9 +39,16 @@ export function readDaemonConfig(file) {
     if (!raw || typeof raw !== "object")
         throw new Error(`hq daemon config at ${file} is not a JSON object`);
     const obj = raw;
-    const config = { version: 1, enable: stringList(obj.enable), disable: stringList(obj.disable) };
+    const config = {
+        version: 1,
+        syncPaused: obj.syncPaused === true,
+        enable: stringList(obj.enable),
+        disable: stringList(obj.disable),
+    };
     if (isMachineType(obj.machineType))
         config.machineType = obj.machineType;
+    if (obj.syncOwner === "daemon" || obj.syncOwner === "desktop-app")
+        config.syncOwner = obj.syncOwner;
     if (typeof obj.hqRoot === "string" && obj.hqRoot.trim())
         config.hqRoot = obj.hqRoot;
     if (typeof obj.hqCloudVersion === "string" && validSemver(obj.hqCloudVersion) === obj.hqCloudVersion) {
@@ -56,6 +66,9 @@ export function writeDaemonConfig(file, config) {
         ordered.hqRoot = config.hqRoot;
     if (config.hqCloudVersion)
         ordered.hqCloudVersion = config.hqCloudVersion;
+    ordered.syncPaused = config.syncPaused === true;
+    if (config.syncOwner)
+        ordered.syncOwner = config.syncOwner;
     ordered.enable = config.enable;
     ordered.disable = config.disable;
     fs.writeFileSync(tmp, `${JSON.stringify(ordered, null, 2)}\n`, { mode: 0o600 });

package/dist/lib/daemon/requests.d.ts

@@ -15,6 +15,10 @@ export type ControlRequest = {
     action: "stop";
     unit: string;
 }
+/** Re-read persisted configuration and apply any changed blocks immediately. */
+ | {
+    action: "reconcile";
+}
 /** Restart the whole daemon, for example after the updater installed a new CLI. */
  | {
     action: "restart-daemon";

package/dist/lib/daemon/requests.js

@@ -19,6 +19,8 @@ function isControlRequest(value) {
         return typeof v.unit === "string";
     if (v.action === "restart-daemon")
         return true;
+    if (v.action === "reconcile")
+        return true;
     return false;
 }
 // Orders requests written by one process within the same millisecond; the

package/dist/lib/daemon/run.js

@@ -16,7 +16,7 @@ import { agentKitPaths } from "../agent-kit/paths.js";
 import { readBotConfig } from "../bot/config.js";
 import { botsRoot } from "../bot/paths.js";
 import { CLAUDE_RUNNER_SCRIPT, CODEX_RUNNER_SCRIPT, enabledServices, planUnits } from "./catalog.js";
-import { readDaemonConfig } from "./config.js";
+import { effectiveSyncOwner, readDaemonConfig } from "./config.js";
 import { findLegacyUnits, legacyOwnerLookup } from "./legacy.js";
 import { readLegacySyncEnv } from "./legacy-sync-env.js";
 import { acquireDaemonLock, releaseDaemonLock } from "./lock.js";
@@ -26,6 +26,7 @@ import { appendPrivate, daemonPaths, ensurePrivateDir, openPrivateAppend } from
 import { takeControlRequests } from "./requests.js";
 import { currentHqCloud, hqCloudRunners } from "./hq-cloud.js";
 import { runnerSessionName } from "./remote-control.js";
+import { listSyncRunnerProcesses, readWatchRunnerOwnerStatus, syncHeartbeatTimeoutMs, syncRunnerNodeOptions, watchRunnerOwnerStatusPath, } from "./sync-runner.js";
 import { STOP_ESCALATION_MS, createSupervisor } from "./supervisor.js";
 import { userToolDirs } from "./tools.js";
 export const LOG_ROTATE_BYTES = 5 * 1024 * 1024;
@@ -213,7 +214,10 @@ function makeLogger(paths, foreground) {
  * an on-failure unit down. The daemon's updater handles CLI updates.
  */
 export function unitProcessEnv(base, unit) {
-    return { ...base, ...unit.env, HQ_DAEMON_UNIT: unit.id, HQ_NO_UPDATE_CHECK: "1" };
+    const env = { ...base, ...unit.env, HQ_DAEMON_UNIT: unit.id, HQ_NO_UPDATE_CHECK: "1" };
+    if (unit.service === "sync")
+        env.NODE_OPTIONS = syncRunnerNodeOptions(env);
+    return env;
 }
 export async function runDaemon(opts) {
     const env = opts.env ?? process.env;
@@ -258,10 +262,11 @@ export async function runDaemon(opts) {
         return legacyCache.units;
     };
     let warnedMissingPin = null;
+    const daemonSyncRunnerPids = new Set();
     const plan = () => {
         const config = readDaemonConfig(paths.configPath);
         const legacy = legacyUnits();
-        const configured = config.hqRoot ?? env.HQ_ROOT?.trim() ?? undefined;
+        const configured = config.hqRoot ?? (env.HQ_ROOT?.trim() || undefined);
         const hqRoot = configured && fs.existsSync(configured) ? configured : undefined;
         // Re-picked every reconcile, so a newer hq-cloud from the updater is used as soon as it lands.
         const hqCloud = currentHqCloud(paths.root);
@@ -273,7 +278,33 @@ export async function runDaemon(opts) {
             }
         }
         const runners = hqCloud ? hqCloudRunners(hqCloud.root) : { syncRunner: null, heartbeat: null, sessionHost: null };
-        return planUnits(enabledServices(machineType, config), {
+        const services = enabledServices(machineType, config);
+        const isSignedIn = signedIn(env, home, log);
+        const syncOwner = effectiveSyncOwner(config, machineType);
+        let syncRunnerProcesses = [];
+        let syncRunnerOwnerPid = null;
+        if (machineType === "desktop" &&
+            syncOwner === "daemon" &&
+            services.includes("sync") &&
+            !config.syncPaused &&
+            hqRoot &&
+            isSignedIn) {
+            const processList = listSyncRunnerProcesses(platform);
+            if (!processList.ok) {
+                syncRunnerProcesses = null;
+                log("warn", `${processList.error}; sync will wait before taking ownership`);
+            }
+            else {
+                syncRunnerProcesses = processList.processes;
+                try {
+                    syncRunnerOwnerPid = readWatchRunnerOwnerStatus(watchRunnerOwnerStatusPath(hqRoot, home, env))?.pid ?? null;
+                }
+                catch (error) {
+                    log("warn", `could not verify the sync watch-owner status: ${error instanceof Error ? error.message : String(error)}`);
+                }
+            }
+        }
+        return planUnits(services, {
             machineType,
             platform,
             home,
@@ -282,14 +313,20 @@ export async function runDaemon(opts) {
             heartbeatRunnerPath: runners.heartbeat,
             sessionHostRunnerPath: runners.sessionHost,
             legacySyncEnv: readLegacySyncEnv(platform),
-            signedIn: signedIn(env, home, log),
+            signedIn: isSignedIn,
             kitConfigured: fs.existsSync(agentKitPaths(home, env).kitConfigPath),
+            syncPaused: config.syncPaused === true,
+            syncOwner,
             desktopAppManagesSync: desktopAppManagesSync({
                 machineType,
                 host: opts.host,
                 legacy,
                 appRunning: () => desktopAppRunning(platform),
             }),
+            syncRunnerProcesses,
+            syncRunnerOwnerPid,
+            daemonSyncRunnerPids,
+            syncHeartbeatTimeoutMs: syncHeartbeatTimeoutMs(env),
             bots: listLocalBots(),
             legacyOwner: legacyOwnerLookup(legacy),
             cliSelfUpdatable: updatable,
@@ -311,20 +348,27 @@ export async function runDaemon(opts) {
             : command.type === "node"
                 ? [opts.nodeBinary, [command.script, ...command.args]]
                 : [command.file, command.args];
-        const fd = openPrivateAppend(paths.unitLogPath(unit.id));
-        try {
-            const cwd = unit.env.HQ_ROOT && fs.existsSync(unit.env.HQ_ROOT) ? unit.env.HQ_ROOT : home;
-            const child = spawn(file, args, {
-                cwd,
-                env: unitProcessEnv(env, unit),
-                stdio: ["ignore", fd, fd],
-                windowsHide: true,
-            });
-            return child;
-        }
-        finally {
-            fs.closeSync(fd);
-        }
+        const logPath = paths.unitLogPath(unit.id);
+        const logFd = openPrivateAppend(logPath);
+        const logStream = fs.createWriteStream(logPath, { fd: logFd, autoClose: true });
+        logStream.on("error", (err) => log("error", `could not write ${unit.id} log ${logPath}: ${err.message}`));
+        const cwd = unit.env.HQ_ROOT && fs.existsSync(unit.env.HQ_ROOT) ? unit.env.HQ_ROOT : home;
+        const child = spawn(file, args, {
+            cwd,
+            env: unitProcessEnv(env, unit),
+            stdio: ["ignore", "pipe", "pipe"],
+            windowsHide: true,
+        });
+        if (unit.service === "sync" && child.pid !== undefined)
+            daemonSyncRunnerPids.add(child.pid);
+        child.stdout?.pipe(logStream, { end: false });
+        child.stderr?.pipe(logStream, { end: false });
+        child.on("close", () => {
+            if (child.pid !== undefined)
+                daemonSyncRunnerPids.delete(child.pid);
+            logStream.end();
+        });
+        return child;
     };
     // Set once the shutdown path exists below; a restart request before then is dropped.
     let restartDaemon = () => { };
@@ -346,6 +390,21 @@ export async function runDaemon(opts) {
         restartDaemon: () => restartDaemon(),
         writeState: (state) => writeJsonAtomic(paths.statePath, state),
         log,
+        syncOwnerBlockReason: () => {
+            const config = readDaemonConfig(paths.configPath);
+            const hqRoot = config.hqRoot ?? env.HQ_ROOT?.trim();
+            if (!hqRoot)
+                return undefined;
+            const statusPath = watchRunnerOwnerStatusPath(hqRoot, home, env);
+            try {
+                const owner = readWatchRunnerOwnerStatus(statusPath);
+                return owner ? `sync owned by ${owner.owner} pid ${owner.pid}` : undefined;
+            }
+            catch (err) {
+                log("error", `could not read sync owner status at ${statusPath}: ${err instanceof Error ? err.message : String(err)}`);
+                return undefined;
+            }
+        },
     });
     return await new Promise((resolve) => {
         let stopping = false;

package/dist/lib/daemon/supervisor.d.ts

@@ -20,12 +20,16 @@
 import type { UnitPlan } from "./catalog.js";
 import type { MachineType } from "./machine-type.js";
 import type { ControlRequest } from "./requests.js";
+import { syncPassResult } from "./sync-runner.js";
 export declare const BACKOFF_BASE_MS = 1000;
 export declare const BACKOFF_MAX_MS = 60000;
 export declare const HEALTHY_RUN_MS = 60000;
 export declare const STOP_ESCALATION_MS = 10000;
 export interface SupervisedChild {
     pid?: number;
+    stdout?: {
+        on(event: "data" | "end", handler: (chunk?: Buffer | string) => void): unknown;
+    };
     on(event: "exit", handler: (code: number | null, signal: string | null) => void): unknown;
     on(event: "error", handler: (err: Error) => void): unknown;
     kill(signal?: NodeJS.Signals | number): boolean;
@@ -49,6 +53,8 @@ export interface UnitState {
     lastRunAt?: number;
     nextRunAt?: number;
     lastResult?: "ok" | "failed";
+    lastHeartbeatAt?: string;
+    lastPassResult?: ReturnType<typeof syncPassResult>;
 }
 /** An app that runs the daemon as its own child process instead of an OS unit. */
 export type HostApp = "desktop";
@@ -79,6 +85,8 @@ export interface SupervisorDeps {
     clearTimeout?: (handle: unknown) => void;
     reconcileMs?: number;
     requestPollMs?: number;
+    /** Read hq-cloud's lease status after it exits with the lease-conflict code. */
+    syncOwnerBlockReason?: () => string | undefined;
 }
 export interface Supervisor {
     start(): void;

package/dist/lib/daemon/supervisor.js

@@ -17,6 +17,8 @@
  *
  * Processes, time, and state output are injected; `run.ts` wires the real ones.
  */
+import { StringDecoder } from "node:string_decoder";
+import { SYNC_RUNNER_LEASE_CONFLICT_EXIT, SYNC_RUNNER_LEASE_RETRY_MS, parseSyncRunnerProtocolLine, syncPassResult, } from "./sync-runner.js";
 export const BACKOFF_BASE_MS = 1_000;
 export const BACKOFF_MAX_MS = 60_000;
 export const HEALTHY_RUN_MS = 60_000;
@@ -25,7 +27,15 @@ export function backoffMs(attempt) {
     return Math.min(BACKOFF_MAX_MS, BACKOFF_BASE_MS * 2 ** Math.max(0, attempt));
 }
 function planKey(plan) {
-    return JSON.stringify([plan.kind, plan.restart, plan.intervalSec, plan.initialDelaySec, plan.command, plan.env]);
+    return JSON.stringify([
+        plan.kind,
+        plan.restart,
+        plan.intervalSec,
+        plan.initialDelaySec,
+        plan.heartbeatTimeoutMs,
+        plan.command,
+        plan.env,
+    ]);
 }
 function errorMessage(err) {
     return err instanceof Error ? err.message : String(err);
@@ -68,6 +78,11 @@ export function createSupervisor(deps) {
             clearT(slot.timer);
         slot.timer = undefined;
     };
+    const clearHeartbeatTimer = (slot) => {
+        if (slot.heartbeatTimer !== undefined)
+            clearT(slot.heartbeatTimer);
+        slot.heartbeatTimer = undefined;
+    };
     const setBlocked = (slot) => {
         const block = slot.plan.blocked;
         slot.state.status = block.state;
@@ -78,6 +93,7 @@ export function createSupervisor(deps) {
     /** Stop the slot's child (if any), then run `then` once it has exited. */
     const terminate = (slot, then) => {
         clearTimer(slot);
+        clearHeartbeatTimer(slot);
         const child = slot.child;
         if (!child) {
             then?.();
@@ -122,6 +138,69 @@ export function createSupervisor(deps) {
     const watch = (slot, child) => {
         live.add(child);
         let ended = false;
+        const timeoutMs = slot.plan.heartbeatTimeoutMs;
+        const decoder = timeoutMs ? new StringDecoder("utf8") : null;
+        let stdoutRemainder = "";
+        let lastHeartbeatPersistedAt = now() - 15_000;
+        const armHeartbeatWatchdog = () => {
+            if (!timeoutMs)
+                return;
+            clearHeartbeatTimer(slot);
+            slot.heartbeatTimer = setT(() => {
+                slot.heartbeatTimer = undefined;
+                if (stopped || slot.child !== child || slot.expectingExit)
+                    return;
+                deps.log("warn", `${slot.plan.id} emitted no sync protocol heartbeat for ${timeoutMs}ms; restarting it`);
+                terminate(slot, () => scheduleRestart(slot));
+                emit();
+            }, timeoutMs);
+        };
+        const acceptProtocolLine = (line) => {
+            let event;
+            try {
+                event = parseSyncRunnerProtocolLine(line);
+            }
+            catch (err) {
+                deps.log("error", `${slot.plan.id} emitted invalid sync protocol output (${Buffer.byteLength(line)} bytes): ${errorMessage(err)}`);
+                return;
+            }
+            if (!event)
+                return;
+            const at = now();
+            slot.state.lastHeartbeatAt = new Date(at).toISOString();
+            const pass = syncPassResult(event, slot.state.lastHeartbeatAt);
+            if (pass)
+                slot.state.lastPassResult = pass;
+            armHeartbeatWatchdog();
+            if (pass || at - lastHeartbeatPersistedAt >= 15_000) {
+                lastHeartbeatPersistedAt = at;
+                emit();
+            }
+        };
+        if (decoder && child.stdout) {
+            child.stdout.on("data", (chunk) => {
+                if (chunk === undefined)
+                    return;
+                stdoutRemainder += decoder.write(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+                let newline = stdoutRemainder.indexOf("\n");
+                while (newline >= 0) {
+                    acceptProtocolLine(stdoutRemainder.slice(0, newline).trim());
+                    stdoutRemainder = stdoutRemainder.slice(newline + 1);
+                    newline = stdoutRemainder.indexOf("\n");
+                }
+                // A malformed or unexpectedly large line is not a heartbeat. Bound
+                // retained data so a broken child cannot grow daemon memory forever.
+                if (Buffer.byteLength(stdoutRemainder) > 1024 * 1024)
+                    stdoutRemainder = "";
+            });
+            child.stdout.on("end", () => {
+                stdoutRemainder += decoder.end();
+                if (stdoutRemainder)
+                    acceptProtocolLine(stdoutRemainder.trim());
+                stdoutRemainder = "";
+            });
+        }
+        armHeartbeatWatchdog();
         const onEnd = (code, signal) => {
             if (ended)
                 return;
@@ -132,6 +211,7 @@ export function createSupervisor(deps) {
             if (slot.child !== child)
                 return;
             slot.child = undefined;
+            clearHeartbeatTimer(slot);
             if (slot.killTimer !== undefined) {
                 clearT(slot.killTimer);
                 slot.killTimer = undefined;
@@ -159,6 +239,20 @@ export function createSupervisor(deps) {
                 emit();
                 return;
             }
+            if (slot.plan.service === "sync" && code === SYNC_RUNNER_LEASE_CONFLICT_EXIT) {
+                const reason = deps.syncOwnerBlockReason?.() ?? "sync owned by another runner";
+                slot.state.status = "waiting";
+                slot.state.reason = reason;
+                slot.state.nextRunAt = now() + SYNC_RUNNER_LEASE_RETRY_MS;
+                deps.log("info", `${reason}; retrying in ${SYNC_RUNNER_LEASE_RETRY_MS}ms`);
+                slot.timer = setT(() => {
+                    slot.timer = undefined;
+                    startLong(slot);
+                    emit();
+                }, SYNC_RUNNER_LEASE_RETRY_MS);
+                emit();
+                return;
+            }
             deps.log("warn", `${slot.plan.id} exited (code=${code ?? "null"} signal=${signal ?? "none"})`);
             if (ranFor >= HEALTHY_RUN_MS)
                 slot.attempts = 0;
@@ -369,6 +463,11 @@ export function createSupervisor(deps) {
         emit();
     };
     const handleRequest = (req) => {
+        if (req.action === "reconcile") {
+            deps.log("info", "reconciling daemon configuration (requested)");
+            reconcile();
+            return;
+        }
         if (req.action === "restart-daemon") {
             deps.log("info", "restarting the daemon (requested)");
             if (deps.restartDaemon)
@@ -475,6 +574,7 @@ export function createSupervisor(deps) {
                 clearT(requestTimer);
             for (const slot of slots.values()) {
                 clearTimer(slot);
+                clearHeartbeatTimer(slot);
                 slot.child = undefined;
                 if (slot.killTimer !== undefined)
                     clearT(slot.killTimer);

package/dist/lib/daemon/sync-runner.d.ts

@@ -0,0 +1,58 @@
+/** Local status and resource controls shared by the daemon and its sync runner. */
+export declare const SYNC_RUNNER_OWNER = "hq-daemon";
+export declare const SYNC_RUNNER_LEASE_CONFLICT_EXIT = 20;
+export declare const SYNC_RUNNER_LEASE_RETRY_MS = 15000;
+export declare const DEFAULT_SYNC_HEARTBEAT_TIMEOUT_MS = 300000;
+export declare const DEFAULT_SYNC_RUNNER_HEAP_MB = 3584;
+export declare const SYNC_HEARTBEAT_TIMEOUT_ENV = "HQ_DAEMON_SYNC_HEARTBEAT_TIMEOUT_SECONDS";
+export declare const SYNC_RUNNER_HEAP_ENV = "HQ_SYNC_RUNNER_MAX_OLD_SPACE_MB";
+export interface WatchRunnerOwnerStatus {
+    owner: string;
+    pid: number;
+    version: string;
+    startedAt: string;
+    heartbeatAt: string;
+}
+export interface SyncRunnerProcess {
+    pid: number;
+    commandLine: string;
+}
+export type SyncRunnerProcessList = {
+    ok: true;
+    processes: SyncRunnerProcess[];
+} | {
+    ok: false;
+    error: string;
+};
+export interface ProcessListCommandResult {
+    status: number | null;
+    stdout: string;
+    stderr: string;
+    error?: Error;
+}
+export type ProcessListRunner = (command: string, args: string[]) => ProcessListCommandResult;
+export interface SyncPassResult {
+    status: "ok" | "partial" | "failed";
+    completedAt: string;
+    companiesAttempted?: number;
+    errors: number;
+    transient: number;
+}
+/** The status file path used by hq-cloud's watch-owner lease. */
+export declare function watchRunnerOwnerStatusPath(hqRoot: string, home?: string, env?: NodeJS.ProcessEnv): string;
+/** Read the watch-owner status JSON without taking or changing its lease. */
+export declare function readWatchRunnerOwnerStatus(statusPath: string): WatchRunnerOwnerStatus | null;
+/** Match a live process command line to the hq-cloud watch runner executable. */
+export declare function isSyncRunnerCommandLine(commandLine: string): boolean;
+/** Return a clear wait reason when an unleased runner already targets this HQ root. */
+export declare function syncRunnerUnleasedBlockReason(hqRoot: string, processes: readonly SyncRunnerProcess[], ownerPid: number | null, platform?: NodeJS.Platform, daemonRunnerPids?: ReadonlySet<number>): string | undefined;
+/** Read complete process command lines so root matching can include trailing arguments. */
+export declare function listSyncRunnerProcesses(platform?: NodeJS.Platform, run?: ProcessListRunner): SyncRunnerProcessList;
+/** The watchdog follows the runner's NDJSON protocol, the same liveness signal used by the desktop app. */
+export declare function parseSyncRunnerProtocolLine(line: string): Record<string, unknown> | null;
+/** Summarize the runner's all-complete event for daemon status. */
+export declare function syncPassResult(event: Record<string, unknown>, completedAt?: string): SyncPassResult | null;
+export declare function syncHeartbeatTimeoutMs(env: NodeJS.ProcessEnv): number;
+/** Preserve an explicit NODE_OPTIONS heap setting; otherwise apply the desktop runner's default/override. */
+export declare function syncRunnerNodeOptions(env: NodeJS.ProcessEnv): string | undefined;
+//# sourceMappingURL=sync-runner.d.ts.map