@indigoai-us/hq-cli 5.162.0 → 5.163.0

package/dist/lib/daemon/supervisor.d.ts

@@ -0,0 +1,87 @@
+/**
+ * The `hq daemon run` supervisor. It keeps the planned units running:
+ *
+ * - long-running units are restarted after they exit, with per-unit backoff
+ *   (1 s doubling to 60 s, reset once a unit has stayed up for a minute);
+ *   `on-failure` units stay down after a clean exit or a SIGTERM/SIGINT from
+ *   outside, and any unit stays down after a stop request (`hq bot stop`),
+ *   until a restart request;
+ * - scheduled units run after their initial delay and then on their interval,
+ *   never overlapping themselves;
+ * - the plan is re-read on a timer, so disabled services stop, new bots start,
+ *   and units whose requirement appears or disappears follow on their own;
+ * - restart/trigger/stop requests from the CLI are polled and applied; a
+ *   request for a unit that is not planned yet re-reads the plan first;
+ * - stop() sends SIGTERM to every child, escalates to SIGKILL after
+ *   STOP_ESCALATION_MS, and resolves once every child has exited.
+ *
+ * Processes, time, and state output are injected; `run.ts` wires the real ones.
+ */
+import type { UnitPlan } from "./catalog.js";
+import type { MachineType } from "./machine-type.js";
+import type { ControlRequest } from "./requests.js";
+export declare const BACKOFF_BASE_MS = 1000;
+export declare const BACKOFF_MAX_MS = 60000;
+export declare const HEALTHY_RUN_MS = 60000;
+export declare const STOP_ESCALATION_MS = 10000;
+export interface SupervisedChild {
+    pid?: number;
+    on(event: "exit", handler: (code: number | null, signal: string | null) => void): unknown;
+    on(event: "error", handler: (err: Error) => void): unknown;
+    kill(signal?: NodeJS.Signals | number): boolean;
+}
+export type UnitStatus = "running" | "waiting" | "unavailable" | "backoff" | "exited" | "idle" | "stopped";
+export interface UnitState {
+    id: string;
+    service: string;
+    kind: UnitPlan["kind"];
+    description: string;
+    status: UnitStatus;
+    reason?: string;
+    pid?: number;
+    startedAt?: number;
+    restarts: number;
+    lastExit?: {
+        code: number | null;
+        signal: string | null;
+        at: number;
+    };
+    lastRunAt?: number;
+    nextRunAt?: number;
+    lastResult?: "ok" | "failed";
+}
+export interface DaemonState {
+    version: 1;
+    pid: number;
+    startedAt: number;
+    updatedAt: number;
+    machineType: MachineType;
+    units: UnitState[];
+}
+export interface SupervisorDeps {
+    machineType: MachineType;
+    pid: number;
+    plan: () => UnitPlan[];
+    spawn: (unit: UnitPlan) => SupervisedChild;
+    runInternal: (task: string) => Promise<boolean>;
+    takeRequests: () => ControlRequest[];
+    /** Restart the whole daemon process (a `restart-daemon` request). */
+    restartDaemon?: () => void;
+    writeState: (state: DaemonState) => void;
+    log: (level: "info" | "warn" | "error", msg: string) => void;
+    now?: () => number;
+    setTimeout?: (fn: () => void, ms: number) => unknown;
+    clearTimeout?: (handle: unknown) => void;
+    reconcileMs?: number;
+    requestPollMs?: number;
+}
+export interface Supervisor {
+    start(): void;
+    /** Resolves once every child process has exited. */
+    stop(): Promise<void>;
+    reconcile(): void;
+    state(): DaemonState;
+}
+export declare function backoffMs(attempt: number): number;
+export declare function createSupervisor(deps: SupervisorDeps): Supervisor;
+//# sourceMappingURL=supervisor.d.ts.map

package/dist/lib/daemon/supervisor.js

@@ -0,0 +1,523 @@
+/**
+ * The `hq daemon run` supervisor. It keeps the planned units running:
+ *
+ * - long-running units are restarted after they exit, with per-unit backoff
+ *   (1 s doubling to 60 s, reset once a unit has stayed up for a minute);
+ *   `on-failure` units stay down after a clean exit or a SIGTERM/SIGINT from
+ *   outside, and any unit stays down after a stop request (`hq bot stop`),
+ *   until a restart request;
+ * - scheduled units run after their initial delay and then on their interval,
+ *   never overlapping themselves;
+ * - the plan is re-read on a timer, so disabled services stop, new bots start,
+ *   and units whose requirement appears or disappears follow on their own;
+ * - restart/trigger/stop requests from the CLI are polled and applied; a
+ *   request for a unit that is not planned yet re-reads the plan first;
+ * - stop() sends SIGTERM to every child, escalates to SIGKILL after
+ *   STOP_ESCALATION_MS, and resolves once every child has exited.
+ *
+ * Processes, time, and state output are injected; `run.ts` wires the real ones.
+ */
+export const BACKOFF_BASE_MS = 1_000;
+export const BACKOFF_MAX_MS = 60_000;
+export const HEALTHY_RUN_MS = 60_000;
+export const STOP_ESCALATION_MS = 10_000;
+export function backoffMs(attempt) {
+    return Math.min(BACKOFF_MAX_MS, BACKOFF_BASE_MS * 2 ** Math.max(0, attempt));
+}
+function planKey(plan) {
+    return JSON.stringify([plan.kind, plan.restart, plan.intervalSec, plan.initialDelaySec, plan.command, plan.env]);
+}
+function errorMessage(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+export function createSupervisor(deps) {
+    const now = deps.now ?? Date.now;
+    const setT = deps.setTimeout ?? ((fn, ms) => setTimeout(fn, ms));
+    const clearT = deps.clearTimeout ?? ((h) => clearTimeout(h));
+    const reconcileMs = deps.reconcileMs ?? 15_000;
+    const requestPollMs = deps.requestPollMs ?? 1_000;
+    const startedAt = now();
+    let slots = new Map();
+    let stopped = false;
+    let started = false;
+    let reconcileTimer;
+    let requestTimer;
+    /** Every child process not yet known to have exited, including ones the slots have let go of. */
+    const live = new Set();
+    let drained = null;
+    let onDrained = null;
+    const snapshot = () => ({
+        version: 1,
+        pid: deps.pid,
+        startedAt,
+        updatedAt: now(),
+        machineType: deps.machineType,
+        units: [...slots.values()].map((s) => ({ ...s.state })),
+    });
+    const emit = () => {
+        try {
+            deps.writeState(snapshot());
+        }
+        catch (err) {
+            deps.log("error", `could not write daemon state: ${errorMessage(err)}`);
+        }
+    };
+    const clearTimer = (slot) => {
+        if (slot.timer !== undefined)
+            clearT(slot.timer);
+        slot.timer = undefined;
+    };
+    const setBlocked = (slot) => {
+        const block = slot.plan.blocked;
+        slot.state.status = block.state;
+        slot.state.reason = block.reason;
+        delete slot.state.pid;
+        delete slot.state.nextRunAt;
+    };
+    /** Stop the slot's child (if any), then run `then` once it has exited. */
+    const terminate = (slot, then) => {
+        clearTimer(slot);
+        const child = slot.child;
+        if (!child) {
+            then?.();
+            return;
+        }
+        slot.expectingExit = true;
+        slot.afterExit = then;
+        slot.killTimer = setT(() => {
+            slot.killTimer = undefined;
+            if (slot.child === child) {
+                deps.log("warn", `${slot.plan.id} ignored SIGTERM; sending SIGKILL`);
+                try {
+                    child.kill("SIGKILL");
+                }
+                catch (err) {
+                    deps.log("error", `could not SIGKILL ${slot.plan.id}: ${errorMessage(err)}`);
+                }
+            }
+        }, STOP_ESCALATION_MS);
+        try {
+            child.kill("SIGTERM");
+        }
+        catch (err) {
+            deps.log("error", `could not stop ${slot.plan.id}: ${errorMessage(err)}`);
+        }
+    };
+    const scheduleRestart = (slot) => {
+        if (stopped)
+            return;
+        const delay = backoffMs(slot.attempts);
+        slot.attempts += 1;
+        slot.state.restarts += 1;
+        slot.state.status = "backoff";
+        slot.state.nextRunAt = now() + delay;
+        deps.log("info", `restarting ${slot.plan.id} in ${delay}ms (restart ${slot.state.restarts})`);
+        slot.timer = setT(() => {
+            slot.timer = undefined;
+            startLong(slot);
+            emit();
+        }, delay);
+    };
+    const watch = (slot, child) => {
+        live.add(child);
+        let ended = false;
+        const onEnd = (code, signal) => {
+            if (ended)
+                return;
+            ended = true;
+            live.delete(child);
+            if (live.size === 0)
+                onDrained?.();
+            if (slot.child !== child)
+                return;
+            slot.child = undefined;
+            if (slot.killTimer !== undefined) {
+                clearT(slot.killTimer);
+                slot.killTimer = undefined;
+            }
+            const ranFor = now() - (slot.childStartedAt ?? now());
+            slot.state.lastExit = { code, signal, at: now() };
+            delete slot.state.pid;
+            if (slot.expectingExit) {
+                slot.expectingExit = false;
+                slot.busy = false;
+                const then = slot.afterExit;
+                slot.afterExit = undefined;
+                then?.();
+                emit();
+                return;
+            }
+            if (stopped)
+                return;
+            if (slot.plan.kind === "scheduled") {
+                slot.busy = false;
+                slot.state.lastResult = code === 0 ? "ok" : "failed";
+                slot.state.status = "idle";
+                if (code !== 0)
+                    deps.log("warn", `${slot.plan.id} run failed (code=${code ?? "null"} signal=${signal ?? "none"})`);
+                emit();
+                return;
+            }
+            deps.log("warn", `${slot.plan.id} exited (code=${code ?? "null"} signal=${signal ?? "none"})`);
+            if (ranFor >= HEALTHY_RUN_MS)
+                slot.attempts = 0;
+            const stoppedFromOutside = signal === "SIGTERM" || signal === "SIGINT";
+            if (slot.plan.restart === "on-failure" && (code === 0 || stoppedFromOutside)) {
+                slot.state.status = "exited";
+                if (stoppedFromOutside) {
+                    slot.state.reason = `stopped outside the daemon; \`hq daemon restart ${slot.plan.id}\` starts it again`;
+                }
+                delete slot.state.nextRunAt;
+            }
+            else {
+                scheduleRestart(slot);
+            }
+            emit();
+        };
+        child.on("exit", onEnd);
+        child.on("error", (err) => {
+            deps.log("error", `${slot.plan.id}: ${errorMessage(err)}`);
+            // A child that never got a pid failed to launch and may never emit
+            // "exit". Any other error (for example a failed kill) leaves it running.
+            if (child.pid === undefined)
+                onEnd(null, null);
+        });
+    };
+    const startLong = (slot) => {
+        if (stopped)
+            return;
+        let child;
+        try {
+            child = deps.spawn(slot.plan);
+        }
+        catch (err) {
+            slot.state.reason = `failed to start: ${errorMessage(err)}`;
+            deps.log("error", `${slot.plan.id} ${slot.state.reason}`);
+            scheduleRestart(slot);
+            return;
+        }
+        slot.child = child;
+        slot.childStartedAt = now();
+        slot.state.status = "running";
+        slot.state.startedAt = slot.childStartedAt;
+        delete slot.state.reason;
+        delete slot.state.nextRunAt;
+        if (child.pid !== undefined)
+            slot.state.pid = child.pid;
+        else
+            delete slot.state.pid;
+        watch(slot, child);
+    };
+    const scheduleNextRun = (slot, delayMs) => {
+        clearTimer(slot);
+        slot.state.nextRunAt = now() + delayMs;
+        slot.timer = setT(() => {
+            slot.timer = undefined;
+            runScheduled(slot);
+        }, delayMs);
+    };
+    const runScheduled = (slot, manual = false) => {
+        if (stopped)
+            return;
+        if (!manual)
+            scheduleNextRun(slot, (slot.plan.intervalSec ?? 60) * 1000);
+        if (slot.busy) {
+            deps.log("info", `${slot.plan.id} is still running; skipping this run`);
+            emit();
+            return;
+        }
+        slot.busy = true;
+        slot.state.lastRunAt = now();
+        const command = slot.plan.command;
+        if (command?.type === "internal") {
+            slot.state.status = "running";
+            deps
+                .runInternal(command.task)
+                .then((ok) => {
+                slot.state.lastResult = ok ? "ok" : "failed";
+            }, (err) => {
+                deps.log("error", `${slot.plan.id} failed: ${errorMessage(err)}`);
+                slot.state.lastResult = "failed";
+            })
+                .finally(() => {
+                slot.busy = false;
+                if (!stopped && !slot.plan.blocked)
+                    slot.state.status = "idle";
+                emit();
+            });
+            emit();
+            return;
+        }
+        let child;
+        try {
+            child = deps.spawn(slot.plan);
+        }
+        catch (err) {
+            slot.busy = false;
+            slot.state.lastResult = "failed";
+            slot.state.status = "idle";
+            slot.state.reason = `failed to start: ${errorMessage(err)}`;
+            deps.log("error", `${slot.plan.id} ${slot.state.reason}`);
+            emit();
+            return;
+        }
+        slot.child = child;
+        slot.childStartedAt = now();
+        slot.state.status = "running";
+        delete slot.state.reason;
+        if (child.pid !== undefined)
+            slot.state.pid = child.pid;
+        watch(slot, child);
+        emit();
+    };
+    const activate = (slot) => {
+        if (slot.plan.blocked) {
+            setBlocked(slot);
+            return;
+        }
+        if (slot.plan.kind === "scheduled") {
+            slot.state.status = "idle";
+            delete slot.state.reason;
+            const delay = (slot.plan.initialDelaySec ?? 0) * 1000;
+            if (delay === 0)
+                runScheduled(slot);
+            else
+                scheduleNextRun(slot, delay);
+            return;
+        }
+        startLong(slot);
+    };
+    const newSlot = (plan) => ({
+        plan,
+        key: planKey(plan),
+        attempts: 0,
+        expectingExit: false,
+        busy: false,
+        state: {
+            id: plan.id,
+            service: plan.service,
+            kind: plan.kind,
+            description: plan.description,
+            status: "idle",
+            restarts: 0,
+        },
+    });
+    const reconcile = () => {
+        if (stopped)
+            return;
+        let plans;
+        try {
+            const result = deps.plan();
+            if (!Array.isArray(result))
+                throw new Error("plan did not return a list of units");
+            plans = result;
+        }
+        catch (err) {
+            deps.log("error", `could not plan units; keeping current ones: ${errorMessage(err)}`);
+            return;
+        }
+        const wanted = new Set(plans.map((p) => p.id));
+        for (const [id, slot] of slots) {
+            if (!wanted.has(id)) {
+                deps.log("info", `stopping ${id}: no longer enabled`);
+                terminate(slot);
+            }
+        }
+        const next = new Map();
+        for (const plan of plans) {
+            const existing = slots.get(plan.id);
+            if (!existing) {
+                const slot = newSlot(plan);
+                next.set(plan.id, slot);
+                activate(slot);
+                continue;
+            }
+            next.set(plan.id, existing);
+            const key = planKey(plan);
+            const wasBlocked = existing.plan.blocked !== undefined;
+            const changed = key !== existing.key;
+            existing.plan = plan;
+            existing.key = key;
+            existing.state.description = plan.description;
+            existing.state.kind = plan.kind;
+            if (plan.blocked) {
+                if (!wasBlocked) {
+                    deps.log("info", `pausing ${plan.id}: ${plan.blocked.reason}`);
+                    terminate(existing);
+                }
+                setBlocked(existing);
+            }
+            else if (wasBlocked) {
+                deps.log("info", `starting ${plan.id}: its requirement is now met`);
+                existing.attempts = 0;
+                // The copy stopped when it was paused may still be exiting; start the
+                // new one only after it is gone, so two never run side by side.
+                if (existing.child)
+                    existing.afterExit = () => activate(existing);
+                else
+                    activate(existing);
+            }
+            else if (changed) {
+                deps.log("info", `restarting ${plan.id}: its command changed`);
+                existing.attempts = 0;
+                existing.state.restarts = 0;
+                terminate(existing, () => activate(existing));
+            }
+        }
+        slots = next;
+        emit();
+    };
+    const handleRequest = (req) => {
+        if (req.action === "restart-daemon") {
+            deps.log("info", "restarting the daemon (requested)");
+            if (deps.restartDaemon)
+                deps.restartDaemon();
+            else
+                deps.log("warn", "this daemon cannot restart itself; restart it to load the new version");
+            return;
+        }
+        // A bot created since the last plan read is asked for before the timer
+        // would pick it up; read the plan now rather than dropping the request.
+        const unplanned = req.unit !== undefined && !slots.has(req.unit);
+        if (unplanned)
+            reconcile();
+        // The re-read already started a new long-running unit; restarting it would only bounce it.
+        if (unplanned && req.action === "restart" && slots.get(req.unit)?.child) {
+            emit();
+            return;
+        }
+        const targets = req.unit === undefined
+            ? [...slots.values()].filter((s) => s.plan.kind === "long-running" && !s.plan.blocked)
+            : [slots.get(req.unit)].filter((s) => s !== undefined);
+        if (targets.length === 0) {
+            deps.log("warn", `ignoring ${req.action} request for unknown unit ${req.unit ?? "(all)"}`);
+            return;
+        }
+        for (const slot of targets) {
+            if (slot.plan.blocked) {
+                deps.log("warn", `ignoring ${req.action} for ${slot.plan.id}: ${slot.plan.blocked.reason}`);
+                continue;
+            }
+            if (req.action === "stop") {
+                deps.log("info", `stopping ${slot.plan.id} (requested)`);
+                // An expected exit is never restarted, including one that needed SIGKILL.
+                terminate(slot, () => {
+                    slot.state.status = "exited";
+                    slot.state.reason = `stopped on request; \`hq daemon restart ${slot.plan.id}\` starts it again`;
+                    delete slot.state.nextRunAt;
+                });
+                if (!slot.child) {
+                    slot.state.status = "exited";
+                    slot.state.reason = `stopped on request; \`hq daemon restart ${slot.plan.id}\` starts it again`;
+                    delete slot.state.nextRunAt;
+                }
+                continue;
+            }
+            if (req.action === "trigger") {
+                if (slot.plan.kind !== "scheduled") {
+                    deps.log("warn", `ignoring trigger for ${slot.plan.id}: only scheduled services can be triggered`);
+                    continue;
+                }
+                deps.log("info", `running ${slot.plan.id} now (requested)`);
+                runScheduled(slot, true);
+                continue;
+            }
+            deps.log("info", `restarting ${slot.plan.id} (requested)`);
+            slot.attempts = 0;
+            if (slot.plan.kind === "scheduled")
+                continue;
+            if (slot.child)
+                terminate(slot, () => startLong(slot));
+            else {
+                clearTimer(slot);
+                startLong(slot);
+            }
+        }
+        emit();
+    };
+    const pollRequests = () => {
+        requestTimer = undefined;
+        if (stopped)
+            return;
+        try {
+            for (const req of deps.takeRequests())
+                handleRequest(req);
+        }
+        catch (err) {
+            deps.log("error", `could not read control requests: ${errorMessage(err)}`);
+        }
+        requestTimer = setT(pollRequests, requestPollMs);
+    };
+    const reconcileLoop = () => {
+        reconcileTimer = undefined;
+        if (stopped)
+            return;
+        reconcile();
+        reconcileTimer = setT(reconcileLoop, reconcileMs);
+    };
+    return {
+        start() {
+            if (started)
+                return;
+            started = true;
+            reconcile();
+            reconcileTimer = setT(reconcileLoop, reconcileMs);
+            requestTimer = setT(pollRequests, requestPollMs);
+        },
+        stop() {
+            if (drained)
+                return drained;
+            stopped = true;
+            if (reconcileTimer !== undefined)
+                clearT(reconcileTimer);
+            if (requestTimer !== undefined)
+                clearT(requestTimer);
+            for (const slot of slots.values()) {
+                clearTimer(slot);
+                slot.child = undefined;
+                if (slot.killTimer !== undefined)
+                    clearT(slot.killTimer);
+                slot.killTimer = undefined;
+                slot.state.status = "stopped";
+                delete slot.state.pid;
+                delete slot.state.nextRunAt;
+            }
+            const children = [...live];
+            for (const child of children) {
+                try {
+                    child.kill("SIGTERM");
+                }
+                catch (err) {
+                    deps.log("error", `could not stop pid ${child.pid ?? "?"}: ${errorMessage(err)}`);
+                }
+            }
+            emit();
+            drained = new Promise((resolve) => {
+                if (live.size === 0) {
+                    resolve();
+                    return;
+                }
+                const escalation = setT(() => {
+                    for (const child of live) {
+                        deps.log("warn", `pid ${child.pid ?? "?"} ignored SIGTERM; sending SIGKILL`);
+                        try {
+                            child.kill("SIGKILL");
+                        }
+                        catch (err) {
+                            deps.log("error", `could not SIGKILL pid ${child.pid ?? "?"}: ${errorMessage(err)}`);
+                        }
+                    }
+                }, STOP_ESCALATION_MS);
+                onDrained = () => {
+                    onDrained = null;
+                    clearT(escalation);
+                    resolve();
+                };
+            });
+            return drained;
+        },
+        reconcile,
+        state: snapshot,
+    };
+}
+//# sourceMappingURL=supervisor.js.map

package/dist/lib/daemon/updater.d.ts

@@ -0,0 +1,39 @@
+/**
+ * One run of the `updater` service (`hq daemon service self-update`), which
+ * replaces the boxes' `hq-cli-update.timer` and `*-codex-update.timer`:
+ *
+ * 1. Update the hq CLI through the shared self-update path (same install
+ *    routing, lock and convergence check as every other CLI update).
+ * 2. Update Codex when it is installed as a global npm package and behind.
+ * 3. After a CLI update, ask the daemon to restart itself so the supervisor and
+ *    every service run the new version.
+ */
+import type { SelfUpdateOutcome } from "../../utils/self-update.js";
+export declare const CODEX_PACKAGE = "@openai/codex";
+export interface CodexUpdater {
+    /** Version of the global npm install, or null when Codex is not installed that way. */
+    installedVersion: () => string | null;
+    latestVersion: () => string | null;
+    install: () => {
+        ok: boolean;
+        detail?: string;
+    };
+}
+export interface UpdaterDeps {
+    updateCli: () => Promise<Pick<SelfUpdateOutcome, "action" | "latest">>;
+    codex: CodexUpdater;
+    requestDaemonRestart: () => void;
+    log: (line: string) => void;
+}
+export declare function runDaemonUpdater(deps: UpdaterDeps): Promise<{
+    ok: boolean;
+}>;
+/**
+ * The variables for the updater's own CLI update. Every supervised service
+ * gets HQ_NO_UPDATE_CHECK=1 so it never updates itself; the updater is the one
+ * process that must, so it drops that opt-out.
+ */
+export declare function updaterEnv(vars: NodeJS.ProcessEnv): NodeJS.ProcessEnv;
+/** Codex through the global npm install, the way the boxes install it. */
+export declare function npmCodexUpdater(): CodexUpdater;
+//# sourceMappingURL=updater.d.ts.map