@indigoai-us/hq-cli 5.314.0 → 5.316.0

package/dist/lib/daemon/succession.d.ts

@@ -0,0 +1,118 @@
+/**
+ * Handing a Linux outpost's services from one daemon to the next.
+ *
+ * Under `KillMode=process` systemd signals only the daemon, so its services
+ * outlive it. The daemon therefore keeps two records under its folder:
+ *
+ * - `children.json`: every service process it started, with the process start
+ *   time from /proc, kept current as they start and exit;
+ * - `handoff.json`: written only when the daemon restarts onto an updated CLI
+ *   with the `daemon.supervisor-handoff` flag on, listing the long-running
+ *   services it left running and the key each was started with.
+ *
+ * On start the next daemon adopts the services named in a fresh handoff whose
+ * pid still belongs to the same process, and stops every other recorded
+ * process that is still alive (after a crash, or a restart with the handoff
+ * off) before it starts anything, so two copies of a service never run. Both
+ * records stay on disk until the adopted services are recorded as its own
+ * (finishSuccession), so a daemon that dies mid-takeover leaves them for the
+ * next one.
+ */
+import { type HostApp, type SupervisedChild } from "./supervisor.js";
+/** A handoff older than this is not trusted; its services are stopped instead. */
+export declare const HANDOFF_MAX_AGE_MS: number;
+export interface ChildRecord {
+    id: string;
+    pid: number;
+    /** Process start time (/proc/<pid>/stat field 22), so a reused pid is never mistaken for it. */
+    start: number;
+}
+export interface HandoffUnit extends ChildRecord {
+    key: string;
+}
+export interface HandoffRecord {
+    releasedAt: number;
+    units: HandoffUnit[];
+}
+export interface SuccessionPaths {
+    childrenPath: string;
+    handoffPath: string;
+}
+type ReadStart = (pid: number) => number | null;
+type Log = (level: "info" | "warn" | "error", msg: string) => void;
+export declare function successionPaths(daemonRoot: string): SuccessionPaths;
+/**
+ * The start time (field 22) from a /proc/<pid>/stat line, or null when the
+ * line is unreadable or the process is a zombie (it has exited).
+ */
+export declare function parseProcStart(text: string): number | null;
+/** The start time of `pid` on Linux, or null when no such live process exists. */
+export declare function readProcStart(pid: number): number | null;
+/** Keeps `children.json` in step with the service processes this daemon started. */
+export declare function createChildTracker(file: string, deps: {
+    readStart: ReadStart;
+    log?: Log;
+}): {
+    track(id: string, child: SupervisedChild): void;
+    records(): ChildRecord[];
+};
+export declare function writeHandoff(file: string, record: HandoffRecord): void;
+/**
+ * Read the previous daemon's records. Returns the services to adopt and the
+ * live processes to stop; a pid whose start time no longer matches belongs to
+ * another process and appears in neither. The records stay until
+ * finishSuccession.
+ */
+export declare function takeSuccession(paths: SuccessionPaths, deps: {
+    now: () => number;
+    readStart: ReadStart;
+    log: Log;
+}): {
+    adopt: HandoffUnit[];
+    orphans: ChildRecord[];
+};
+/**
+ * Complete a takeover: `records` (the adopted services, now this daemon's)
+ * replace the children record, then the handoff is dropped. Call it once the
+ * orphans are stopped.
+ */
+export declare function finishSuccession(paths: SuccessionPaths, records: readonly ChildRecord[], log?: Log): void;
+/** SIGTERM each orphan, wait up to `graceMs` for it to go, then SIGKILL what is left. */
+export declare function stopOrphans(orphans: readonly ChildRecord[], deps: {
+    readStart: ReadStart;
+    signal: (pid: number, sig: NodeJS.Signals) => void;
+    sleep: (ms: number) => Promise<void>;
+    log: Log;
+    graceMs?: number;
+}): Promise<void>;
+/**
+ * A service process started by the previous daemon. It is not this process's
+ * child, so its exit is noticed by polling and its exit code is unknown.
+ */
+export declare function adoptChild(record: ChildRecord, deps: {
+    readStart: ReadStart;
+    signal: (pid: number, sig: NodeJS.Signals) => void;
+    setInterval?: (fn: () => void, ms: number) => unknown;
+    clearInterval?: (handle: unknown) => void;
+    pollMs?: number;
+    /** Its protocol output, read from the file it writes to (see protocol-tail.ts). */
+    stdout?: SupervisedChild["stdout"];
+}): SupervisedChild;
+/** The loaded daemon unit's KillMode, or null when systemctl cannot tell. */
+export declare function systemdKillMode(run: (file: string, args: string[]) => {
+    status: number | null;
+    stdout: string;
+}): string | null;
+/**
+ * Services survive the daemon only on Linux under the systemd system unit
+ * with `KillMode=process` (a unit installed by this CLI or later). Anywhere
+ * else the service manager or the host app stops them with the daemon.
+ */
+export declare function handoffEligible(opts: {
+    platform: NodeJS.Platform;
+    managed: boolean;
+    host: HostApp | undefined;
+    killMode: string | null;
+}): boolean;
+export {};
+//# sourceMappingURL=succession.d.ts.map

package/dist/lib/daemon/succession.js

@@ -0,0 +1,263 @@
+/**
+ * Handing a Linux outpost's services from one daemon to the next.
+ *
+ * Under `KillMode=process` systemd signals only the daemon, so its services
+ * outlive it. The daemon therefore keeps two records under its folder:
+ *
+ * - `children.json`: every service process it started, with the process start
+ *   time from /proc, kept current as they start and exit;
+ * - `handoff.json`: written only when the daemon restarts onto an updated CLI
+ *   with the `daemon.supervisor-handoff` flag on, listing the long-running
+ *   services it left running and the key each was started with.
+ *
+ * On start the next daemon adopts the services named in a fresh handoff whose
+ * pid still belongs to the same process, and stops every other recorded
+ * process that is still alive (after a crash, or a restart with the handoff
+ * off) before it starts anything, so two copies of a service never run. Both
+ * records stay on disk until the adopted services are recorded as its own
+ * (finishSuccession), so a daemon that dies mid-takeover leaves them for the
+ * next one.
+ */
+import { EventEmitter } from "node:events";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { DAEMON_SYSTEMD_UNIT } from "./install.js";
+import { ensurePrivateDir } from "./paths.js";
+import { STOP_ESCALATION_MS } from "./supervisor.js";
+/** A handoff older than this is not trusted; its services are stopped instead. */
+export const HANDOFF_MAX_AGE_MS = 5 * 60_000;
+const ADOPTED_POLL_MS = 1_000;
+const ORPHAN_POLL_MS = 200;
+export function successionPaths(daemonRoot) {
+    return {
+        childrenPath: path.join(daemonRoot, "children.json"),
+        handoffPath: path.join(daemonRoot, "handoff.json"),
+    };
+}
+function writeJsonAtomic(file, value) {
+    ensurePrivateDir(path.dirname(file));
+    const tmp = `${file}.${process.pid}.tmp`;
+    fs.writeFileSync(tmp, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 });
+    fs.renameSync(tmp, file);
+}
+function isChildRecord(value) {
+    const r = value;
+    return (typeof r === "object" &&
+        r !== null &&
+        typeof r.id === "string" &&
+        Number.isInteger(r.pid) &&
+        r.pid > 0 &&
+        typeof r.start === "number" &&
+        Number.isFinite(r.start));
+}
+function removeRecord(file, log) {
+    try {
+        fs.rmSync(file, { force: true });
+    }
+    catch (err) {
+        log("warn", `could not remove ${file}: ${err.message}`);
+    }
+}
+/** Read a record file; null when it is missing. An unreadable one is removed, since no later read can use it. */
+function readRecord(file, log) {
+    let text;
+    try {
+        text = fs.readFileSync(file, "utf8");
+    }
+    catch (err) {
+        if (err.code !== "ENOENT")
+            log("warn", `could not read ${file}: ${err.message}`);
+        return null;
+    }
+    try {
+        return JSON.parse(text);
+    }
+    catch (err) {
+        log("warn", `ignoring unreadable ${file}: ${err.message}`);
+        removeRecord(file, log);
+        return null;
+    }
+}
+/**
+ * The start time (field 22) from a /proc/<pid>/stat line, or null when the
+ * line is unreadable or the process is a zombie (it has exited).
+ */
+export function parseProcStart(text) {
+    const close = text.lastIndexOf(")");
+    if (close < 0)
+        return null;
+    // Fields after the command name start at field 3 (state), so field 22 is index 19.
+    const rest = text.slice(close + 1).trim().split(/\s+/);
+    if (rest[0] === "Z" || rest[0] === "X")
+        return null;
+    const start = Number(rest[19]);
+    return Number.isInteger(start) ? start : null;
+}
+/** The start time of `pid` on Linux, or null when no such live process exists. */
+export function readProcStart(pid) {
+    try {
+        return parseProcStart(fs.readFileSync(`/proc/${pid}/stat`, "utf8"));
+    }
+    catch {
+        return null;
+    }
+}
+/** Keeps `children.json` in step with the service processes this daemon started. */
+export function createChildTracker(file, deps) {
+    const current = new Map();
+    const save = () => {
+        try {
+            writeJsonAtomic(file, [...current.values()]);
+        }
+        catch (err) {
+            deps.log?.("error", `could not record service processes: ${err.message}`);
+        }
+    };
+    return {
+        track(id, child) {
+            if (child.pid === undefined)
+                return;
+            const start = deps.readStart(child.pid);
+            if (start === null)
+                return;
+            current.set(child, { id, pid: child.pid, start });
+            save();
+            child.on("exit", () => {
+                if (current.delete(child))
+                    save();
+            });
+        },
+        records: () => [...current.values()],
+    };
+}
+export function writeHandoff(file, record) {
+    writeJsonAtomic(file, { version: 1, ...record });
+}
+/**
+ * Read the previous daemon's records. Returns the services to adopt and the
+ * live processes to stop; a pid whose start time no longer matches belongs to
+ * another process and appears in neither. The records stay until
+ * finishSuccession.
+ */
+export function takeSuccession(paths, deps) {
+    const children = readRecord(paths.childrenPath, deps.log);
+    const handoff = readRecord(paths.handoffPath, deps.log);
+    const recorded = Array.isArray(children) ? children.filter(isChildRecord) : [];
+    let handoffUnits = [];
+    let fresh = false;
+    if (handoff !== null) {
+        if (typeof handoff.releasedAt !== "number" || !Array.isArray(handoff.units)) {
+            deps.log("warn", "ignoring a malformed handoff record");
+            removeRecord(paths.handoffPath, deps.log);
+        }
+        else {
+            handoffUnits = handoff.units.filter((u) => isChildRecord(u) && typeof u.key === "string");
+            fresh = deps.now() - handoff.releasedAt <= HANDOFF_MAX_AGE_MS;
+            if (!fresh)
+                deps.log("warn", "the previous daemon's handoff is too old to trust; stopping its services");
+        }
+    }
+    const alive = (r) => deps.readStart(r.pid) === r.start;
+    const adopt = fresh ? handoffUnits.filter(alive) : [];
+    const adopted = new Set(adopt.map((u) => u.pid));
+    const orphans = new Map();
+    for (const r of [...recorded, ...handoffUnits]) {
+        if (adopted.has(r.pid) || orphans.has(r.pid) || !alive(r))
+            continue;
+        orphans.set(r.pid, { id: r.id, pid: r.pid, start: r.start });
+    }
+    return { adopt, orphans: [...orphans.values()] };
+}
+/**
+ * Complete a takeover: `records` (the adopted services, now this daemon's)
+ * replace the children record, then the handoff is dropped. Call it once the
+ * orphans are stopped.
+ */
+export function finishSuccession(paths, records, log) {
+    writeJsonAtomic(paths.childrenPath, records);
+    removeRecord(paths.handoffPath, log ?? (() => { }));
+}
+/** SIGTERM each orphan, wait up to `graceMs` for it to go, then SIGKILL what is left. */
+export async function stopOrphans(orphans, deps) {
+    const graceMs = deps.graceMs ?? STOP_ESCALATION_MS;
+    const same = (r) => deps.readStart(r.pid) === r.start;
+    const send = (r, sig) => {
+        try {
+            deps.signal(r.pid, sig);
+        }
+        catch (err) {
+            deps.log("warn", `could not send ${sig} to ${r.id} (pid ${r.pid}): ${err.message}`);
+        }
+    };
+    let remaining = orphans.filter(same);
+    for (const r of remaining) {
+        deps.log("info", `stopping ${r.id} (pid ${r.pid}), left running by a previous daemon`);
+        send(r, "SIGTERM");
+    }
+    let waited = 0;
+    remaining = remaining.filter(same);
+    while (remaining.length > 0 && waited < graceMs) {
+        await deps.sleep(ORPHAN_POLL_MS);
+        waited += ORPHAN_POLL_MS;
+        remaining = remaining.filter(same);
+    }
+    for (const r of remaining) {
+        deps.log("warn", `${r.id} (pid ${r.pid}) ignored SIGTERM; sending SIGKILL`);
+        send(r, "SIGKILL");
+    }
+}
+/**
+ * A service process started by the previous daemon. It is not this process's
+ * child, so its exit is noticed by polling and its exit code is unknown.
+ */
+export function adoptChild(record, deps) {
+    const events = new EventEmitter();
+    const setI = deps.setInterval ?? ((fn, ms) => setInterval(fn, ms));
+    const clearI = deps.clearInterval ?? ((h) => clearInterval(h));
+    const same = () => deps.readStart(record.pid) === record.start;
+    let ended = false;
+    const poll = setI(() => {
+        if (ended || same())
+            return;
+        ended = true;
+        clearI(poll);
+        events.emit("exit", null, null);
+    }, deps.pollMs ?? ADOPTED_POLL_MS);
+    poll.unref?.();
+    const child = {
+        pid: record.pid,
+        ...(deps.stdout ? { stdout: deps.stdout } : {}),
+        on(event, handler) {
+            events.on(event, handler);
+            return child;
+        },
+        kill(signal = "SIGTERM") {
+            if (ended || !same())
+                return false;
+            deps.signal(record.pid, (typeof signal === "number" ? "SIGTERM" : signal));
+            return true;
+        },
+    };
+    return child;
+}
+/** The loaded daemon unit's KillMode, or null when systemctl cannot tell. */
+export function systemdKillMode(run) {
+    try {
+        const r = run("systemctl", ["show", "--property=KillMode", "--value", DAEMON_SYSTEMD_UNIT]);
+        if (r.status !== 0)
+            return null;
+        return r.stdout.trim() || null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Services survive the daemon only on Linux under the systemd system unit
+ * with `KillMode=process` (a unit installed by this CLI or later). Anywhere
+ * else the service manager or the host app stops them with the daemon.
+ */
+export function handoffEligible(opts) {
+    return opts.platform === "linux" && opts.managed && opts.host === undefined && opts.killMode === "process";
+}
+//# sourceMappingURL=succession.js.map

package/dist/lib/daemon/supervisor.d.ts

@@ -0,0 +1,132 @@
+/**
+ * The `hq daemon run` supervisor. It keeps the planned units running:
+ *
+ * - long-running units are restarted after they exit, with per-unit backoff
+ *   (1 s doubling to 60 s, reset once a unit has stayed up for a minute);
+ *   `on-failure` units stay down after a clean exit or a SIGTERM/SIGINT from
+ *   outside, and any unit stays down after a stop request (`hq bot stop`),
+ *   until a restart request;
+ * - scheduled units run after their initial delay and then on their interval,
+ *   never overlapping themselves;
+ * - the plan is re-read on a timer, so disabled services stop, new bots start,
+ *   and units whose requirement appears or disappears follow on their own;
+ * - restart/trigger/stop requests from the CLI are polled and applied; a
+ *   request for a unit that is not planned yet re-reads the plan first;
+ * - stop() sends SIGTERM to every child, escalates to SIGKILL after
+ *   STOP_ESCALATION_MS, and resolves once every child has exited;
+ * - release() instead leaves every child running and reports the long-running
+ *   ones, so the next daemon can adopt them (`adopt`). Adopted units whose
+ *   command is unchanged keep running; changed ones are replaced one at a
+ *   time, draining first when the unit has a drain signal.
+ *
+ * Processes, time, and state output are injected; `run.ts` wires the real ones.
+ */
+import type { UnitPlan } from "./catalog.js";
+import type { MachineType } from "./machine-type.js";
+import type { ControlRequest } from "./requests.js";
+import { syncPassResult } from "./sync-runner.js";
+export declare const STOP_ESCALATION_MS = 10000;
+/** After a handoff, how long a replaced unit must stay up before the next one is replaced. */
+export declare const HANDOFF_STAGGER_MS = 15000;
+/** How long a draining unit may take to finish its work before it gets SIGTERM. */
+export declare const HANDOFF_DRAIN_MAX_MS: number;
+export interface SupervisedChild {
+    pid?: number;
+    stdout?: {
+        on(event: "data" | "end", handler: (chunk?: Buffer | string) => void): unknown;
+    };
+    on(event: "exit", handler: (code: number | null, signal: string | null) => void): unknown;
+    on(event: "error", handler: (err: Error) => void): unknown;
+    kill(signal?: NodeJS.Signals | number): boolean;
+}
+type UnitStatus = "running" | "waiting" | "unavailable" | "backoff" | "exited" | "idle" | "stopped";
+interface UnitState {
+    id: string;
+    service: string;
+    kind: UnitPlan["kind"];
+    description: string;
+    status: UnitStatus;
+    reason?: string;
+    pid?: number;
+    startedAt?: number;
+    restarts: number;
+    lastExit?: {
+        code: number | null;
+        signal: string | null;
+        at: number;
+    };
+    lastRunAt?: number;
+    nextRunAt?: number;
+    lastResult?: "ok" | "failed";
+    lastHeartbeatAt?: string;
+    lastPassResult?: ReturnType<typeof syncPassResult>;
+}
+/** An app that runs the daemon as its own child process instead of an OS unit. */
+export type HostApp = "desktop";
+export interface DaemonState {
+    version: 1;
+    pid: number;
+    startedAt: number;
+    updatedAt: number;
+    machineType: MachineType;
+    /** Set when an app runs the daemon as its child; absent under the OS unit. */
+    host?: HostApp;
+    units: UnitState[];
+}
+/** A running service process left by the previous daemon, and the key it was started with. */
+export interface AdoptedUnit {
+    id: string;
+    key: string;
+    child: SupervisedChild;
+}
+/** A long-running service left running by release(). */
+interface ReleasedUnit {
+    id: string;
+    pid: number;
+    key: string;
+}
+export interface SupervisorDeps {
+    machineType: MachineType;
+    host?: HostApp;
+    pid: number;
+    plan: () => UnitPlan[];
+    spawn: (unit: UnitPlan) => SupervisedChild;
+    runInternal: (task: string) => Promise<boolean>;
+    takeRequests: () => ControlRequest[];
+    /** Restart the whole daemon process (a `restart-daemon` request). */
+    restartDaemon?: () => void;
+    writeState: (state: DaemonState) => void;
+    log: (level: "info" | "warn" | "error", msg: string) => void;
+    now?: () => number;
+    setTimeout?: (fn: () => void, ms: number) => unknown;
+    clearTimeout?: (handle: unknown) => void;
+    reconcileMs?: number;
+    requestPollMs?: number;
+    /** Read hq-cloud's lease status after it exits with the lease-conflict code. */
+    syncOwnerBlockReason?: () => string | undefined;
+    /** A separate mesh daemon currently owns its pid lock after our mesh child exits. */
+    meshOwnerReason?: () => string | undefined;
+    /** The running CLI's version; part of the key of units that run `hq` commands. */
+    cliVersion?: string;
+    /** Services to take over from the previous daemon instead of starting them. */
+    adopt?: readonly AdoptedUnit[];
+}
+export interface Supervisor {
+    start(): void;
+    /** Resolves once every child process has exited. */
+    stop(): Promise<void>;
+    reconcile(): void;
+    state(): DaemonState;
+    /** Stop supervising without signalling any child; returns the long-running units left running. */
+    release(): ReleasedUnit[];
+}
+/**
+ * What a unit's process was started from; a unit whose key changes is
+ * restarted. The CLI version counts only for `hq` commands, so a CLI update
+ * leaves hq-cloud runners, other programs, and units marked
+ * `keepAcrossCliUpdates` running.
+ */
+export declare function planKey(plan: UnitPlan, cliVersion?: string): string;
+export declare function createSupervisor(deps: SupervisorDeps): Supervisor;
+export {};
+//# sourceMappingURL=supervisor.d.ts.map