@indigoai-us/hq-cli 5.314.0 → 5.316.0

package/dist/lib/daemon/supervisor.js

@@ -0,0 +1,836 @@
+/**
+ * The `hq daemon run` supervisor. It keeps the planned units running:
+ *
+ * - long-running units are restarted after they exit, with per-unit backoff
+ *   (1 s doubling to 60 s, reset once a unit has stayed up for a minute);
+ *   `on-failure` units stay down after a clean exit or a SIGTERM/SIGINT from
+ *   outside, and any unit stays down after a stop request (`hq bot stop`),
+ *   until a restart request;
+ * - scheduled units run after their initial delay and then on their interval,
+ *   never overlapping themselves;
+ * - the plan is re-read on a timer, so disabled services stop, new bots start,
+ *   and units whose requirement appears or disappears follow on their own;
+ * - restart/trigger/stop requests from the CLI are polled and applied; a
+ *   request for a unit that is not planned yet re-reads the plan first;
+ * - stop() sends SIGTERM to every child, escalates to SIGKILL after
+ *   STOP_ESCALATION_MS, and resolves once every child has exited;
+ * - release() instead leaves every child running and reports the long-running
+ *   ones, so the next daemon can adopt them (`adopt`). Adopted units whose
+ *   command is unchanged keep running; changed ones are replaced one at a
+ *   time, draining first when the unit has a drain signal.
+ *
+ * Processes, time, and state output are injected; `run.ts` wires the real ones.
+ */
+import { StringDecoder } from "node:string_decoder";
+import { SYNC_RUNNER_LEASE_CONFLICT_EXIT, SYNC_RUNNER_LEASE_RETRY_MS, parseSyncRunnerProtocolLine, syncPassResult, } from "./sync-runner.js";
+const BACKOFF_BASE_MS = 1_000;
+const BACKOFF_MAX_MS = 60_000;
+const HEALTHY_RUN_MS = 60_000;
+export const STOP_ESCALATION_MS = 10_000;
+const MESH_OWNER_RECHECK_MS = 15_000;
+const MESH_OWNER_LOG_INTERVAL_MS = 10 * 60_000;
+/** After a handoff, how long a replaced unit must stay up before the next one is replaced. */
+export const HANDOFF_STAGGER_MS = 15_000;
+/** How long a draining unit may take to finish its work before it gets SIGTERM. */
+export const HANDOFF_DRAIN_MAX_MS = 15 * 60_000;
+/** Plan reads in a row that must find a running remote-control host blocked before it is paused. */
+const BLOCKED_READS_BEFORE_PAUSE = 2;
+function backoffMs(attempt) {
+    return Math.min(BACKOFF_MAX_MS, BACKOFF_BASE_MS * 2 ** Math.max(0, attempt));
+}
+/**
+ * What a unit's process was started from; a unit whose key changes is
+ * restarted. The CLI version counts only for `hq` commands, so a CLI update
+ * leaves hq-cloud runners, other programs, and units marked
+ * `keepAcrossCliUpdates` running.
+ */
+export function planKey(plan, cliVersion) {
+    const fields = [
+        plan.kind,
+        plan.restart,
+        plan.intervalSec,
+        plan.initialDelaySec,
+        plan.heartbeatTimeoutMs,
+        plan.command,
+        plan.env,
+    ];
+    if (cliVersion !== undefined && plan.command?.type === "hq" && !plan.keepAcrossCliUpdates)
+        fields.push(cliVersion);
+    return JSON.stringify(fields);
+}
+function errorMessage(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+export function createSupervisor(deps) {
+    const now = deps.now ?? Date.now;
+    const setT = deps.setTimeout ?? ((fn, ms) => setTimeout(fn, ms));
+    const clearT = deps.clearTimeout ?? ((h) => clearTimeout(h));
+    const reconcileMs = deps.reconcileMs ?? 15_000;
+    const requestPollMs = deps.requestPollMs ?? 1_000;
+    const startedAt = now();
+    let slots = new Map();
+    let stopped = false;
+    let started = false;
+    let reconcileTimer;
+    let requestTimer;
+    /** Adopted units whose command changed, replaced one at a time. */
+    const handoffQueue = [];
+    let handoffActive = false;
+    let handoffTimer;
+    /** Every child process not yet known to have exited, including ones the slots have let go of. */
+    const live = new Set();
+    let drained = null;
+    let onDrained = null;
+    const snapshot = () => ({
+        version: 1,
+        pid: deps.pid,
+        startedAt,
+        updatedAt: now(),
+        machineType: deps.machineType,
+        ...(deps.host ? { host: deps.host } : {}),
+        units: [...slots.values()].map((s) => ({ ...s.state })),
+    });
+    const emit = () => {
+        try {
+            deps.writeState(snapshot());
+        }
+        catch (err) {
+            deps.log("error", `could not write daemon state: ${errorMessage(err)}`);
+        }
+    };
+    const clearTimer = (slot) => {
+        if (slot.timer !== undefined)
+            clearT(slot.timer);
+        slot.timer = undefined;
+    };
+    const clearHeartbeatTimer = (slot) => {
+        if (slot.heartbeatTimer !== undefined)
+            clearT(slot.heartbeatTimer);
+        slot.heartbeatTimer = undefined;
+    };
+    const setBlocked = (slot) => {
+        const block = slot.plan.blocked;
+        slot.state.status = block.state;
+        slot.state.reason = block.reason;
+        delete slot.state.pid;
+        delete slot.state.nextRunAt;
+    };
+    const signalChild = (slot, child, signal) => {
+        try {
+            child.kill(signal);
+        }
+        catch (err) {
+            deps.log("error", `could not send ${signal} to ${slot.plan.id}: ${errorMessage(err)}`);
+        }
+    };
+    /**
+     * Stop the slot's child (if any), then run `then` once it has exited. With
+     * `drain`, a unit that has a drain signal gets it first and SIGTERM only
+     * after HANDOFF_DRAIN_MAX_MS; SIGKILL follows SIGTERM after STOP_ESCALATION_MS.
+     */
+    const terminate = (slot, then, drain = false) => {
+        clearTimer(slot);
+        clearHeartbeatTimer(slot);
+        const child = slot.child;
+        if (!child) {
+            then?.();
+            return;
+        }
+        slot.expectingExit = true;
+        slot.afterExit = then;
+        if (slot.killTimer !== undefined)
+            clearT(slot.killTimer);
+        const escalateToKill = () => {
+            slot.killTimer = setT(() => {
+                slot.killTimer = undefined;
+                if (slot.child !== child)
+                    return;
+                deps.log("warn", `${slot.plan.id} ignored SIGTERM; sending SIGKILL`);
+                signalChild(slot, child, "SIGKILL");
+            }, STOP_ESCALATION_MS);
+        };
+        const drainSignal = drain ? slot.plan.drainSignal : undefined;
+        if (drainSignal) {
+            slot.killTimer = setT(() => {
+                slot.killTimer = undefined;
+                if (slot.child !== child)
+                    return;
+                deps.log("warn", `${slot.plan.id} did not finish draining; sending SIGTERM`);
+                escalateToKill();
+                signalChild(slot, child, "SIGTERM");
+            }, HANDOFF_DRAIN_MAX_MS);
+            signalChild(slot, child, drainSignal);
+            return;
+        }
+        escalateToKill();
+        signalChild(slot, child, "SIGTERM");
+    };
+    const scheduleRestart = (slot) => {
+        if (stopped)
+            return;
+        const delay = backoffMs(slot.attempts);
+        slot.attempts += 1;
+        slot.state.restarts += 1;
+        slot.state.status = "backoff";
+        slot.state.nextRunAt = now() + delay;
+        deps.log("info", `restarting ${slot.plan.id} in ${delay}ms (restart ${slot.state.restarts})`);
+        slot.timer = setT(() => {
+            slot.timer = undefined;
+            startLong(slot);
+            emit();
+        }, delay);
+    };
+    const watch = (slot, child) => {
+        live.add(child);
+        let ended = false;
+        const timeoutMs = slot.plan.heartbeatTimeoutMs;
+        const decoder = timeoutMs ? new StringDecoder("utf8") : null;
+        let stdoutRemainder = "";
+        let lastHeartbeatPersistedAt = now() - 15_000;
+        const armHeartbeatWatchdog = () => {
+            if (!timeoutMs)
+                return;
+            clearHeartbeatTimer(slot);
+            slot.heartbeatTimer = setT(() => {
+                slot.heartbeatTimer = undefined;
+                if (stopped || slot.child !== child || slot.expectingExit)
+                    return;
+                deps.log("warn", `${slot.plan.id} emitted no sync protocol heartbeat for ${timeoutMs}ms; restarting it`);
+                terminate(slot, () => scheduleRestart(slot));
+                emit();
+            }, timeoutMs);
+        };
+        const acceptProtocolLine = (line) => {
+            let event;
+            try {
+                event = parseSyncRunnerProtocolLine(line);
+            }
+            catch (err) {
+                deps.log("error", `${slot.plan.id} emitted invalid sync protocol output (${Buffer.byteLength(line)} bytes): ${errorMessage(err)}`);
+                return;
+            }
+            if (!event)
+                return;
+            const at = now();
+            slot.state.lastHeartbeatAt = new Date(at).toISOString();
+            const pass = syncPassResult(event, slot.state.lastHeartbeatAt);
+            if (pass)
+                slot.state.lastPassResult = pass;
+            armHeartbeatWatchdog();
+            if (pass || at - lastHeartbeatPersistedAt >= 15_000) {
+                lastHeartbeatPersistedAt = at;
+                emit();
+            }
+        };
+        if (decoder && child.stdout) {
+            child.stdout.on("data", (chunk) => {
+                if (chunk === undefined)
+                    return;
+                stdoutRemainder += decoder.write(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+                let newline = stdoutRemainder.indexOf("\n");
+                while (newline >= 0) {
+                    acceptProtocolLine(stdoutRemainder.slice(0, newline).trim());
+                    stdoutRemainder = stdoutRemainder.slice(newline + 1);
+                    newline = stdoutRemainder.indexOf("\n");
+                }
+                // A malformed or unexpectedly large line is not a heartbeat. Bound
+                // retained data so a broken child cannot grow daemon memory forever.
+                if (Buffer.byteLength(stdoutRemainder) > 1024 * 1024)
+                    stdoutRemainder = "";
+            });
+            child.stdout.on("end", () => {
+                stdoutRemainder += decoder.end();
+                if (stdoutRemainder)
+                    acceptProtocolLine(stdoutRemainder.trim());
+                stdoutRemainder = "";
+            });
+        }
+        armHeartbeatWatchdog();
+        const onEnd = (code, signal) => {
+            if (ended)
+                return;
+            ended = true;
+            live.delete(child);
+            if (live.size === 0)
+                onDrained?.();
+            if (slot.child !== child)
+                return;
+            slot.child = undefined;
+            clearHeartbeatTimer(slot);
+            if (slot.killTimer !== undefined) {
+                clearT(slot.killTimer);
+                slot.killTimer = undefined;
+            }
+            const ranFor = now() - (slot.childStartedAt ?? now());
+            slot.state.lastExit = { code, signal, at: now() };
+            delete slot.state.pid;
+            if (slot.expectingExit) {
+                slot.expectingExit = false;
+                slot.busy = false;
+                const then = slot.afterExit;
+                slot.afterExit = undefined;
+                then?.();
+                emit();
+                return;
+            }
+            if (stopped)
+                return;
+            if (slot.plan.kind === "scheduled") {
+                slot.busy = false;
+                slot.state.lastResult = code === 0 ? "ok" : "failed";
+                slot.state.status = "idle";
+                if (code !== 0)
+                    deps.log("warn", `${slot.plan.id} run failed (code=${code ?? "null"} signal=${signal ?? "none"})`);
+                emit();
+                return;
+            }
+            if (slot.plan.service === "sync" && code === SYNC_RUNNER_LEASE_CONFLICT_EXIT) {
+                const reason = deps.syncOwnerBlockReason?.() ?? "sync owned by another runner";
+                slot.state.status = "waiting";
+                slot.state.reason = reason;
+                slot.state.nextRunAt = now() + SYNC_RUNNER_LEASE_RETRY_MS;
+                deps.log("info", `${reason}; retrying in ${SYNC_RUNNER_LEASE_RETRY_MS}ms`);
+                slot.timer = setT(() => {
+                    slot.timer = undefined;
+                    startLong(slot);
+                    emit();
+                }, SYNC_RUNNER_LEASE_RETRY_MS);
+                emit();
+                return;
+            }
+            const meshOwnerReason = slot.plan.service === "mesh" ? deps.meshOwnerReason?.() : undefined;
+            if (meshOwnerReason) {
+                let lastLoggedReason;
+                let lastLoggedAt = Number.NEGATIVE_INFINITY;
+                const waitForOwner = (reason) => {
+                    slot.state.status = "waiting";
+                    slot.state.reason = reason;
+                    slot.state.nextRunAt = now() + MESH_OWNER_RECHECK_MS;
+                    const at = now();
+                    if (reason !== lastLoggedReason || at - lastLoggedAt >= MESH_OWNER_LOG_INTERVAL_MS) {
+                        deps.log("warn", `${reason}; retrying mesh ownership check in ${MESH_OWNER_RECHECK_MS}ms`);
+                        lastLoggedReason = reason;
+                        lastLoggedAt = at;
+                    }
+                    slot.timer = setT(() => {
+                        slot.timer = undefined;
+                        const current = deps.meshOwnerReason?.();
+                        if (current)
+                            waitForOwner(current);
+                        else {
+                            slot.attempts = 0;
+                            delete slot.state.reason;
+                            delete slot.state.nextRunAt;
+                            startLong(slot);
+                        }
+                        emit();
+                    }, MESH_OWNER_RECHECK_MS);
+                };
+                waitForOwner(meshOwnerReason);
+                emit();
+                return;
+            }
+            deps.log("warn", `${slot.plan.id} exited (code=${code ?? "null"} signal=${signal ?? "none"})`);
+            if (ranFor >= HEALTHY_RUN_MS)
+                slot.attempts = 0;
+            const stoppedFromOutside = signal === "SIGTERM" || signal === "SIGINT";
+            if (slot.plan.restart === "on-failure" && (code === 0 || stoppedFromOutside)) {
+                slot.state.status = "exited";
+                if (stoppedFromOutside) {
+                    slot.state.reason = `stopped outside the daemon; \`hq daemon restart ${slot.plan.id}\` starts it again`;
+                }
+                delete slot.state.nextRunAt;
+            }
+            else {
+                scheduleRestart(slot);
+            }
+            emit();
+        };
+        child.on("exit", onEnd);
+        child.on("error", (err) => {
+            deps.log("error", `${slot.plan.id}: ${errorMessage(err)}`);
+            // A child that never got a pid failed to launch and may never emit
+            // "exit". Any other error (for example a failed kill) leaves it running.
+            if (child.pid === undefined)
+                onEnd(null, null);
+        });
+    };
+    const startLong = (slot) => {
+        if (stopped)
+            return;
+        let child;
+        try {
+            child = deps.spawn(slot.plan);
+        }
+        catch (err) {
+            slot.state.reason = `failed to start: ${errorMessage(err)}`;
+            deps.log("error", `${slot.plan.id} ${slot.state.reason}`);
+            scheduleRestart(slot);
+            return;
+        }
+        slot.child = child;
+        slot.childKey = slot.key;
+        slot.childStartedAt = now();
+        slot.state.status = "running";
+        slot.state.startedAt = slot.childStartedAt;
+        delete slot.state.reason;
+        delete slot.state.nextRunAt;
+        if (child.pid !== undefined)
+            slot.state.pid = child.pid;
+        else
+            delete slot.state.pid;
+        watch(slot, child);
+    };
+    const scheduleNextRun = (slot, delayMs) => {
+        clearTimer(slot);
+        slot.state.nextRunAt = now() + delayMs;
+        slot.timer = setT(() => {
+            slot.timer = undefined;
+            runScheduled(slot);
+        }, delayMs);
+    };
+    const runScheduled = (slot, manual = false) => {
+        if (stopped)
+            return;
+        if (!manual)
+            scheduleNextRun(slot, (slot.plan.intervalSec ?? 60) * 1000);
+        if (slot.busy) {
+            deps.log("info", `${slot.plan.id} is still running; skipping this run`);
+            emit();
+            return;
+        }
+        slot.busy = true;
+        slot.state.lastRunAt = now();
+        const command = slot.plan.command;
+        if (command?.type === "internal") {
+            slot.state.status = "running";
+            deps
+                .runInternal(command.task)
+                .then((ok) => {
+                slot.state.lastResult = ok ? "ok" : "failed";
+            }, (err) => {
+                deps.log("error", `${slot.plan.id} failed: ${errorMessage(err)}`);
+                slot.state.lastResult = "failed";
+            })
+                .finally(() => {
+                slot.busy = false;
+                if (!stopped && !slot.plan.blocked)
+                    slot.state.status = "idle";
+                emit();
+            });
+            emit();
+            return;
+        }
+        let child;
+        try {
+            child = deps.spawn(slot.plan);
+        }
+        catch (err) {
+            slot.busy = false;
+            slot.state.lastResult = "failed";
+            slot.state.status = "idle";
+            slot.state.reason = `failed to start: ${errorMessage(err)}`;
+            deps.log("error", `${slot.plan.id} ${slot.state.reason}`);
+            emit();
+            return;
+        }
+        slot.child = child;
+        slot.childKey = slot.key;
+        slot.childStartedAt = now();
+        slot.state.status = "running";
+        delete slot.state.reason;
+        if (child.pid !== undefined)
+            slot.state.pid = child.pid;
+        watch(slot, child);
+        emit();
+    };
+    const activate = (slot) => {
+        if (slot.plan.blocked) {
+            setBlocked(slot);
+            return;
+        }
+        if (slot.plan.kind === "scheduled") {
+            slot.state.status = "idle";
+            delete slot.state.reason;
+            const delay = (slot.plan.initialDelaySec ?? 0) * 1000;
+            if (delay === 0)
+                runScheduled(slot);
+            else
+                scheduleNextRun(slot, delay);
+            return;
+        }
+        startLong(slot);
+    };
+    const newSlot = (plan) => ({
+        plan,
+        key: planKey(plan, deps.cliVersion),
+        attempts: 0,
+        expectingExit: false,
+        busy: false,
+        state: {
+            id: plan.id,
+            service: plan.service,
+            kind: plan.kind,
+            description: plan.description,
+            status: "idle",
+            restarts: 0,
+        },
+    });
+    const reconcile = () => {
+        if (stopped)
+            return;
+        let plans;
+        try {
+            const result = deps.plan();
+            if (!Array.isArray(result))
+                throw new Error("plan did not return a list of units");
+            plans = result;
+        }
+        catch (err) {
+            deps.log("error", `could not plan units; keeping current ones: ${errorMessage(err)}`);
+            return;
+        }
+        const wanted = new Set(plans.map((p) => p.id));
+        for (const [id, slot] of slots) {
+            if (!wanted.has(id)) {
+                deps.log("info", `stopping ${id}: no longer enabled`);
+                terminate(slot);
+            }
+        }
+        const next = new Map();
+        for (const plan of plans) {
+            const existing = slots.get(plan.id);
+            if (!existing) {
+                const slot = newSlot(plan);
+                next.set(plan.id, slot);
+                activate(slot);
+                continue;
+            }
+            next.set(plan.id, existing);
+            // A remote-control host is what people's sessions run in. Its tool can
+            // look missing for one plan read while it is being reinstalled, so it is
+            // only paused once the tool is missing on two reads in a row.
+            if (plan.blocked && existing.plan.blocked === undefined && existing.child && plan.keepAcrossCliUpdates) {
+                existing.blockedReads = (existing.blockedReads ?? 0) + 1;
+                if (existing.blockedReads < BLOCKED_READS_BEFORE_PAUSE)
+                    continue;
+            }
+            existing.blockedReads = 0;
+            const adopted = existing.adopted === true;
+            existing.adopted = false;
+            const key = planKey(plan, deps.cliVersion);
+            const wasBlocked = existing.plan.blocked !== undefined;
+            const changed = key !== existing.key;
+            existing.plan = plan;
+            existing.key = key;
+            existing.state.description = plan.description;
+            existing.state.kind = plan.kind;
+            if (plan.blocked) {
+                if (!wasBlocked) {
+                    deps.log("info", `pausing ${plan.id}: ${plan.blocked.reason}`);
+                    terminate(existing);
+                }
+                setBlocked(existing);
+            }
+            else if (wasBlocked) {
+                deps.log("info", `starting ${plan.id}: its requirement is now met`);
+                existing.attempts = 0;
+                // The copy stopped when it was paused may still be exiting; start the
+                // new one only after it is gone, so two never run side by side.
+                if (existing.child)
+                    existing.afterExit = () => activate(existing);
+                else
+                    activate(existing);
+            }
+            else if (changed && adopted && existing.child) {
+                deps.log("info", `will replace ${plan.id}: it runs an older CLI or command`);
+                handoffQueue.push(existing);
+            }
+            else if (changed) {
+                deps.log("info", `restarting ${plan.id}: its command changed`);
+                existing.attempts = 0;
+                existing.state.restarts = 0;
+                terminate(existing, () => activate(existing));
+            }
+        }
+        slots = next;
+        emit();
+        pumpHandoff();
+    };
+    /** Replace the next queued adopted unit once the previous replacement has settled. */
+    const pumpHandoff = () => {
+        if (stopped || handoffActive)
+            return;
+        let slot;
+        while ((slot = handoffQueue.shift())) {
+            // Skip units that left the plan, were paused, or already restarted on their own.
+            if (slots.get(slot.plan.id) === slot && !slot.plan.blocked && slot.child && slot.childKey !== slot.key)
+                break;
+        }
+        if (!slot)
+            return;
+        const target = slot;
+        handoffActive = true;
+        deps.log("info", `replacing ${target.plan.id}${target.plan.drainSignal ? " once it finishes its current work" : ""}`);
+        target.attempts = 0;
+        target.state.restarts = 0;
+        terminate(target, () => activate(target), true);
+        const settle = (delay) => {
+            handoffTimer = setT(() => {
+                handoffTimer = undefined;
+                if (stopped)
+                    return;
+                const gone = slots.get(target.plan.id) !== target || target.plan.blocked !== undefined;
+                if (!gone && target.expectingExit)
+                    return settle(HANDOFF_STAGGER_MS);
+                if (!gone && target.child) {
+                    const ranFor = now() - (target.childStartedAt ?? now());
+                    if (ranFor < HANDOFF_STAGGER_MS)
+                        return settle(HANDOFF_STAGGER_MS - ranFor);
+                }
+                else if (!gone && target.state.status === "backoff") {
+                    return settle(HANDOFF_STAGGER_MS);
+                }
+                handoffActive = false;
+                pumpHandoff();
+            }, delay);
+        };
+        settle(HANDOFF_STAGGER_MS);
+    };
+    /** Stop a process the plan has no place for. */
+    const stopStray = (id, child) => {
+        const slot = newSlot({ id, service: id, description: id, kind: "long-running", restart: "always", command: null, env: {} });
+        slot.child = child;
+        watch(slot, child);
+        terminate(slot);
+    };
+    /** Take over the previous daemon's services before the first plan read starts anything. */
+    const adoptPrevious = () => {
+        const adopt = deps.adopt ?? [];
+        if (adopt.length === 0)
+            return;
+        let plans = [];
+        try {
+            const result = deps.plan();
+            if (!Array.isArray(result))
+                throw new Error("plan did not return a list of units");
+            plans = result;
+        }
+        catch (err) {
+            deps.log("error", `could not plan units; stopping the services left by the previous daemon: ${errorMessage(err)}`);
+        }
+        const byId = new Map(plans.map((p) => [p.id, p]));
+        for (const unit of adopt) {
+            const plan = byId.get(unit.id);
+            if (!plan || plan.kind !== "long-running" || slots.has(unit.id)) {
+                deps.log("info", `stopping ${unit.id} left by the previous daemon: no longer planned as a service`);
+                stopStray(unit.id, unit.child);
+                continue;
+            }
+            // Recorded as runnable, so the first plan read pauses it when it is now blocked.
+            const slot = newSlot({ ...plan, blocked: undefined });
+            slot.key = unit.key;
+            slot.adopted = true;
+            slot.child = unit.child;
+            slot.childKey = unit.key;
+            slot.childStartedAt = now();
+            slot.state.status = "running";
+            slot.state.startedAt = slot.childStartedAt;
+            if (unit.child.pid !== undefined)
+                slot.state.pid = unit.child.pid;
+            watch(slot, unit.child);
+            slots.set(unit.id, slot);
+            deps.log("info", `adopted ${unit.id} (pid ${unit.child.pid ?? "?"}) from the previous daemon`);
+        }
+    };
+    const handleRequest = (req) => {
+        if (req.action === "reconcile") {
+            deps.log("info", "reconciling daemon configuration (requested)");
+            reconcile();
+            return;
+        }
+        if (req.action === "restart-daemon") {
+            deps.log("info", "restarting the daemon (requested)");
+            if (deps.restartDaemon)
+                deps.restartDaemon();
+            else
+                deps.log("warn", "this daemon cannot restart itself; restart it to load the new version");
+            return;
+        }
+        // A bot created since the last plan read is asked for before the timer
+        // would pick it up; read the plan now rather than dropping the request.
+        const unplanned = req.unit !== undefined && !slots.has(req.unit);
+        if (unplanned)
+            reconcile();
+        // The re-read already started a new long-running unit; restarting it would only bounce it.
+        if (unplanned && req.action === "restart" && slots.get(req.unit)?.child) {
+            emit();
+            return;
+        }
+        const targets = req.unit === undefined
+            ? [...slots.values()].filter((s) => s.plan.kind === "long-running" && !s.plan.blocked)
+            : [slots.get(req.unit)].filter((s) => s !== undefined);
+        if (targets.length === 0) {
+            deps.log("warn", `ignoring ${req.action} request for unknown unit ${req.unit ?? "(all)"}`);
+            return;
+        }
+        for (const slot of targets) {
+            if (slot.plan.blocked) {
+                deps.log("warn", `ignoring ${req.action} for ${slot.plan.id}: ${slot.plan.blocked.reason}`);
+                continue;
+            }
+            if (req.action === "stop") {
+                deps.log("info", `stopping ${slot.plan.id} (requested)`);
+                // An expected exit is never restarted, including one that needed SIGKILL.
+                terminate(slot, () => {
+                    slot.state.status = "exited";
+                    slot.state.reason = `stopped on request; \`hq daemon restart ${slot.plan.id}\` starts it again`;
+                    delete slot.state.nextRunAt;
+                });
+                if (!slot.child) {
+                    slot.state.status = "exited";
+                    slot.state.reason = `stopped on request; \`hq daemon restart ${slot.plan.id}\` starts it again`;
+                    delete slot.state.nextRunAt;
+                }
+                continue;
+            }
+            if (req.action === "trigger") {
+                if (slot.plan.kind !== "scheduled") {
+                    deps.log("warn", `ignoring trigger for ${slot.plan.id}: only scheduled services can be triggered`);
+                    continue;
+                }
+                deps.log("info", `running ${slot.plan.id} now (requested)`);
+                runScheduled(slot, true);
+                continue;
+            }
+            deps.log("info", `restarting ${slot.plan.id} (requested)`);
+            slot.attempts = 0;
+            if (slot.plan.kind === "scheduled")
+                continue;
+            if (slot.child)
+                terminate(slot, () => startLong(slot));
+            else {
+                clearTimer(slot);
+                startLong(slot);
+            }
+        }
+        emit();
+    };
+    const pollRequests = () => {
+        requestTimer = undefined;
+        if (stopped)
+            return;
+        try {
+            for (const req of deps.takeRequests())
+                handleRequest(req);
+        }
+        catch (err) {
+            deps.log("error", `could not read control requests: ${errorMessage(err)}`);
+        }
+        requestTimer = setT(pollRequests, requestPollMs);
+    };
+    const reconcileLoop = () => {
+        reconcileTimer = undefined;
+        if (stopped)
+            return;
+        reconcile();
+        reconcileTimer = setT(reconcileLoop, reconcileMs);
+    };
+    return {
+        start() {
+            if (started)
+                return;
+            started = true;
+            adoptPrevious();
+            reconcile();
+            reconcileTimer = setT(reconcileLoop, reconcileMs);
+            requestTimer = setT(pollRequests, requestPollMs);
+        },
+        stop() {
+            if (drained)
+                return drained;
+            stopped = true;
+            if (reconcileTimer !== undefined)
+                clearT(reconcileTimer);
+            if (requestTimer !== undefined)
+                clearT(requestTimer);
+            if (handoffTimer !== undefined)
+                clearT(handoffTimer);
+            for (const slot of slots.values()) {
+                clearTimer(slot);
+                clearHeartbeatTimer(slot);
+                slot.child = undefined;
+                if (slot.killTimer !== undefined)
+                    clearT(slot.killTimer);
+                slot.killTimer = undefined;
+                slot.state.status = "stopped";
+                delete slot.state.pid;
+                delete slot.state.nextRunAt;
+            }
+            const children = [...live];
+            for (const child of children) {
+                try {
+                    child.kill("SIGTERM");
+                }
+                catch (err) {
+                    deps.log("error", `could not stop pid ${child.pid ?? "?"}: ${errorMessage(err)}`);
+                }
+            }
+            emit();
+            drained = new Promise((resolve) => {
+                if (live.size === 0) {
+                    resolve();
+                    return;
+                }
+                const escalation = setT(() => {
+                    for (const child of live) {
+                        deps.log("warn", `pid ${child.pid ?? "?"} ignored SIGTERM; sending SIGKILL`);
+                        try {
+                            child.kill("SIGKILL");
+                        }
+                        catch (err) {
+                            deps.log("error", `could not SIGKILL pid ${child.pid ?? "?"}: ${errorMessage(err)}`);
+                        }
+                    }
+                }, STOP_ESCALATION_MS);
+                onDrained = () => {
+                    onDrained = null;
+                    clearT(escalation);
+                    resolve();
+                };
+            });
+            return drained;
+        },
+        reconcile,
+        state: snapshot,
+        release() {
+            const released = [];
+            for (const slot of slots.values()) {
+                const pid = slot.child?.pid;
+                if (slot.plan.kind === "long-running" && pid !== undefined && slot.childKey !== undefined) {
+                    released.push({ id: slot.plan.id, pid, key: slot.childKey });
+                }
+            }
+            stopped = true;
+            if (reconcileTimer !== undefined)
+                clearT(reconcileTimer);
+            if (requestTimer !== undefined)
+                clearT(requestTimer);
+            if (handoffTimer !== undefined)
+                clearT(handoffTimer);
+            for (const slot of slots.values()) {
+                clearTimer(slot);
+                if (slot.killTimer !== undefined)
+                    clearT(slot.killTimer);
+                slot.killTimer = undefined;
+                slot.child = undefined;
+                slot.state.status = "stopped";
+                delete slot.state.pid;
+                delete slot.state.nextRunAt;
+            }
+            emit();
+            return released;
+        },
+    };
+}
+//# sourceMappingURL=supervisor.js.map