@indigoai-us/hq-cli 5.312.2-beta.0 → 5.314.0

package/dist/lib/daemon/supervisor.js

@@ -1,836 +0,0 @@
-/**
- * The `hq daemon run` supervisor. It keeps the planned units running:
- *
- * - long-running units are restarted after they exit, with per-unit backoff
- *   (1 s doubling to 60 s, reset once a unit has stayed up for a minute);
- *   `on-failure` units stay down after a clean exit or a SIGTERM/SIGINT from
- *   outside, and any unit stays down after a stop request (`hq bot stop`),
- *   until a restart request;
- * - scheduled units run after their initial delay and then on their interval,
- *   never overlapping themselves;
- * - the plan is re-read on a timer, so disabled services stop, new bots start,
- *   and units whose requirement appears or disappears follow on their own;
- * - restart/trigger/stop requests from the CLI are polled and applied; a
- *   request for a unit that is not planned yet re-reads the plan first;
- * - stop() sends SIGTERM to every child, escalates to SIGKILL after
- *   STOP_ESCALATION_MS, and resolves once every child has exited;
- * - release() instead leaves every child running and reports the long-running
- *   ones, so the next daemon can adopt them (`adopt`). Adopted units whose
- *   command is unchanged keep running; changed ones are replaced one at a
- *   time, draining first when the unit has a drain signal.
- *
- * Processes, time, and state output are injected; `run.ts` wires the real ones.
- */
-import { StringDecoder } from "node:string_decoder";
-import { SYNC_RUNNER_LEASE_CONFLICT_EXIT, SYNC_RUNNER_LEASE_RETRY_MS, parseSyncRunnerProtocolLine, syncPassResult, } from "./sync-runner.js";
-const BACKOFF_BASE_MS = 1_000;
-const BACKOFF_MAX_MS = 60_000;
-const HEALTHY_RUN_MS = 60_000;
-export const STOP_ESCALATION_MS = 10_000;
-const MESH_OWNER_RECHECK_MS = 15_000;
-const MESH_OWNER_LOG_INTERVAL_MS = 10 * 60_000;
-/** After a handoff, how long a replaced unit must stay up before the next one is replaced. */
-export const HANDOFF_STAGGER_MS = 15_000;
-/** How long a draining unit may take to finish its work before it gets SIGTERM. */
-export const HANDOFF_DRAIN_MAX_MS = 15 * 60_000;
-/** Plan reads in a row that must find a running remote-control host blocked before it is paused. */
-const BLOCKED_READS_BEFORE_PAUSE = 2;
-function backoffMs(attempt) {
-    return Math.min(BACKOFF_MAX_MS, BACKOFF_BASE_MS * 2 ** Math.max(0, attempt));
-}
-/**
- * What a unit's process was started from; a unit whose key changes is
- * restarted. The CLI version counts only for `hq` commands, so a CLI update
- * leaves hq-cloud runners, other programs, and units marked
- * `keepAcrossCliUpdates` running.
- */
-export function planKey(plan, cliVersion) {
-    const fields = [
-        plan.kind,
-        plan.restart,
-        plan.intervalSec,
-        plan.initialDelaySec,
-        plan.heartbeatTimeoutMs,
-        plan.command,
-        plan.env,
-    ];
-    if (cliVersion !== undefined && plan.command?.type === "hq" && !plan.keepAcrossCliUpdates)
-        fields.push(cliVersion);
-    return JSON.stringify(fields);
-}
-function errorMessage(err) {
-    return err instanceof Error ? err.message : String(err);
-}
-export function createSupervisor(deps) {
-    const now = deps.now ?? Date.now;
-    const setT = deps.setTimeout ?? ((fn, ms) => setTimeout(fn, ms));
-    const clearT = deps.clearTimeout ?? ((h) => clearTimeout(h));
-    const reconcileMs = deps.reconcileMs ?? 15_000;
-    const requestPollMs = deps.requestPollMs ?? 1_000;
-    const startedAt = now();
-    let slots = new Map();
-    let stopped = false;
-    let started = false;
-    let reconcileTimer;
-    let requestTimer;
-    /** Adopted units whose command changed, replaced one at a time. */
-    const handoffQueue = [];
-    let handoffActive = false;
-    let handoffTimer;
-    /** Every child process not yet known to have exited, including ones the slots have let go of. */
-    const live = new Set();
-    let drained = null;
-    let onDrained = null;
-    const snapshot = () => ({
-        version: 1,
-        pid: deps.pid,
-        startedAt,
-        updatedAt: now(),
-        machineType: deps.machineType,
-        ...(deps.host ? { host: deps.host } : {}),
-        units: [...slots.values()].map((s) => ({ ...s.state })),
-    });
-    const emit = () => {
-        try {
-            deps.writeState(snapshot());
-        }
-        catch (err) {
-            deps.log("error", `could not write daemon state: ${errorMessage(err)}`);
-        }
-    };
-    const clearTimer = (slot) => {
-        if (slot.timer !== undefined)
-            clearT(slot.timer);
-        slot.timer = undefined;
-    };
-    const clearHeartbeatTimer = (slot) => {
-        if (slot.heartbeatTimer !== undefined)
-            clearT(slot.heartbeatTimer);
-        slot.heartbeatTimer = undefined;
-    };
-    const setBlocked = (slot) => {
-        const block = slot.plan.blocked;
-        slot.state.status = block.state;
-        slot.state.reason = block.reason;
-        delete slot.state.pid;
-        delete slot.state.nextRunAt;
-    };
-    const signalChild = (slot, child, signal) => {
-        try {
-            child.kill(signal);
-        }
-        catch (err) {
-            deps.log("error", `could not send ${signal} to ${slot.plan.id}: ${errorMessage(err)}`);
-        }
-    };
-    /**
-     * Stop the slot's child (if any), then run `then` once it has exited. With
-     * `drain`, a unit that has a drain signal gets it first and SIGTERM only
-     * after HANDOFF_DRAIN_MAX_MS; SIGKILL follows SIGTERM after STOP_ESCALATION_MS.
-     */
-    const terminate = (slot, then, drain = false) => {
-        clearTimer(slot);
-        clearHeartbeatTimer(slot);
-        const child = slot.child;
-        if (!child) {
-            then?.();
-            return;
-        }
-        slot.expectingExit = true;
-        slot.afterExit = then;
-        if (slot.killTimer !== undefined)
-            clearT(slot.killTimer);
-        const escalateToKill = () => {
-            slot.killTimer = setT(() => {
-                slot.killTimer = undefined;
-                if (slot.child !== child)
-                    return;
-                deps.log("warn", `${slot.plan.id} ignored SIGTERM; sending SIGKILL`);
-                signalChild(slot, child, "SIGKILL");
-            }, STOP_ESCALATION_MS);
-        };
-        const drainSignal = drain ? slot.plan.drainSignal : undefined;
-        if (drainSignal) {
-            slot.killTimer = setT(() => {
-                slot.killTimer = undefined;
-                if (slot.child !== child)
-                    return;
-                deps.log("warn", `${slot.plan.id} did not finish draining; sending SIGTERM`);
-                escalateToKill();
-                signalChild(slot, child, "SIGTERM");
-            }, HANDOFF_DRAIN_MAX_MS);
-            signalChild(slot, child, drainSignal);
-            return;
-        }
-        escalateToKill();
-        signalChild(slot, child, "SIGTERM");
-    };
-    const scheduleRestart = (slot) => {
-        if (stopped)
-            return;
-        const delay = backoffMs(slot.attempts);
-        slot.attempts += 1;
-        slot.state.restarts += 1;
-        slot.state.status = "backoff";
-        slot.state.nextRunAt = now() + delay;
-        deps.log("info", `restarting ${slot.plan.id} in ${delay}ms (restart ${slot.state.restarts})`);
-        slot.timer = setT(() => {
-            slot.timer = undefined;
-            startLong(slot);
-            emit();
-        }, delay);
-    };
-    const watch = (slot, child) => {
-        live.add(child);
-        let ended = false;
-        const timeoutMs = slot.plan.heartbeatTimeoutMs;
-        const decoder = timeoutMs ? new StringDecoder("utf8") : null;
-        let stdoutRemainder = "";
-        let lastHeartbeatPersistedAt = now() - 15_000;
-        const armHeartbeatWatchdog = () => {
-            if (!timeoutMs)
-                return;
-            clearHeartbeatTimer(slot);
-            slot.heartbeatTimer = setT(() => {
-                slot.heartbeatTimer = undefined;
-                if (stopped || slot.child !== child || slot.expectingExit)
-                    return;
-                deps.log("warn", `${slot.plan.id} emitted no sync protocol heartbeat for ${timeoutMs}ms; restarting it`);
-                terminate(slot, () => scheduleRestart(slot));
-                emit();
-            }, timeoutMs);
-        };
-        const acceptProtocolLine = (line) => {
-            let event;
-            try {
-                event = parseSyncRunnerProtocolLine(line);
-            }
-            catch (err) {
-                deps.log("error", `${slot.plan.id} emitted invalid sync protocol output (${Buffer.byteLength(line)} bytes): ${errorMessage(err)}`);
-                return;
-            }
-            if (!event)
-                return;
-            const at = now();
-            slot.state.lastHeartbeatAt = new Date(at).toISOString();
-            const pass = syncPassResult(event, slot.state.lastHeartbeatAt);
-            if (pass)
-                slot.state.lastPassResult = pass;
-            armHeartbeatWatchdog();
-            if (pass || at - lastHeartbeatPersistedAt >= 15_000) {
-                lastHeartbeatPersistedAt = at;
-                emit();
-            }
-        };
-        if (decoder && child.stdout) {
-            child.stdout.on("data", (chunk) => {
-                if (chunk === undefined)
-                    return;
-                stdoutRemainder += decoder.write(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
-                let newline = stdoutRemainder.indexOf("\n");
-                while (newline >= 0) {
-                    acceptProtocolLine(stdoutRemainder.slice(0, newline).trim());
-                    stdoutRemainder = stdoutRemainder.slice(newline + 1);
-                    newline = stdoutRemainder.indexOf("\n");
-                }
-                // A malformed or unexpectedly large line is not a heartbeat. Bound
-                // retained data so a broken child cannot grow daemon memory forever.
-                if (Buffer.byteLength(stdoutRemainder) > 1024 * 1024)
-                    stdoutRemainder = "";
-            });
-            child.stdout.on("end", () => {
-                stdoutRemainder += decoder.end();
-                if (stdoutRemainder)
-                    acceptProtocolLine(stdoutRemainder.trim());
-                stdoutRemainder = "";
-            });
-        }
-        armHeartbeatWatchdog();
-        const onEnd = (code, signal) => {
-            if (ended)
-                return;
-            ended = true;
-            live.delete(child);
-            if (live.size === 0)
-                onDrained?.();
-            if (slot.child !== child)
-                return;
-            slot.child = undefined;
-            clearHeartbeatTimer(slot);
-            if (slot.killTimer !== undefined) {
-                clearT(slot.killTimer);
-                slot.killTimer = undefined;
-            }
-            const ranFor = now() - (slot.childStartedAt ?? now());
-            slot.state.lastExit = { code, signal, at: now() };
-            delete slot.state.pid;
-            if (slot.expectingExit) {
-                slot.expectingExit = false;
-                slot.busy = false;
-                const then = slot.afterExit;
-                slot.afterExit = undefined;
-                then?.();
-                emit();
-                return;
-            }
-            if (stopped)
-                return;
-            if (slot.plan.kind === "scheduled") {
-                slot.busy = false;
-                slot.state.lastResult = code === 0 ? "ok" : "failed";
-                slot.state.status = "idle";
-                if (code !== 0)
-                    deps.log("warn", `${slot.plan.id} run failed (code=${code ?? "null"} signal=${signal ?? "none"})`);
-                emit();
-                return;
-            }
-            if (slot.plan.service === "sync" && code === SYNC_RUNNER_LEASE_CONFLICT_EXIT) {
-                const reason = deps.syncOwnerBlockReason?.() ?? "sync owned by another runner";
-                slot.state.status = "waiting";
-                slot.state.reason = reason;
-                slot.state.nextRunAt = now() + SYNC_RUNNER_LEASE_RETRY_MS;
-                deps.log("info", `${reason}; retrying in ${SYNC_RUNNER_LEASE_RETRY_MS}ms`);
-                slot.timer = setT(() => {
-                    slot.timer = undefined;
-                    startLong(slot);
-                    emit();
-                }, SYNC_RUNNER_LEASE_RETRY_MS);
-                emit();
-                return;
-            }
-            const meshOwnerReason = slot.plan.service === "mesh" ? deps.meshOwnerReason?.() : undefined;
-            if (meshOwnerReason) {
-                let lastLoggedReason;
-                let lastLoggedAt = Number.NEGATIVE_INFINITY;
-                const waitForOwner = (reason) => {
-                    slot.state.status = "waiting";
-                    slot.state.reason = reason;
-                    slot.state.nextRunAt = now() + MESH_OWNER_RECHECK_MS;
-                    const at = now();
-                    if (reason !== lastLoggedReason || at - lastLoggedAt >= MESH_OWNER_LOG_INTERVAL_MS) {
-                        deps.log("warn", `${reason}; retrying mesh ownership check in ${MESH_OWNER_RECHECK_MS}ms`);
-                        lastLoggedReason = reason;
-                        lastLoggedAt = at;
-                    }
-                    slot.timer = setT(() => {
-                        slot.timer = undefined;
-                        const current = deps.meshOwnerReason?.();
-                        if (current)
-                            waitForOwner(current);
-                        else {
-                            slot.attempts = 0;
-                            delete slot.state.reason;
-                            delete slot.state.nextRunAt;
-                            startLong(slot);
-                        }
-                        emit();
-                    }, MESH_OWNER_RECHECK_MS);
-                };
-                waitForOwner(meshOwnerReason);
-                emit();
-                return;
-            }
-            deps.log("warn", `${slot.plan.id} exited (code=${code ?? "null"} signal=${signal ?? "none"})`);
-            if (ranFor >= HEALTHY_RUN_MS)
-                slot.attempts = 0;
-            const stoppedFromOutside = signal === "SIGTERM" || signal === "SIGINT";
-            if (slot.plan.restart === "on-failure" && (code === 0 || stoppedFromOutside)) {
-                slot.state.status = "exited";
-                if (stoppedFromOutside) {
-                    slot.state.reason = `stopped outside the daemon; \`hq daemon restart ${slot.plan.id}\` starts it again`;
-                }
-                delete slot.state.nextRunAt;
-            }
-            else {
-                scheduleRestart(slot);
-            }
-            emit();
-        };
-        child.on("exit", onEnd);
-        child.on("error", (err) => {
-            deps.log("error", `${slot.plan.id}: ${errorMessage(err)}`);
-            // A child that never got a pid failed to launch and may never emit
-            // "exit". Any other error (for example a failed kill) leaves it running.
-            if (child.pid === undefined)
-                onEnd(null, null);
-        });
-    };
-    const startLong = (slot) => {
-        if (stopped)
-            return;
-        let child;
-        try {
-            child = deps.spawn(slot.plan);
-        }
-        catch (err) {
-            slot.state.reason = `failed to start: ${errorMessage(err)}`;
-            deps.log("error", `${slot.plan.id} ${slot.state.reason}`);
-            scheduleRestart(slot);
-            return;
-        }
-        slot.child = child;
-        slot.childKey = slot.key;
-        slot.childStartedAt = now();
-        slot.state.status = "running";
-        slot.state.startedAt = slot.childStartedAt;
-        delete slot.state.reason;
-        delete slot.state.nextRunAt;
-        if (child.pid !== undefined)
-            slot.state.pid = child.pid;
-        else
-            delete slot.state.pid;
-        watch(slot, child);
-    };
-    const scheduleNextRun = (slot, delayMs) => {
-        clearTimer(slot);
-        slot.state.nextRunAt = now() + delayMs;
-        slot.timer = setT(() => {
-            slot.timer = undefined;
-            runScheduled(slot);
-        }, delayMs);
-    };
-    const runScheduled = (slot, manual = false) => {
-        if (stopped)
-            return;
-        if (!manual)
-            scheduleNextRun(slot, (slot.plan.intervalSec ?? 60) * 1000);
-        if (slot.busy) {
-            deps.log("info", `${slot.plan.id} is still running; skipping this run`);
-            emit();
-            return;
-        }
-        slot.busy = true;
-        slot.state.lastRunAt = now();
-        const command = slot.plan.command;
-        if (command?.type === "internal") {
-            slot.state.status = "running";
-            deps
-                .runInternal(command.task)
-                .then((ok) => {
-                slot.state.lastResult = ok ? "ok" : "failed";
-            }, (err) => {
-                deps.log("error", `${slot.plan.id} failed: ${errorMessage(err)}`);
-                slot.state.lastResult = "failed";
-            })
-                .finally(() => {
-                slot.busy = false;
-                if (!stopped && !slot.plan.blocked)
-                    slot.state.status = "idle";
-                emit();
-            });
-            emit();
-            return;
-        }
-        let child;
-        try {
-            child = deps.spawn(slot.plan);
-        }
-        catch (err) {
-            slot.busy = false;
-            slot.state.lastResult = "failed";
-            slot.state.status = "idle";
-            slot.state.reason = `failed to start: ${errorMessage(err)}`;
-            deps.log("error", `${slot.plan.id} ${slot.state.reason}`);
-            emit();
-            return;
-        }
-        slot.child = child;
-        slot.childKey = slot.key;
-        slot.childStartedAt = now();
-        slot.state.status = "running";
-        delete slot.state.reason;
-        if (child.pid !== undefined)
-            slot.state.pid = child.pid;
-        watch(slot, child);
-        emit();
-    };
-    const activate = (slot) => {
-        if (slot.plan.blocked) {
-            setBlocked(slot);
-            return;
-        }
-        if (slot.plan.kind === "scheduled") {
-            slot.state.status = "idle";
-            delete slot.state.reason;
-            const delay = (slot.plan.initialDelaySec ?? 0) * 1000;
-            if (delay === 0)
-                runScheduled(slot);
-            else
-                scheduleNextRun(slot, delay);
-            return;
-        }
-        startLong(slot);
-    };
-    const newSlot = (plan) => ({
-        plan,
-        key: planKey(plan, deps.cliVersion),
-        attempts: 0,
-        expectingExit: false,
-        busy: false,
-        state: {
-            id: plan.id,
-            service: plan.service,
-            kind: plan.kind,
-            description: plan.description,
-            status: "idle",
-            restarts: 0,
-        },
-    });
-    const reconcile = () => {
-        if (stopped)
-            return;
-        let plans;
-        try {
-            const result = deps.plan();
-            if (!Array.isArray(result))
-                throw new Error("plan did not return a list of units");
-            plans = result;
-        }
-        catch (err) {
-            deps.log("error", `could not plan units; keeping current ones: ${errorMessage(err)}`);
-            return;
-        }
-        const wanted = new Set(plans.map((p) => p.id));
-        for (const [id, slot] of slots) {
-            if (!wanted.has(id)) {
-                deps.log("info", `stopping ${id}: no longer enabled`);
-                terminate(slot);
-            }
-        }
-        const next = new Map();
-        for (const plan of plans) {
-            const existing = slots.get(plan.id);
-            if (!existing) {
-                const slot = newSlot(plan);
-                next.set(plan.id, slot);
-                activate(slot);
-                continue;
-            }
-            next.set(plan.id, existing);
-            // A remote-control host is what people's sessions run in. Its tool can
-            // look missing for one plan read while it is being reinstalled, so it is
-            // only paused once the tool is missing on two reads in a row.
-            if (plan.blocked && existing.plan.blocked === undefined && existing.child && plan.keepAcrossCliUpdates) {
-                existing.blockedReads = (existing.blockedReads ?? 0) + 1;
-                if (existing.blockedReads < BLOCKED_READS_BEFORE_PAUSE)
-                    continue;
-            }
-            existing.blockedReads = 0;
-            const adopted = existing.adopted === true;
-            existing.adopted = false;
-            const key = planKey(plan, deps.cliVersion);
-            const wasBlocked = existing.plan.blocked !== undefined;
-            const changed = key !== existing.key;
-            existing.plan = plan;
-            existing.key = key;
-            existing.state.description = plan.description;
-            existing.state.kind = plan.kind;
-            if (plan.blocked) {
-                if (!wasBlocked) {
-                    deps.log("info", `pausing ${plan.id}: ${plan.blocked.reason}`);
-                    terminate(existing);
-                }
-                setBlocked(existing);
-            }
-            else if (wasBlocked) {
-                deps.log("info", `starting ${plan.id}: its requirement is now met`);
-                existing.attempts = 0;
-                // The copy stopped when it was paused may still be exiting; start the
-                // new one only after it is gone, so two never run side by side.
-                if (existing.child)
-                    existing.afterExit = () => activate(existing);
-                else
-                    activate(existing);
-            }
-            else if (changed && adopted && existing.child) {
-                deps.log("info", `will replace ${plan.id}: it runs an older CLI or command`);
-                handoffQueue.push(existing);
-            }
-            else if (changed) {
-                deps.log("info", `restarting ${plan.id}: its command changed`);
-                existing.attempts = 0;
-                existing.state.restarts = 0;
-                terminate(existing, () => activate(existing));
-            }
-        }
-        slots = next;
-        emit();
-        pumpHandoff();
-    };
-    /** Replace the next queued adopted unit once the previous replacement has settled. */
-    const pumpHandoff = () => {
-        if (stopped || handoffActive)
-            return;
-        let slot;
-        while ((slot = handoffQueue.shift())) {
-            // Skip units that left the plan, were paused, or already restarted on their own.
-            if (slots.get(slot.plan.id) === slot && !slot.plan.blocked && slot.child && slot.childKey !== slot.key)
-                break;
-        }
-        if (!slot)
-            return;
-        const target = slot;
-        handoffActive = true;
-        deps.log("info", `replacing ${target.plan.id}${target.plan.drainSignal ? " once it finishes its current work" : ""}`);
-        target.attempts = 0;
-        target.state.restarts = 0;
-        terminate(target, () => activate(target), true);
-        const settle = (delay) => {
-            handoffTimer = setT(() => {
-                handoffTimer = undefined;
-                if (stopped)
-                    return;
-                const gone = slots.get(target.plan.id) !== target || target.plan.blocked !== undefined;
-                if (!gone && target.expectingExit)
-                    return settle(HANDOFF_STAGGER_MS);
-                if (!gone && target.child) {
-                    const ranFor = now() - (target.childStartedAt ?? now());
-                    if (ranFor < HANDOFF_STAGGER_MS)
-                        return settle(HANDOFF_STAGGER_MS - ranFor);
-                }
-                else if (!gone && target.state.status === "backoff") {
-                    return settle(HANDOFF_STAGGER_MS);
-                }
-                handoffActive = false;
-                pumpHandoff();
-            }, delay);
-        };
-        settle(HANDOFF_STAGGER_MS);
-    };
-    /** Stop a process the plan has no place for. */
-    const stopStray = (id, child) => {
-        const slot = newSlot({ id, service: id, description: id, kind: "long-running", restart: "always", command: null, env: {} });
-        slot.child = child;
-        watch(slot, child);
-        terminate(slot);
-    };
-    /** Take over the previous daemon's services before the first plan read starts anything. */
-    const adoptPrevious = () => {
-        const adopt = deps.adopt ?? [];
-        if (adopt.length === 0)
-            return;
-        let plans = [];
-        try {
-            const result = deps.plan();
-            if (!Array.isArray(result))
-                throw new Error("plan did not return a list of units");
-            plans = result;
-        }
-        catch (err) {
-            deps.log("error", `could not plan units; stopping the services left by the previous daemon: ${errorMessage(err)}`);
-        }
-        const byId = new Map(plans.map((p) => [p.id, p]));
-        for (const unit of adopt) {
-            const plan = byId.get(unit.id);
-            if (!plan || plan.kind !== "long-running" || slots.has(unit.id)) {
-                deps.log("info", `stopping ${unit.id} left by the previous daemon: no longer planned as a service`);
-                stopStray(unit.id, unit.child);
-                continue;
-            }
-            // Recorded as runnable, so the first plan read pauses it when it is now blocked.
-            const slot = newSlot({ ...plan, blocked: undefined });
-            slot.key = unit.key;
-            slot.adopted = true;
-            slot.child = unit.child;
-            slot.childKey = unit.key;
-            slot.childStartedAt = now();
-            slot.state.status = "running";
-            slot.state.startedAt = slot.childStartedAt;
-            if (unit.child.pid !== undefined)
-                slot.state.pid = unit.child.pid;
-            watch(slot, unit.child);
-            slots.set(unit.id, slot);
-            deps.log("info", `adopted ${unit.id} (pid ${unit.child.pid ?? "?"}) from the previous daemon`);
-        }
-    };
-    const handleRequest = (req) => {
-        if (req.action === "reconcile") {
-            deps.log("info", "reconciling daemon configuration (requested)");
-            reconcile();
-            return;
-        }
-        if (req.action === "restart-daemon") {
-            deps.log("info", "restarting the daemon (requested)");
-            if (deps.restartDaemon)
-                deps.restartDaemon();
-            else
-                deps.log("warn", "this daemon cannot restart itself; restart it to load the new version");
-            return;
-        }
-        // A bot created since the last plan read is asked for before the timer
-        // would pick it up; read the plan now rather than dropping the request.
-        const unplanned = req.unit !== undefined && !slots.has(req.unit);
-        if (unplanned)
-            reconcile();
-        // The re-read already started a new long-running unit; restarting it would only bounce it.
-        if (unplanned && req.action === "restart" && slots.get(req.unit)?.child) {
-            emit();
-            return;
-        }
-        const targets = req.unit === undefined
-            ? [...slots.values()].filter((s) => s.plan.kind === "long-running" && !s.plan.blocked)
-            : [slots.get(req.unit)].filter((s) => s !== undefined);
-        if (targets.length === 0) {
-            deps.log("warn", `ignoring ${req.action} request for unknown unit ${req.unit ?? "(all)"}`);
-            return;
-        }
-        for (const slot of targets) {
-            if (slot.plan.blocked) {
-                deps.log("warn", `ignoring ${req.action} for ${slot.plan.id}: ${slot.plan.blocked.reason}`);
-                continue;
-            }
-            if (req.action === "stop") {
-                deps.log("info", `stopping ${slot.plan.id} (requested)`);
-                // An expected exit is never restarted, including one that needed SIGKILL.
-                terminate(slot, () => {
-                    slot.state.status = "exited";
-                    slot.state.reason = `stopped on request; \`hq daemon restart ${slot.plan.id}\` starts it again`;
-                    delete slot.state.nextRunAt;
-                });
-                if (!slot.child) {
-                    slot.state.status = "exited";
-                    slot.state.reason = `stopped on request; \`hq daemon restart ${slot.plan.id}\` starts it again`;
-                    delete slot.state.nextRunAt;
-                }
-                continue;
-            }
-            if (req.action === "trigger") {
-                if (slot.plan.kind !== "scheduled") {
-                    deps.log("warn", `ignoring trigger for ${slot.plan.id}: only scheduled services can be triggered`);
-                    continue;
-                }
-                deps.log("info", `running ${slot.plan.id} now (requested)`);
-                runScheduled(slot, true);
-                continue;
-            }
-            deps.log("info", `restarting ${slot.plan.id} (requested)`);
-            slot.attempts = 0;
-            if (slot.plan.kind === "scheduled")
-                continue;
-            if (slot.child)
-                terminate(slot, () => startLong(slot));
-            else {
-                clearTimer(slot);
-                startLong(slot);
-            }
-        }
-        emit();
-    };
-    const pollRequests = () => {
-        requestTimer = undefined;
-        if (stopped)
-            return;
-        try {
-            for (const req of deps.takeRequests())
-                handleRequest(req);
-        }
-        catch (err) {
-            deps.log("error", `could not read control requests: ${errorMessage(err)}`);
-        }
-        requestTimer = setT(pollRequests, requestPollMs);
-    };
-    const reconcileLoop = () => {
-        reconcileTimer = undefined;
-        if (stopped)
-            return;
-        reconcile();
-        reconcileTimer = setT(reconcileLoop, reconcileMs);
-    };
-    return {
-        start() {
-            if (started)
-                return;
-            started = true;
-            adoptPrevious();
-            reconcile();
-            reconcileTimer = setT(reconcileLoop, reconcileMs);
-            requestTimer = setT(pollRequests, requestPollMs);
-        },
-        stop() {
-            if (drained)
-                return drained;
-            stopped = true;
-            if (reconcileTimer !== undefined)
-                clearT(reconcileTimer);
-            if (requestTimer !== undefined)
-                clearT(requestTimer);
-            if (handoffTimer !== undefined)
-                clearT(handoffTimer);
-            for (const slot of slots.values()) {
-                clearTimer(slot);
-                clearHeartbeatTimer(slot);
-                slot.child = undefined;
-                if (slot.killTimer !== undefined)
-                    clearT(slot.killTimer);
-                slot.killTimer = undefined;
-                slot.state.status = "stopped";
-                delete slot.state.pid;
-                delete slot.state.nextRunAt;
-            }
-            const children = [...live];
-            for (const child of children) {
-                try {
-                    child.kill("SIGTERM");
-                }
-                catch (err) {
-                    deps.log("error", `could not stop pid ${child.pid ?? "?"}: ${errorMessage(err)}`);
-                }
-            }
-            emit();
-            drained = new Promise((resolve) => {
-                if (live.size === 0) {
-                    resolve();
-                    return;
-                }
-                const escalation = setT(() => {
-                    for (const child of live) {
-                        deps.log("warn", `pid ${child.pid ?? "?"} ignored SIGTERM; sending SIGKILL`);
-                        try {
-                            child.kill("SIGKILL");
-                        }
-                        catch (err) {
-                            deps.log("error", `could not SIGKILL pid ${child.pid ?? "?"}: ${errorMessage(err)}`);
-                        }
-                    }
-                }, STOP_ESCALATION_MS);
-                onDrained = () => {
-                    onDrained = null;
-                    clearT(escalation);
-                    resolve();
-                };
-            });
-            return drained;
-        },
-        reconcile,
-        state: snapshot,
-        release() {
-            const released = [];
-            for (const slot of slots.values()) {
-                const pid = slot.child?.pid;
-                if (slot.plan.kind === "long-running" && pid !== undefined && slot.childKey !== undefined) {
-                    released.push({ id: slot.plan.id, pid, key: slot.childKey });
-                }
-            }
-            stopped = true;
-            if (reconcileTimer !== undefined)
-                clearT(reconcileTimer);
-            if (requestTimer !== undefined)
-                clearT(requestTimer);
-            if (handoffTimer !== undefined)
-                clearT(handoffTimer);
-            for (const slot of slots.values()) {
-                clearTimer(slot);
-                if (slot.killTimer !== undefined)
-                    clearT(slot.killTimer);
-                slot.killTimer = undefined;
-                slot.child = undefined;
-                slot.state.status = "stopped";
-                delete slot.state.pid;
-                delete slot.state.nextRunAt;
-            }
-            emit();
-            return released;
-        },
-    };
-}
-//# sourceMappingURL=supervisor.js.map