@indigoai-us/hq-cli 5.345.156 → 5.345.160

package/dist/lib/daemon/run.js

@@ -56,6 +56,7 @@ import { adoptChild, createChildTracker, finishSuccession, handoffEligible, read
 import { listSyncRunnerProcesses, readWatchRunnerOwnerStatus, syncHeartbeatTimeoutMs, syncRunnerNodeOptions, watchRunnerOwnerStatusPath, } from "./sync-runner.js";
 import { STOP_ESCALATION_MS, SYNC_RUNNER_PERSIST_GRACE_MS, createSupervisor, } from "./supervisor.js";
 import { userToolDirs } from "./tools.js";
+import { logOffset, readLogSince, readUnitHolds, writeUnitHolds } from "./unit-holds.js";
 import { daemonRunGuard } from "./lanes-reconcile-guard.js";
 export const LOG_ROTATE_BYTES = 5 * 1024 * 1024;
 /** Exit code for "restart me": non-zero so every service manager relaunches (Windows tasks only restart on failure). */
@@ -1002,10 +1003,13 @@ export async function runDaemon(opts) {
         }
         return plans;
     };
+    /** Where each unit's log ended when its current run started, so its exit is classified on that run's output only. */
+    const unitLogOffsets = new Map();
     const spawnUnit = (unit) => {
         const command = unit.command;
         if (!command || command.type === "internal")
             throw new Error(`${unit.id} has no process to start`);
+        unitLogOffsets.set(unit.id, logOffset(paths.unitLogPath(unit.id)));
         let [file, args] = command.type === "hq"
             ? daemonHqChildCommand(opts.nodeBinary, opts.hqBinary, command.args, stagedHqChildBootstrap?.bootstrapPath ?? null, cliPackagePrefix, childBootstrapHandoffWatchFlag.status().enabled)
             : command.type === "node"
@@ -1136,6 +1140,15 @@ export async function runDaemon(opts) {
         restartDaemon: () => restartDaemon(),
         runGuard: daemonRunGuard,
         writeState: (state) => writeJsonAtomic(paths.statePath, state),
+        holds: {
+            read: () => readUnitHolds(paths.holdsPath),
+            write: (holds) => writeUnitHolds(paths.holdsPath, holds),
+        },
+        exitOutputTail: (unitId) => {
+            // An adopted unit was started by an earlier daemon; its output start is unknown.
+            const offset = unitLogOffsets.get(unitId);
+            return offset === undefined ? undefined : readLogSince(paths.unitLogPath(unitId), offset);
+        },
         log,
         onSyncRunnerStopping: () => extendSyncShutdownCap?.(),
         syncOwnerBlockReason: () => {

package/dist/lib/daemon/supervisor.d.ts

@@ -3,6 +3,11 @@
  *
  * - long-running units are restarted after they exit, with per-unit backoff
  *   (1 s doubling to 60 s, reset once a unit has stayed up for a minute);
+ *   a unit that fails EPISODE_MAX_FAILURES times without a run of
+ *   EPISODE_HEALTHY_RESET_MS, or fails for a permanent cause (see exit-cause.ts), is held
+ *   instead. Permanent holds need a restart request, command change or CLI update.
+ *   Exhausted transient sync/mesh units make one durable supervised probe after
+ *   a 30-minute cooldown; a failed probe re-holds, and an explicit stop stays stopped;
  *   `on-failure` units stay down after a clean exit or a SIGTERM/SIGINT from
  *   outside, and any unit stays down after a stop request (`hq bot stop`),
  *   until a restart request;
@@ -26,6 +31,14 @@ import type { UnitPlan } from "./catalog.js";
 import type { MachineType } from "./machine-type.js";
 import type { ControlRequest } from "./requests.js";
 import { syncPassResult } from "./sync-runner.js";
+export declare const HEALTHY_RUN_MS = 60000;
+/**
+ * Failures in one episode before the unit is held: the first start plus five
+ * automatic retries. Matches the Desktop app's MAX_FAILURES.
+ */
+export declare const EPISODE_MAX_FAILURES = 6;
+/** A run this long ends the failure episode. Matches the Desktop app's HEALTHY_RUN_WINDOW. */
+export declare const EPISODE_HEALTHY_RESET_MS: number;
 export declare const STOP_ESCALATION_MS = 10000;
 export declare const SYNC_RUNNER_PERSIST_GRACE_MS = 60000;
 export declare const CLI_UPDATE_SYNC_IDLE_KILL_GRACE_MS = 60000;
@@ -42,7 +55,38 @@ export interface SupervisedChild {
     on(event: "error", handler: (err: Error) => void): unknown;
     kill(signal?: NodeJS.Signals | number): boolean;
 }
-type UnitStatus = "running" | "waiting" | "unavailable" | "backoff" | "exited" | "idle" | "stopped";
+type UnitStatus = "running" | "waiting" | "unavailable" | "backoff" | "exited" | "idle" | "stopped" | "held";
+/**
+ * Why a long-running unit is held down. `permanent`: the last exit had a cause
+ * a restart cannot fix. `exhausted`: EPISODE_MAX_FAILURES failures without a
+ * run of EPISODE_HEALTHY_RESET_MS. Persisted (see SupervisorDeps.holds)
+ * so a daemon restart keeps the unit held.
+ */
+export interface UnitHold {
+    kind: "permanent" | "exhausted";
+    /** A PermanentExitCause, or "transient" for an exhausted episode. */
+    cause: string;
+    detail: string;
+    /** Epoch ms when the hold started. */
+    since: number;
+    /** Failures in the episode that ended in the hold, the last one included. */
+    failures: number;
+    lastExit?: {
+        code: number | null;
+        signal: string | null;
+        at: number;
+    };
+    /** sha256 of the unit's plan key; a hold only applies to the command it was recorded for. */
+    keyHash: string;
+    /** Exhausted sync/mesh only: next durable supervised-probe deadline. */
+    retryAt?: number;
+    /** A probe claim remains durable until that running generation is healthy. */
+    probing?: boolean;
+    /** Explicit stop of a recovering unit; only an explicit restart clears it. */
+    retryStopped?: boolean;
+}
+/** The holds file: unit id to hold. */
+export type PersistedHolds = Record<string, UnitHold>;
 interface UnitState {
     id: string;
     service: string;
@@ -65,6 +109,8 @@ interface UnitState {
     lastPassResult?: ReturnType<typeof syncPassResult>;
     /** The pass-request folder the running child was started with; absent when it was not. */
     passRequestDir?: string;
+    /** Also retained during a supervised recovery probe or its explicit manual stop. */
+    hold?: UnitHold;
 }
 /** An app that runs the daemon as its own child process instead of an OS unit. */
 export type HostApp = "desktop";
@@ -119,6 +165,13 @@ export interface SupervisorDeps {
     adopt?: readonly AdoptedUnit[];
     /** Pre-run check for scheduled units that declare `runGuard`; a throw counts as `spawn`. */
     runGuard?: (unit: UnitPlan) => "spawn" | "idle" | "skip";
+    /** What a long-running unit wrote during the run that just ended, for exit classification. */
+    exitOutputTail?: (unitId: string) => string | undefined;
+    /** Durable storage for unit holds, so a held unit stays held across daemon restarts. */
+    holds?: {
+        read(): PersistedHolds;
+        write(holds: PersistedHolds): void;
+    };
 }
 export interface Supervisor {
     start(): void;

package/dist/lib/daemon/supervisor.js

@@ -3,6 +3,11 @@
  *
  * - long-running units are restarted after they exit, with per-unit backoff
  *   (1 s doubling to 60 s, reset once a unit has stayed up for a minute);
+ *   a unit that fails EPISODE_MAX_FAILURES times without a run of
+ *   EPISODE_HEALTHY_RESET_MS, or fails for a permanent cause (see exit-cause.ts), is held
+ *   instead. Permanent holds need a restart request, command change or CLI update.
+ *   Exhausted transient sync/mesh units make one durable supervised probe after
+ *   a 30-minute cooldown; a failed probe re-holds, and an explicit stop stays stopped;
  *   `on-failure` units stay down after a clean exit or a SIGTERM/SIGINT from
  *   outside, and any unit stays down after a stop request (`hq bot stop`),
  *   until a restart request;
@@ -22,11 +27,20 @@
  *
  * Processes, time, and state output are injected; `run.ts` wires the real ones.
  */
+import { createHash } from "node:crypto";
 import { StringDecoder } from "node:string_decoder";
+import { classifyExit } from "./exit-cause.js";
 import { SYNC_RUNNER_LEASE_CONFLICT_EXIT, SYNC_RUNNER_LEASE_RETRY_MS, parseSyncRunnerProtocolLine, syncPassResult, } from "./sync-runner.js";
 const BACKOFF_BASE_MS = 1_000;
 const BACKOFF_MAX_MS = 60_000;
-const HEALTHY_RUN_MS = 60_000;
+export const HEALTHY_RUN_MS = 60_000;
+/**
+ * Failures in one episode before the unit is held: the first start plus five
+ * automatic retries. Matches the Desktop app's MAX_FAILURES.
+ */
+export const EPISODE_MAX_FAILURES = 6;
+/** A run this long ends the failure episode. Matches the Desktop app's HEALTHY_RUN_WINDOW. */
+export const EPISODE_HEALTHY_RESET_MS = 30 * 60_000;
 export const STOP_ESCALATION_MS = 10_000;
 export const SYNC_RUNNER_PERSIST_GRACE_MS = 60_000;
 const CLI_UPDATE_SYNC_IDLE_MAX_MS = 10 * 60_000;
@@ -69,6 +83,17 @@ const BLOCKED_READS_BEFORE_PAUSE = 2;
 function backoffMs(attempt) {
     return Math.min(BACKOFF_MAX_MS, BACKOFF_BASE_MS * 2 ** Math.max(0, attempt));
 }
+function keyHash(key) {
+    return createHash("sha256").update(key).digest("hex");
+}
+function recoverableHold(plan, hold) {
+    return plan.kind === "long-running" && (plan.service === "sync" || plan.service === "mesh")
+        && hold?.kind === "exhausted" && hold.cause === "transient";
+}
+function heldReason(unitId, hold) {
+    const what = hold.kind === "permanent" ? `${hold.cause}: ${hold.detail}` : hold.detail;
+    return `held (${what}); \`hq daemon restart ${unitId}\` retries`;
+}
 /**
  * What a unit's process was started from; a unit whose key changes is
  * restarted. The CLI version counts only for `hq` commands, so a CLI update
@@ -109,6 +134,15 @@ export function createSupervisor(deps) {
     const reconcileMs = deps.reconcileMs ?? 15_000;
     const requestPollMs = deps.requestPollMs ?? 1_000;
     const startedAt = now();
+    let persistedHolds = {};
+    /** Holds dropped at plan time (a changed command) that holds.json still lists. */
+    let holdsDirty = false;
+    try {
+        persistedHolds = deps.holds?.read() ?? {};
+    }
+    catch (err) {
+        deps.log("error", `could not read unit holds; starting without them: ${errorMessage(err)}`);
+    }
     let slots = new Map();
     let stopped = false;
     let started = false;
@@ -144,7 +178,13 @@ export function createSupervisor(deps) {
             deps.log("error", `could not write daemon state: ${errorMessage(err)}`);
         }
     };
+    const clearRecoveryWindow = (slot) => {
+        if (slot.recoveryHealthyTimer !== undefined)
+            clearT(slot.recoveryHealthyTimer);
+        slot.recoveryHealthyTimer = undefined;
+    };
     const clearTimer = (slot) => {
+        clearRecoveryWindow(slot);
         if (slot.timer !== undefined)
             clearT(slot.timer);
         slot.timer = undefined;
@@ -269,6 +309,89 @@ export function createSupervisor(deps) {
         delete slot.state.pid;
         delete slot.state.nextRunAt;
     };
+    const writeHolds = () => {
+        if (!deps.holds)
+            return false;
+        const next = {};
+        // Keep holds for units not planned right now (for example a paused bot).
+        for (const [id, hold] of Object.entries(persistedHolds))
+            if (!slots.has(id))
+                next[id] = hold;
+        for (const [id, slot] of slots)
+            if (slot.state.hold)
+                next[id] = slot.state.hold;
+        persistedHolds = next;
+        holdsDirty = false;
+        try {
+            deps.holds.write(next);
+            return true;
+        }
+        catch (err) {
+            deps.log("error", `could not write unit holds: ${errorMessage(err)}`);
+            return false;
+        }
+    };
+    const showHeld = (slot) => {
+        if (recoverableHold(slot.plan, slot.state.hold) && slot.state.hold.retryStopped === true) {
+            slot.state.status = "exited";
+            slot.state.reason = `stopped on request; \`hq daemon restart ${slot.plan.id}\` starts it again`;
+            delete slot.state.pid;
+            delete slot.state.nextRunAt;
+            return;
+        }
+        slot.state.status = "held";
+        slot.state.reason = heldReason(slot.plan.id, slot.state.hold);
+        delete slot.state.pid;
+        delete slot.state.nextRunAt;
+        if (recoverableHold(slot.plan, slot.state.hold) && Number.isSafeInteger(slot.state.hold.retryAt)
+            && slot.state.hold.retryAt >= 0 && slot.state.hold.retryAt <= 8_640_000_000_000_000) {
+            slot.state.nextRunAt = slot.state.hold.retryAt;
+            slot.state.reason += `; one supervised retry at ${new Date(slot.state.hold.retryAt).toISOString()}`;
+        }
+    };
+    const holdUnit = (slot, kind, cause, detail) => {
+        clearTimer(slot);
+        slot.state.hold = {
+            kind,
+            cause,
+            detail,
+            since: now(),
+            failures: slot.failures,
+            ...(slot.state.lastExit ? { lastExit: { ...slot.state.lastExit } } : {}),
+            keyHash: keyHash(slot.key),
+        };
+        if (deps.holds && recoverableHold(slot.plan, slot.state.hold)) {
+            slot.state.hold.retryAt = now() + EPISODE_HEALTHY_RESET_MS;
+        }
+        persistedHolds[slot.plan.id] = slot.state.hold;
+        showHeld(slot);
+        deps.log("error", `holding ${slot.plan.id}: ${slot.state.reason}`);
+        writeHolds();
+        armRecoveryProbe(slot);
+    };
+    const clearHold = (slot) => {
+        clearRecoveryWindow(slot);
+        slot.attempts = 0;
+        slot.failures = 0;
+        if (!slot.state.hold && !persistedHolds[slot.plan.id])
+            return;
+        delete slot.state.hold;
+        if (slot.state.status === "held") {
+            slot.state.status = "idle";
+            delete slot.state.reason;
+        }
+        delete persistedHolds[slot.plan.id];
+        writeHolds();
+    };
+    /** Hold the unit when its exit has a permanent cause; true when it was held. */
+    const holdIfPermanent = (slot, evidence) => {
+        const cause = classifyExit(evidence);
+        if (cause.kind !== "permanent")
+            return false;
+        slot.failures += 1;
+        holdUnit(slot, "permanent", cause.cause, cause.detail);
+        return true;
+    };
     const signalChild = (slot, child, signal) => {
         try {
             child.kill(signal);
@@ -344,6 +467,11 @@ export function createSupervisor(deps) {
     const scheduleRestart = (slot) => {
         if (stopped)
             return;
+        slot.failures += 1;
+        if (slot.failures >= EPISODE_MAX_FAILURES) {
+            holdUnit(slot, "exhausted", "transient", `${slot.failures} failures without staying up for ${EPISODE_HEALTHY_RESET_MS / 60_000} min`);
+            return;
+        }
         const delay = backoffMs(slot.attempts);
         slot.attempts += 1;
         slot.state.restarts += 1;
@@ -469,12 +597,19 @@ export function createSupervisor(deps) {
                 clearT(slot.killTimer);
                 slot.killTimer = undefined;
             }
+            clearRecoveryWindow(slot);
             const ranFor = now() - (slot.childStartedAt ?? now());
             slot.state.lastExit = { code, signal, at: now() };
             delete slot.state.pid;
             if (slot.expectingExit) {
                 slot.expectingExit = false;
                 slot.busy = false;
+                // A unit we stopped (for example a stalled sync runner) after a healthy
+                // run starts a new failure episode, as an unexpected exit would.
+                if (ranFor >= HEALTHY_RUN_MS)
+                    slot.attempts = 0;
+                if (ranFor >= EPISODE_HEALTHY_RESET_MS)
+                    slot.failures = 0;
                 const then = slot.afterExit;
                 slot.afterExit = undefined;
                 then?.();
@@ -492,6 +627,17 @@ export function createSupervisor(deps) {
                 emit();
                 return;
             }
+            const meshOwnerReason = slot.plan.service === "mesh" ? deps.meshOwnerReason?.() : undefined;
+            const recoveryOwnerReason = slot.plan.service === "sync" && code === SYNC_RUNNER_LEASE_CONFLICT_EXIT
+                ? deps.syncOwnerBlockReason?.() ?? "sync owned by another runner"
+                : meshOwnerReason;
+            if (recoveryOwnerReason && recoverableHold(slot.plan, slot.state.hold) && slot.state.hold.probing === true) {
+                // The ordinary ownership retry is 15 s. A recovery probe must not use it
+                // to bypass its persisted cooldown and start a second probe early.
+                holdUnit(slot, "exhausted", "transient", `recovery probe deferred: ${recoveryOwnerReason}`);
+                emit();
+                return;
+            }
             if (slot.plan.service === "sync" && code === SYNC_RUNNER_LEASE_CONFLICT_EXIT) {
                 const reason = deps.syncOwnerBlockReason?.() ?? "sync owned by another runner";
                 slot.state.status = "waiting";
@@ -506,7 +652,6 @@ export function createSupervisor(deps) {
                 emit();
                 return;
             }
-            const meshOwnerReason = slot.plan.service === "mesh" ? deps.meshOwnerReason?.() : undefined;
             if (meshOwnerReason) {
                 let lastLoggedReason;
                 let lastLoggedAt = Number.NEGATIVE_INFINITY;
@@ -541,6 +686,8 @@ export function createSupervisor(deps) {
             deps.log("warn", `${slot.plan.id} exited (code=${code ?? "null"} signal=${signal ?? "none"})`);
             if (ranFor >= HEALTHY_RUN_MS)
                 slot.attempts = 0;
+            if (ranFor >= EPISODE_HEALTHY_RESET_MS)
+                slot.failures = 0;
             const stoppedFromOutside = signal === "SIGTERM" || signal === "SIGINT";
             if (slot.plan.restart === "on-failure" && (code === 0 || stoppedFromOutside)) {
                 slot.state.status = "exited";
@@ -550,7 +697,11 @@ export function createSupervisor(deps) {
                 delete slot.state.nextRunAt;
             }
             else {
-                scheduleRestart(slot);
+                const spawnError = slot.lastSpawnError;
+                delete slot.lastSpawnError;
+                const outputTail = spawnError === undefined ? readExitOutputTail(slot) : undefined;
+                if (!holdIfPermanent(slot, { code, signal, spawnError, outputTail }))
+                    scheduleRestart(slot);
             }
             emit();
         };
@@ -559,10 +710,22 @@ export function createSupervisor(deps) {
             deps.log("error", `${slot.plan.id}: ${errorMessage(err)}`);
             // A child that never got a pid failed to launch and may never emit
             // "exit". Any other error (for example a failed kill) leaves it running.
-            if (child.pid === undefined)
+            if (child.pid === undefined) {
+                if (slot.child === child)
+                    slot.lastSpawnError = err;
                 onEnd(null, null);
+            }
         });
     };
+    const readExitOutputTail = (slot) => {
+        try {
+            return deps.exitOutputTail?.(slot.plan.id);
+        }
+        catch (err) {
+            deps.log("warn", `could not read ${slot.plan.id} output to classify its exit: ${errorMessage(err)}`);
+            return undefined;
+        }
+    };
     const startLong = (slot) => {
         if (stopped)
             return;
@@ -582,7 +745,8 @@ export function createSupervisor(deps) {
         catch (err) {
             slot.state.reason = `failed to start: ${errorMessage(err)}`;
             deps.log("error", `${slot.plan.id} ${slot.state.reason}`);
-            scheduleRestart(slot);
+            if (!holdIfPermanent(slot, { code: null, signal: null, spawnError: err }))
+                scheduleRestart(slot);
             return;
         }
         slot.child = child;
@@ -598,6 +762,57 @@ export function createSupervisor(deps) {
         else
             delete slot.state.pid;
         watch(slot, child);
+        armRecoveryHealthyWindow(slot, child);
+    };
+    const armRecoveryHealthyWindow = (slot, child) => {
+        clearRecoveryWindow(slot);
+        const hold = slot.state.hold;
+        if (!recoverableHold(slot.plan, hold) || hold.probing !== true || hold.retryStopped === true)
+            return;
+        // This timer belongs to this continuously running child, never to time spent held.
+        slot.recoveryHealthyTimer = setT(() => {
+            slot.recoveryHealthyTimer = undefined;
+            if (stopped || slot.child !== child || slot.state.hold !== hold || slot.expectingExit)
+                return;
+            clearHold(slot);
+            emit();
+        }, EPISODE_HEALTHY_RESET_MS);
+    };
+    const armRecoveryProbe = (slot) => {
+        const hold = slot.state.hold;
+        if (!recoverableHold(slot.plan, hold) || hold.retryStopped === true || !deps.holds
+            || stopped || slot.child || slot.plan.blocked)
+            return;
+        clearTimer(slot);
+        const at = now();
+        // Legacy/corrupt deadlines and a clock moving backwards get one full cooldown,
+        // rather than an immediate retry or an unbounded wait.
+        if (!Number.isSafeInteger(hold.retryAt) || hold.retryAt < 0 || hold.retryAt > at + EPISODE_HEALTHY_RESET_MS) {
+            hold.retryAt = at + EPISODE_HEALTHY_RESET_MS;
+            persistedHolds[slot.plan.id] = hold;
+            writeHolds();
+        }
+        showHeld(slot);
+        slot.timer = setT(() => {
+            slot.timer = undefined;
+            if (stopped || slots.get(slot.plan.id) !== slot || slot.child || slot.plan.blocked
+                || slot.state.hold !== hold || hold.retryStopped === true)
+                return;
+            // Advance the durable claim BEFORE spawning. A daemon crash during a probe
+            // cannot turn its old overdue deadline into a restart storm.
+            hold.retryAt = now() + EPISODE_HEALTHY_RESET_MS;
+            hold.probing = true;
+            persistedHolds[slot.plan.id] = hold;
+            if (!writeHolds()) {
+                delete hold.probing;
+                armRecoveryProbe(slot);
+                emit();
+                return;
+            }
+            slot.state.restarts += 1;
+            startLong(slot);
+            emit();
+        }, Math.max(0, hold.retryAt - at));
     };
     const scheduleNextRun = (slot, delayMs) => {
         clearTimer(slot);
@@ -703,23 +918,47 @@ export function createSupervisor(deps) {
                 scheduleNextRun(slot, delay);
             return;
         }
+        if (slot.state.hold) {
+            showHeld(slot);
+            armRecoveryProbe(slot);
+            return;
+        }
         startLong(slot);
     };
-    const newSlot = (plan) => ({
-        plan,
-        key: planKey(plan, deps.cliVersion),
-        attempts: 0,
-        expectingExit: false,
-        busy: false,
-        state: {
-            id: plan.id,
-            service: plan.service,
-            kind: plan.kind,
-            description: plan.description,
-            status: "idle",
-            restarts: 0,
-        },
-    });
+    const newSlot = (plan) => {
+        const key = planKey(plan, deps.cliVersion);
+        const slot = {
+            plan,
+            key,
+            attempts: 0,
+            failures: 0,
+            expectingExit: false,
+            busy: false,
+            state: {
+                id: plan.id,
+                service: plan.service,
+                kind: plan.kind,
+                description: plan.description,
+                status: "idle",
+                restarts: 0,
+            },
+        };
+        const saved = persistedHolds[plan.id];
+        if (saved && plan.kind === "long-running") {
+            if (saved.keyHash === keyHash(key) || (recoverableHold(plan, saved) && saved.retryStopped === true)) {
+                slot.state.hold = saved;
+                slot.failures = saved.failures;
+                deps.log("info", `${plan.id} stays held from an earlier daemon: ${heldReason(plan.id, saved)}`);
+            }
+            else {
+                // The command changed (for example a CLI update), so the old cause may be fixed.
+                delete persistedHolds[plan.id];
+                holdsDirty = true;
+                deps.log("info", `releasing the hold on ${plan.id}: its command changed`);
+            }
+        }
+        return slot;
+    };
     const reconcile = () => {
         if (stopped)
             return;
@@ -774,6 +1013,12 @@ export function createSupervisor(deps) {
             const changed = key !== existing.key;
             existing.plan = plan;
             existing.key = key;
+            if (changed)
+                existing.failures = 0;
+            if (changed && existing.state.hold && existing.state.hold.retryStopped !== true) {
+                deps.log("info", `releasing the hold on ${plan.id}: its command changed`);
+                clearHold(existing);
+            }
             existing.state.description = plan.description;
             existing.state.kind = plan.kind;
             if (plan.blocked) {
@@ -786,6 +1031,11 @@ export function createSupervisor(deps) {
             else if (wasBlocked) {
                 deps.log("info", `starting ${plan.id}: its requirement is now met`);
                 existing.attempts = 0;
+                if (existing.state.hold && !changed) {
+                    // Still the same command that was held; keep it held.
+                    activate(existing);
+                    continue;
+                }
                 // The copy stopped when it was paused may still be exiting; start the
                 // new one only after it is gone, so two never run side by side.
                 if (existing.child)
@@ -836,6 +1086,8 @@ export function createSupervisor(deps) {
             }
         }
         slots = next;
+        if (holdsDirty)
+            writeHolds();
         emit();
         pumpHandoff();
     };
@@ -911,6 +1163,16 @@ export function createSupervisor(deps) {
             }
             // Recorded as runnable, so the first plan read pauses it when it is now blocked.
             const slot = newSlot({ ...plan, blocked: undefined });
+            const manualRecoveryStop = recoverableHold(slot.plan, slot.state.hold) && slot.state.hold.retryStopped === true;
+            const keepProbe = recoverableHold(slot.plan, slot.state.hold) && slot.state.hold.probing === true
+                && slot.state.hold.retryStopped !== true && slot.state.hold.keyHash === keyHash(unit.key);
+            if (!keepProbe && !manualRecoveryStop && (slot.state.hold || persistedHolds[unit.id])) {
+                // A running copy drops stale holds, except a claimed recovery probe whose
+                // failure episode must survive the daemon handoff.
+                delete slot.state.hold;
+                delete persistedHolds[unit.id];
+                holdsDirty = true;
+            }
             slot.key = unit.key;
             slot.adopted = true;
             slot.child = unit.child;
@@ -922,7 +1184,10 @@ export function createSupervisor(deps) {
             if (unit.child.pid !== undefined)
                 slot.state.pid = unit.child.pid;
             watch(slot, unit.child);
+            armRecoveryHealthyWindow(slot, unit.child);
             slots.set(unit.id, slot);
+            if (manualRecoveryStop)
+                terminate(slot, () => showHeld(slot));
             deps.log("info", `adopted ${unit.id} (pid ${unit.child.pid ?? "?"}) from the previous daemon`);
         }
     };
@@ -972,6 +1237,15 @@ export function createSupervisor(deps) {
             }
             if (req.action === "stop") {
                 deps.log("info", `stopping ${slot.plan.id} (requested)`);
+                if (recoverableHold(slot.plan, slot.state.hold)) {
+                    slot.state.hold.retryStopped = true;
+                    delete slot.state.hold.retryAt;
+                    delete slot.state.hold.probing;
+                    persistedHolds[slot.plan.id] = slot.state.hold;
+                    writeHolds();
+                }
+                else
+                    clearHold(slot);
                 // An expected exit is never restarted, including one that needed SIGKILL.
                 terminate(slot, () => {
                     slot.state.status = "exited";
@@ -995,7 +1269,9 @@ export function createSupervisor(deps) {
                 continue;
             }
             deps.log("info", `restarting ${slot.plan.id} (requested)`);
-            slot.attempts = 0;
+            if (slot.state.hold)
+                deps.log("info", `releasing the hold on ${slot.plan.id} (requested)`);
+            clearHold(slot);
             if (slot.plan.kind === "scheduled")
                 continue;
             if (slot.child)

package/dist/lib/daemon/unit-holds.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Host-side storage for the supervisor's unit holds and the per-run output
+ * read used to classify a unit's exit.
+ *
+ * holds.json (`DaemonPaths.holdsPath`, ~/.hq/daemon/holds.json) maps a unit id
+ * to its UnitHold. The daemon rewrites it whenever a hold starts or clears; a
+ * missing or unreadable file means no holds.
+ */
+import type { PersistedHolds } from "./supervisor.js";
+/** How much of a run's output the exit classifier sees. */
+export declare const EXIT_OUTPUT_TAIL_BYTES: number;
+export declare function readUnitHolds(file: string): PersistedHolds;
+export declare function writeUnitHolds(file: string, holds: PersistedHolds): void;
+/** The size of a log file now, so a later read sees only what was written after. */
+export declare function logOffset(file: string): number;
+/**
+ * What was appended to `file` since `offset`, at most the last `maxBytes`.
+ * A file smaller than `offset` was rotated (copy-and-truncate), so it is read
+ * from the start.
+ */
+export declare function readLogSince(file: string, offset: number, maxBytes?: number): string;
+//# sourceMappingURL=unit-holds.d.ts.map