@edspencer/paddock 0.80.0 → 0.81.0

package/packages/server/dist/cli/service/launchd.d.ts

@@ -75,4 +75,18 @@ export declare function parseLaunchctlPrint(out: string): {
     running: boolean;
     pid?: string;
 };
-export declare function createLaunchdBackend(run: Runner, homeDir?: string): ServiceBackend;
+/**
+ * Block this thread for `ms`.
+ *
+ * The backend's {@link Runner} is synchronous (`execFileSync`) and so is every
+ * `ServiceBackend` method, so the wait has to be too — making `install` async
+ * would ripple through the whole service CLI for a sleep. `Atomics.wait` on a
+ * throwaway buffer is the built-in way to do it; `setTimeout` cannot help here
+ * and spinning on `Date.now()` would burn a core.
+ *
+ * Injectable because the unit tests drive the retry path with a fake runner
+ * that fails every attempt: with the real sleep, pinning the retry count would
+ * cost three seconds per case.
+ */
+export declare function sleepSync(ms: number): void;
+export declare function createLaunchdBackend(run: Runner, homeDir?: string, sleep?: (ms: number) => void): ServiceBackend;

package/packages/server/dist/cli/service/launchd.js

@@ -188,8 +188,107 @@ function target() {
 function domain() {
     return `gui/${process.getuid?.() ?? 0}`;
 }
-export function createLaunchdBackend(run, homeDir = os.homedir()) {
+/**
+ * ## The bootout/bootstrap race (#984)
+ *
+ * `launchctl bootout` returns as soon as launchd has ACCEPTED the request, not
+ * once the job has gone. Every sequence below that boots out and then
+ * bootstraps the same label therefore races launchd's own teardown, and when it
+ * loses, `bootstrap` fails with:
+ *
+ * ```
+ * launchctl bootstrap failed (exit 5): Input/output error
+ * ```
+ *
+ * Exit 5 is `EIO`, which is launchd's generic "I cannot do that right now" and
+ * the classic symptom of this race. It is timing-dependent, so it passes
+ * locally and on a PR and then fails on a release: Paddock 0.80.0's first
+ * release run failed exactly here, in CI's real-launchd job, on
+ * `paddock service restart`.
+ *
+ * Both halves of the fix are needed, and neither is sufficient:
+ *
+ *  - **Wait for the job to go.** After a bootout, poll `launchctl print` until
+ *    it reports the label gone (a non-zero exit), up to {@link BOOTOUT_POLLS}
+ *    times. That removes the race in the case we caused it.
+ *  - **Retry the bootstrap.** A bootout we did NOT issue can still be in
+ *    flight — `service start` after a separate process's `service stop`, or a
+ *    login racing a logout — and there is nothing to poll for there, because
+ *    from this process's point of view the job was already gone. So a
+ *    bootstrap that fails with a race code is retried with a short backoff.
+ *
+ * Timing out the poll is not an error: the retry is the backstop, and a job
+ * that will not go is a real failure the bootstrap reports in its own words.
+ */
+const BOOTOUT_POLLS = 40;
+const BOOTOUT_POLL_MS = 250; // 40 × 250ms = 10s
+const BOOTSTRAP_ATTEMPTS = 6;
+const BOOTSTRAP_BACKOFF_MS = 500;
+/**
+ * launchctl exits that mean "try again", as opposed to "no".
+ *
+ * `5` is `EIO`, what the release failure printed. `37` is launchd's "operation
+ * already in progress", which is the same race seen from the other side: the
+ * bootout is still being processed. Anything else — a malformed plist, a label
+ * genuinely already loaded, a permission problem — is reported, not retried,
+ * because retrying it only delays the message by three seconds.
+ */
+const BOOTSTRAP_RACE_CODES = new Set([5, 37]);
+/**
+ * Block this thread for `ms`.
+ *
+ * The backend's {@link Runner} is synchronous (`execFileSync`) and so is every
+ * `ServiceBackend` method, so the wait has to be too — making `install` async
+ * would ripple through the whole service CLI for a sleep. `Atomics.wait` on a
+ * throwaway buffer is the built-in way to do it; `setTimeout` cannot help here
+ * and spinning on `Date.now()` would burn a core.
+ *
+ * Injectable because the unit tests drive the retry path with a fake runner
+ * that fails every attempt: with the real sleep, pinning the retry count would
+ * cost three seconds per case.
+ */
+export function sleepSync(ms) {
+    if (ms <= 0)
+        return;
+    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+export function createLaunchdBackend(run, homeDir = os.homedir(), sleep = sleepSync) {
     const file = plistPath(homeDir);
+    /**
+     * Boot the job out and wait until launchd agrees it has gone — see the race
+     * note above. The bootout's own result is deliberately not inspected: "there
+     * was no previous service" is the expected case on a first install, and a
+     * bootout that genuinely failed shows up as a job that is still there, which
+     * is what the poll below is looking at anyway.
+     */
+    const bootoutAndWait = () => {
+        run("launchctl", ["bootout", target()]);
+        // Counted, not timed: an injected no-op sleep has to terminate the loop
+        // too, and a wall-clock deadline with a no-op sleep is a ten-second spin.
+        for (let poll = 1; poll <= BOOTOUT_POLLS; poll++) {
+            // A non-zero `print` is launchd saying it has never heard of the label,
+            // which is precisely the state `bootstrap` needs.
+            if (run("launchctl", ["print", target()]).status !== 0)
+                return;
+            if (poll < BOOTOUT_POLLS)
+                sleep(BOOTOUT_POLL_MS);
+        }
+    };
+    /** Bootstrap the plist, retrying the race codes. Throws with launchctl's own words. */
+    const bootstrapOrThrow = () => {
+        for (let attempt = 1;; attempt++) {
+            // NOT `load`/`unload`: those are the legacy API, and on a modern macOS
+            // they are shims over this one that report success in cases where it
+            // reports the actual error.
+            const boot = run("launchctl", ["bootstrap", domain(), file]);
+            if (boot.status === 0)
+                return;
+            if (attempt >= BOOTSTRAP_ATTEMPTS || !BOOTSTRAP_RACE_CODES.has(boot.status ?? -1)) {
+                throw new Error(launchctlFailure("bootstrap", boot));
+            }
+            sleep(BOOTSTRAP_BACKOFF_MS);
+        }
+    };
     return {
         platform: "darwin",
         unitPath: file,
@@ -202,16 +301,12 @@ export function createLaunchdBackend(run, homeDir = os.homedir()) {
             fs.mkdirSync(path.dirname(spec.logPath), { recursive: true });
             fs.writeFileSync(file, renderPlist(spec), "utf8");
             // `bootstrap` fails outright if the label is already loaded, so a
-            // reinstall has to unload first. A first install has nothing to unload,
-            // and says so on stderr — ignored rather than reported, because "there
-            // was no previous service" is the expected case, not a problem.
-            run("launchctl", ["bootout", target()]);
-            // NOT `load`/`unload`: those are the legacy API, and on a modern macOS
-            // they are shims over this one that report success in cases where it
-            // reports the actual error.
-            const boot = run("launchctl", ["bootstrap", domain(), file]);
-            if (boot.status !== 0)
-                throw new Error(launchctlFailure("bootstrap", boot));
+            // reinstall has to unload first — and has to WAIT for that, which is what
+            // `bootoutAndWait` is for (see the race note). A first install has
+            // nothing to unload, and says so on stderr: ignored rather than reported,
+            // because "there was no previous service" is the expected case.
+            bootoutAndWait();
+            bootstrapOrThrow();
             // `RunAtLoad` should already have started it; `kickstart -k` makes that
             // true rather than probable, and is the same command a later restart uses.
             run("launchctl", ["kickstart", "-k", target()]);
@@ -230,11 +325,13 @@ export function createLaunchdBackend(run, homeDir = os.homedir()) {
          * Paddock is not merely idle, it is absent from the domain — and `start` on
          * a label launchd has never heard of fails. Bootstrapping is also what a
          * login does, so `service start` and the next login take the same path.
+         *
+         * No bootout to wait on here, but the retry in `bootstrapOrThrow` still
+         * earns its keep: the `stop` that booted it out may have been a different
+         * `paddock` process seconds ago, and launchd may not have finished with it.
          */
         start() {
-            const boot = run("launchctl", ["bootstrap", domain(), file]);
-            if (boot.status !== 0)
-                throw new Error(launchctlFailure("bootstrap", boot));
+            bootstrapOrThrow();
             run("launchctl", ["kickstart", target()]);
         },
         /**
@@ -263,12 +360,13 @@ export function createLaunchdBackend(run, homeDir = os.homedir()) {
          * service that is stopped. This sequence is identical to `install`'s minus
          * writing the plist, which also gives `restart` the property the issue asks
          * for — the unit is re-read on the way up, so an edited plist takes effect.
+         *
+         * This is the sequence that failed Paddock 0.80.0's release run: bootout
+         * and bootstrap back to back, with nothing between them. See the race note.
          */
         restart() {
-            run("launchctl", ["bootout", target()]);
-            const boot = run("launchctl", ["bootstrap", domain(), file]);
-            if (boot.status !== 0)
-                throw new Error(launchctlFailure("bootstrap", boot));
+            bootoutAndWait();
+            bootstrapOrThrow();
             run("launchctl", ["kickstart", "-k", target()]);
         },
         status() {

package/packages/server/dist/herdctl.d.ts

@@ -371,6 +371,25 @@ export declare class HerdctlService {
      * error, returns `false` — the pre-#397 behaviour.
      */
     isSdkSessionLive(sessionId: string): boolean;
+    /**
+     * How many scheduler jobs are running right now, read from memory.
+     *
+     * This exists for `GET /api/busy` (CLI contract SPEC §4), which must read
+     * in-memory state and never walk the disk. The number the contract's ROADMAP
+     * names is `fleetStatus().counts.runningJobs` — but {@link fleetStatus} is
+     * built from `getAgentInfo()`, which re-reads the fleet state FILE on every
+     * call, and that re-parse has been this server's single largest CPU cost.
+     *
+     * The two are the same number. `counts.runningJobs` is
+     * `Σ scheduler.getRunningJobCount(agent.qualifiedName)` over the configured
+     * agents, and `getTotalRunningJobCount()` is the size of the same scheduler
+     * map keyed `<agent>/<schedule>` — one entry per running agent+schedule pair
+     * either way. So this reads the authority directly instead of a derived
+     * snapshot of it.
+     *
+     * Null-safe like {@link isSdkSessionLive}: no fleet or no scheduler means 0.
+     */
+    runningJobCount(): number;
     fleetStatus(): Promise<FleetStatus>;
     agents(): Promise<AgentInfo[]>;
     /**

package/packages/server/dist/herdctl.js

@@ -528,6 +528,32 @@ export class HerdctlService {
             return false;
         }
     }
+    /**
+     * How many scheduler jobs are running right now, read from memory.
+     *
+     * This exists for `GET /api/busy` (CLI contract SPEC §4), which must read
+     * in-memory state and never walk the disk. The number the contract's ROADMAP
+     * names is `fleetStatus().counts.runningJobs` — but {@link fleetStatus} is
+     * built from `getAgentInfo()`, which re-reads the fleet state FILE on every
+     * call, and that re-parse has been this server's single largest CPU cost.
+     *
+     * The two are the same number. `counts.runningJobs` is
+     * `Σ scheduler.getRunningJobCount(agent.qualifiedName)` over the configured
+     * agents, and `getTotalRunningJobCount()` is the size of the same scheduler
+     * map keyed `<agent>/<schedule>` — one entry per running agent+schedule pair
+     * either way. So this reads the authority directly instead of a derived
+     * snapshot of it.
+     *
+     * Null-safe like {@link isSdkSessionLive}: no fleet or no scheduler means 0.
+     */
+    runningJobCount() {
+        try {
+            return this.fleet?.getScheduler()?.getTotalRunningJobCount() ?? 0;
+        }
+        catch {
+            return 0;
+        }
+    }
     async fleetStatus() {
         return this.manager.getFleetStatus();
     }

package/packages/server/dist/routes/meta.js

@@ -6,8 +6,14 @@ import { resolveModels, resolveDefaultModel } from "../models.js";
 import { sendProjectError } from "../route-errors.js";
 import { cspFor, parseRangeHeader } from "../http-bytes.js";
 import { PADDOCK_VERSION } from "../version.js";
+import { collectBusy } from "../busy.js";
 export function registerMetaRoutes(app, ctx) {
     const { herdctl, transcriber, attachments, cfg } = ctx;
+    // Both optional for the same reason `turn-interlock.ts` takes an optional hub:
+    // a server built without the WS layer (several integration tests) has neither,
+    // and the honest answer for it is zero rather than a crash.
+    const hub = ctx.managementOpsContext?.hub;
+    const background = ctx.managementOpsContext?.background;
     // --- voice dictation (#voice): capability probe + transcription -------
     // The composer polls this to decide whether to show a mic button. `available`
     // is false on instances with dictation off (or a misconfigured remote).
@@ -128,6 +134,45 @@ export function registerMetaRoutes(app, ctx) {
             },
         },
     }, async () => ({ ok: true, version: PADDOCK_VERSION }));
+    // The contract's second server endpoint (SPEC §4, §8): is this instance busy?
+    //
+    // **Counts only, and no auth, and those two facts are the same fact.** There
+    // is nothing in the answer to protect — no chat title, no session id, no
+    // project name, just three integers and the version — which is why the route
+    // can sit beside `/api/health` in `HEALTH_PATHS` without widening what an
+    // unauthenticated caller can learn beyond "something is happening here".
+    // `reasons` (a line per piece of work, for a person) waits on SPEC §10's first
+    // question: how a CLI proves it is local, given that a reverse proxy on the
+    // same host also connects from loopback. Until that is designed, this answer
+    // is the whole answer for every caller — so nothing here may grow a string
+    // field without revisiting the auth decision above it.
+    //
+    // The callers that make the cheapness requirement real: `paddock service
+    // stop`, `paddock update`, and Watchtower's pre-update hook, which runs
+    // `paddock busy` before every image pull. `collectBusy` is synchronous and
+    // reads three in-memory structures — see busy.ts for why `job` does NOT come
+    // from `fleetStatus()`.
+    app.get("/api/busy", {
+        schema: {
+            tags: ["System"],
+            summary: "Is this instance busy?",
+            description: "The CLI contract's busy check (SPEC §4). Returns `{ busy, version, counts }`, where `counts` has `chat` (chat turns running, including fired triggers and cron fires), `background` (live sessions holding background work after their turn ended) and `job` (herdctl scheduler jobs). `busy` is true only for work a restart would lose or cut short. Counts only — never labels, ids or titles — so it needs no authentication. Reads in-memory state, never the disk.",
+            response: {
+                200: {
+                    description: "Whether work a restart would cut short is under way.",
+                    type: "object",
+                    additionalProperties: true,
+                },
+            },
+        },
+    }, async () => collectBusy({
+        runningChats: () => hub?.runningSessions().length ?? 0,
+        // Tasks in a session count only while its SDK session is still live —
+        // see `BackgroundRegistry.liveTaskCount`, which explains why a stale
+        // batch-mode entry must not wedge this at busy.
+        backgroundTasks: () => background?.liveTaskCount((id) => herdctl.isSdkSessionLive(id)) ?? 0,
+        runningJobs: () => herdctl.runningJobCount(),
+    }, PADDOCK_VERSION));
     // --- instance-wide settings (issue #385) --------------------------------
     // A top-level admin screen over the frozen instance config. GET reports every
     // surfaced field (value / default / editable / sensitive / env-shadow); PUT

package/packages/server/dist/runtime-file.d.ts

@@ -0,0 +1,26 @@
+export interface RuntimeInfo {
+    pid: number;
+    port: number;
+    /** Where the CLI reaches it, e.g. `http://127.0.0.1:3729`. */
+    url: string;
+    version: string;
+    /** ISO 8601. */
+    startedAt: string;
+}
+export declare const RUNTIME_FILE = "runtime.json";
+export declare function runtimeFilePath(home: string): string;
+/** Writes the file atomically (a temporary file, then a rename), readable only by this user. */
+export declare function writeRuntimeFile(home: string, info: Omit<RuntimeInfo, "startedAt"> & {
+    startedAt?: string;
+}): RuntimeInfo;
+/** The file's contents, or null if it's missing or isn't a runtime file (a half-written or foreign file). */
+export declare function readRuntimeFile(home: string): RuntimeInfo | null;
+/**
+ * Removes the file, but only if it's still ours: a second instance may have started since and written its own.
+ * Call it on a clean exit. Returns whether it removed anything.
+ */
+export declare function removeRuntimeFile(home: string, pid?: number): boolean;
+/** Whether a process with this pid exists. EPERM means it exists but isn't ours, which still counts. */
+export declare function isPidAlive(pid: number): boolean;
+/** The runtime file, if the process it names is still alive. A crashed instance leaves a stale file behind. */
+export declare function readLiveRuntimeFile(home: string): RuntimeInfo | null;

package/packages/server/dist/runtime-file.js

@@ -0,0 +1,91 @@
+// From edspencer/cli-contract reference/runtime-file.ts @ SPEC 0.1; local changes: `node:fs` / `node:path` imports
+// unchanged, nothing else — the module is Node built-ins only and Paddock's `<home>` is its data dir, so it needed
+// no adaptation. Kept verbatim on purpose: wave C4 re-syncs it, and a divergence here would be a divergence in the
+// one file every tool's `busy`, `service status` and `update` agree on.
+//
+// `<home>/runtime.json`: what a running instance tells its own CLI (SPEC §4, ROADMAP). `serve` writes it once it's
+// listening and removes it on a clean exit; `busy`, `service status` and `update` read it to find the instance.
+//
+// It holds `{ pid, port, url, version, startedAt }`. A token for /api/busy will join it once SPEC §10's first question
+// is settled; the file is written 0600 already, so adding one later changes nothing about who can read it.
+// Node built-ins only.
+import fs from "node:fs";
+import path from "node:path";
+export const RUNTIME_FILE = "runtime.json";
+export function runtimeFilePath(home) {
+    return path.join(home, RUNTIME_FILE);
+}
+/** Writes the file atomically (a temporary file, then a rename), readable only by this user. */
+export function writeRuntimeFile(home, info) {
+    const full = {
+        pid: info.pid,
+        port: info.port,
+        url: info.url,
+        version: info.version,
+        startedAt: info.startedAt ?? new Date().toISOString(),
+    };
+    fs.mkdirSync(home, { recursive: true });
+    const file = runtimeFilePath(home);
+    const tmp = `${file}.${process.pid}.tmp`;
+    fs.writeFileSync(tmp, `${JSON.stringify(full, null, 2)}\n`, { mode: 0o600 });
+    try {
+        fs.renameSync(tmp, file);
+    }
+    catch (err) {
+        fs.rmSync(tmp, { force: true });
+        throw err;
+    }
+    return full;
+}
+/** The file's contents, or null if it's missing or isn't a runtime file (a half-written or foreign file). */
+export function readRuntimeFile(home) {
+    let raw;
+    try {
+        raw = fs.readFileSync(runtimeFilePath(home), "utf8");
+    }
+    catch {
+        return null;
+    }
+    try {
+        const v = JSON.parse(raw);
+        if (Number.isInteger(v.pid) &&
+            Number.isInteger(v.port) &&
+            typeof v.url === "string" &&
+            typeof v.version === "string" &&
+            typeof v.startedAt === "string") {
+            return v;
+        }
+    }
+    catch {
+        // fall through
+    }
+    return null;
+}
+/**
+ * Removes the file, but only if it's still ours: a second instance may have started since and written its own.
+ * Call it on a clean exit. Returns whether it removed anything.
+ */
+export function removeRuntimeFile(home, pid = process.pid) {
+    const info = readRuntimeFile(home);
+    if (!info || info.pid !== pid)
+        return false;
+    fs.rmSync(runtimeFilePath(home), { force: true });
+    return true;
+}
+/** Whether a process with this pid exists. EPERM means it exists but isn't ours, which still counts. */
+export function isPidAlive(pid) {
+    if (!Number.isInteger(pid) || pid <= 0)
+        return false;
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (err) {
+        return err.code === "EPERM";
+    }
+}
+/** The runtime file, if the process it names is still alive. A crashed instance leaves a stale file behind. */
+export function readLiveRuntimeFile(home) {
+    const info = readRuntimeFile(home);
+    return info && isPidAlive(info.pid) ? info : null;
+}

package/packages/server/dist/start.d.ts

@@ -1 +1,34 @@
+/**
+ * How long a clean shutdown gets before we exit anyway.
+ *
+ * The CLI contract requires the SIGTERM path to exit "within the unit's stop
+ * timeout" (SPEC §4, §8), and the tightest of those is **launchd's**: with no
+ * `ExitTimeOut` in the plist, a LaunchAgent gets 20 seconds before SIGKILL
+ * (systemd's user default is 90). So the budget is 20 seconds, and this sits
+ * inside it with room for the exit itself.
+ *
+ * The deadline matters because the two awaits below are not bounded by anything
+ * else. `app.close()` waits for in-flight requests and open WebSockets, and a
+ * browser holding `/ws` is the normal state of a running Paddock; `fleet.stop()`
+ * talks to subprocesses. Exceeding the budget is strictly worse than exiting at
+ * it: SIGKILL skips the rest of this function, so the runtime file would be left
+ * behind for the next instance's `busy` to find and the log would have no record
+ * of what was cut short.
+ */
+export declare const SHUTDOWN_DEADLINE_MS = 15000;
+/**
+ * The URL to record in `runtime.json` — the one a CLI on this machine can
+ * actually fetch, which is not always the one we bound.
+ *
+ * A wildcard bind is the normal case for a container or a dev box (`HOST=0.0.0.0`
+ * is what this repo's own `pm` wrapper injects), and `http://0.0.0.0:<port>` is
+ * not an address to connect TO. Node happens to resolve it to loopback on Linux
+ * and fails on some other stacks, so the file records loopback explicitly rather
+ * than relying on that. A real interface address is left alone: someone who
+ * bound `192.168.1.10` meant it, and that address reaches the server from here
+ * as well as from anywhere else.
+ *
+ * IPv6 gets brackets, because `new URL('http://::1:7233')` is not a URL.
+ */
+export declare function runtimeUrl(host: string, port: number): string;
 export declare function start(): Promise<void>;

package/packages/server/dist/start.js

@@ -1,7 +1,8 @@
 /**
- * Process lifecycle for a Paddock server: bind the port and wire signal-driven
- * shutdown. Split out of `index.ts` so the `paddock` CLI (src/cli/paddock.ts)
- * and the plain `node dist/index.js` entrypoint share one implementation.
+ * Process lifecycle for a Paddock server: bind the port, publish
+ * `runtime.json`, and wire signal-driven shutdown. Split out of `index.ts` so
+ * the `paddock` CLI (src/cli/paddock.ts) and the plain `node dist/index.js`
+ * entrypoint share one implementation.
  *
  * Config is resolved INSIDE `buildApp()`, not at module load, so a caller may
  * still mutate `process.env` right up until this function is invoked. The CLI
@@ -9,8 +10,51 @@
  */
 import { buildApp } from "./app.js";
 import { startServiceLogRotation } from "./service-log.js";
+import { removeRuntimeFile, writeRuntimeFile } from "./runtime-file.js";
+import { PADDOCK_VERSION } from "./version.js";
+/**
+ * How long a clean shutdown gets before we exit anyway.
+ *
+ * The CLI contract requires the SIGTERM path to exit "within the unit's stop
+ * timeout" (SPEC §4, §8), and the tightest of those is **launchd's**: with no
+ * `ExitTimeOut` in the plist, a LaunchAgent gets 20 seconds before SIGKILL
+ * (systemd's user default is 90). So the budget is 20 seconds, and this sits
+ * inside it with room for the exit itself.
+ *
+ * The deadline matters because the two awaits below are not bounded by anything
+ * else. `app.close()` waits for in-flight requests and open WebSockets, and a
+ * browser holding `/ws` is the normal state of a running Paddock; `fleet.stop()`
+ * talks to subprocesses. Exceeding the budget is strictly worse than exiting at
+ * it: SIGKILL skips the rest of this function, so the runtime file would be left
+ * behind for the next instance's `busy` to find and the log would have no record
+ * of what was cut short.
+ */
+export const SHUTDOWN_DEADLINE_MS = 15_000;
+/**
+ * The URL to record in `runtime.json` — the one a CLI on this machine can
+ * actually fetch, which is not always the one we bound.
+ *
+ * A wildcard bind is the normal case for a container or a dev box (`HOST=0.0.0.0`
+ * is what this repo's own `pm` wrapper injects), and `http://0.0.0.0:<port>` is
+ * not an address to connect TO. Node happens to resolve it to loopback on Linux
+ * and fails on some other stacks, so the file records loopback explicitly rather
+ * than relying on that. A real interface address is left alone: someone who
+ * bound `192.168.1.10` meant it, and that address reaches the server from here
+ * as well as from anywhere else.
+ *
+ * IPv6 gets brackets, because `new URL('http://::1:7233')` is not a URL.
+ */
+export function runtimeUrl(host, port) {
+    const loopback = {
+        "0.0.0.0": "127.0.0.1",
+        "::": "::1",
+        "": "127.0.0.1",
+    };
+    const h = loopback[host] ?? host;
+    return `http://${h.includes(":") ? `[${h}]` : h}:${String(port)}`;
+}
 export async function start() {
-    const { app, cfg, close } = await buildApp();
+    const { app, cfg, close, hub } = await buildApp();
     // Nothing else rotates the service log: the supervisor appends to it through
     // its own descriptor, a `systemd --user` unit writing to a file gets no
     // journal rotation, and macOS has nothing that would (SPEC §7). A no-op on a
@@ -20,13 +64,76 @@ export async function start() {
             app.log.warn(message);
         },
     });
-    const shutdown = async () => {
+    /**
+     * Exit on SIGTERM/SIGINT having said what was cut short (SPEC §4, §8).
+     *
+     * Three things are deliberate here.
+     *
+     * **It records rather than finishes.** A chat turn is a `claude` subprocess
+     * that can run for minutes; no 20-second stop timeout can wait for one, and
+     * `herdctl.stop()` ends the fleet with `waitForJobs: false` precisely because
+     * waiting is not an option. So the contract's other half applies — record the
+     * work under way — and the record is this `warn` line in the service log,
+     * which is the one artefact that survives the restart and which
+     * `paddock service logs` reads. What is NOT done, and should not be, is
+     * writing a note into the cut-short chat's transcript: `claude` owns that file
+     * and may append to it after we are gone, so a server-side write cannot be
+     * ordered against the subprocess's own and would be the resurrection race
+     * `turn-interlock.ts` exists to avoid.
+     *
+     * The reason a count is enough: `/api/busy` reports the same numbers BEFORE
+     * the stop, and `paddock service stop` refuses on them. The log line is for
+     * the race §4 admits — work that started between the check and the signal.
+     *
+     * **It is idempotent.** A supervisor that sends SIGTERM twice, or a Ctrl-C
+     * after a SIGTERM, used to re-enter `close()` concurrently: a second
+     * `app.close()` on a closing Fastify rejects, and a second `fleet.stop()`
+     * races the first. The guard makes the extra signals no-ops.
+     *
+     * **It exits 0.** The unit's `Restart=always` / `KeepAlive` means the
+     * supervisor brings Paddock back whatever the code; 0 is what makes a logout
+     * or a `service stop` read as a clean stop rather than a crash in the logs.
+     */
+    let shuttingDown = false;
+    const shutdown = async (signal) => {
+        if (shuttingDown)
+            return;
+        shuttingDown = true;
+        const running = hub.runningSessions().length;
+        app.log.warn(running > 0
+            ? `${signal}: shutting down with ${running} chat turn(s) in flight — they are being cut short. ` +
+                "Run `paddock busy` before a stop or an update to see this before it happens."
+            : `${signal}: shutting down, no chat turn in flight.`);
         stopRotation();
-        await close();
+        // Published once we were listening; removed here so a `busy` or a
+        // `service status` run after this exit does not find a live-looking
+        // instance. Before the slow awaits, because this is the part that must land
+        // even if they overrun the deadline below.
+        removeRuntimeFile(cfg.dataDir);
+        // Bounded: see SHUTDOWN_DEADLINE_MS. `exit(0)` either way, so a hung
+        // `app.close()` cannot turn a stop into a SIGKILL.
+        await Promise.race([close(), new Promise((r) => setTimeout(r, SHUTDOWN_DEADLINE_MS))]).catch(() => undefined);
         process.exit(0);
     };
-    process.on("SIGINT", shutdown);
-    process.on("SIGTERM", shutdown);
+    process.on("SIGINT", () => void shutdown("SIGINT"));
+    process.on("SIGTERM", () => void shutdown("SIGTERM"));
     await app.listen({ port: cfg.port, host: cfg.host });
+    // `<dataDir>/runtime.json`, 0600, written only once we are actually listening
+    // (SPEC §4) — it is how `paddock busy`, and later `update`, find THIS instance
+    // without guessing a port. A failure to write it is logged and not fatal: the
+    // server is up and serving, and the one thing that stops working is a CLI
+    // finding it, which is better than refusing to run.
+    try {
+        writeRuntimeFile(cfg.dataDir, {
+            pid: process.pid,
+            port: cfg.port,
+            url: runtimeUrl(cfg.host, cfg.port),
+            version: PADDOCK_VERSION,
+        });
+    }
+    catch (err) {
+        app.log.warn(`could not write ${cfg.dataDir}/runtime.json (${err.message}) — ` +
+            "`paddock busy` will report this instance as not running");
+    }
     app.log.info(`paddock-server listening on http://${cfg.host}:${cfg.port}`);
 }

package/packages/server/dist/ws-context.d.ts

@@ -14,6 +14,7 @@ import type { SweepService } from "./sweep.js";
 import type { PaddockConfig } from "./config.js";
 import type { DriveMode } from "./models.js";
 import type { SessionHub } from "./session-hub.js";
+import type { BackgroundRegistry } from "./background-live.js";
 import type { AttachmentStore } from "./attachments.js";
 import type { QueuedMessageStore } from "./queued-message.js";
 import type { ArchiveStore } from "./archive.js";
@@ -152,6 +153,14 @@ export type FireTrigger = (slug: string, triggerName: string) => Promise<string
 export interface ChatHandlerContext {
     deps: ChatHandlerDeps;
     hub: SessionHub;
+    /**
+     * The live background-work registry (#604). Here for the same reason `hub` is:
+     * it is per-server state owned by the WS layer that a NON-socket caller has to
+     * be able to read. `GET /api/busy` is that caller — a turn can end while its
+     * background tasks keep running, so the hub alone would report a session with
+     * a live sub-agent as idle (CLI contract SPEC §4).
+     */
+    background: BackgroundRegistry;
     startAgentTurn: StartAgentTurn;
     composePreloadedPrompt: ComposePreloadedPrompt;
     fireTrigger: FireTrigger;

package/packages/server/dist/ws-turn.js

@@ -85,6 +85,7 @@ export function makeTurnEngine(engine) {
     const selfMcpCtx = {
         deps,
         hub,
+        background,
         startAgentTurn,
         composePreloadedPrompt: triggers.composePreloadedPrompt,
         fireTrigger: triggers.fireTrigger,

package/packages/server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@paddock/server",
-  "version": "0.80.0",
+  "version": "0.81.0",
   "private": true,
   "type": "module"
 }