@ccmsg/cli 0.3.2 → 0.3.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ccmsg/cli",
-  "version": "0.3.2",
+  "version": "0.3.3",
   "description": "The ccmsg daemon, CLI and agent plugins for one instance (= one config home)",
   "license": "MIT",
   "author": "kawaz",

package/src/daemon/supervise.ts

@@ -45,6 +45,9 @@ export interface Backoff {
 
 export const BACKOFF: Backoff = { minMs: 500, maxMs: 30_000, steadyMs: 60_000 };
 
+/** The time each cooperative stop stage gets before the supervisor escalates. */
+export const STOP_TIMEOUT_MS = 10_000;
+
 export interface SuperviseOptions {
   readonly env?: Env;
   readonly spawn?: SpawnInstance;
@@ -54,6 +57,8 @@ export interface SuperviseOptions {
   /** How long a start waits for the child to be serving before it is reported
    * as having failed. */
   readonly startTimeoutMs?: number;
+  /** How long each graceful and SIGTERM stop stage may hold shutdown. */
+  readonly stopTimeoutMs?: number;
 }
 
 /** One config home the supervisor looks after, and how it is doing.
@@ -98,6 +103,7 @@ export class Supervisor {
   readonly #backoff: Backoff;
   readonly #log: (line: Record<string, unknown>) => void;
   readonly #startTimeoutMs: number;
+  readonly #stopTimeoutMs: number;
   readonly #units = new Map<string, Supervised>();
   readonly #waits = new Set<() => void>();
   #listener: ReturnType<typeof Bun.listen> | undefined;
@@ -110,6 +116,7 @@ export class Supervisor {
     this.#spawn = options.spawn ?? spawnInstance;
     this.#backoff = options.backoff ?? BACKOFF;
     this.#startTimeoutMs = options.startTimeoutMs ?? START_TIMEOUT_MS;
+    this.#stopTimeoutMs = options.stopTimeoutMs ?? STOP_TIMEOUT_MS;
     this.#log = options.log ?? ((line) => process.stderr.write(`${JSON.stringify(line)}\n`));
     for (const target of registered(this.#env)) this.#units.set(target.dir, new Supervised(target));
   }
@@ -278,12 +285,7 @@ export class Supervisor {
     // asked for rather than one to recover from.
     unit.wanted = false;
     for (const cancel of new Set(this.#waits)) cancel();
-    try {
-      await askToStop(unit.target);
-    } catch {
-      child.kill("SIGTERM");
-    }
-    await child.exited;
+    await this.#stopChild(unit, child);
     await unit.loop;
     return { dir, stopped: true };
   }
@@ -394,27 +396,70 @@ export class Supervisor {
     });
   }
 
-  /** Stop every child, stop restarting them, and give up the socket.
-   *
-   * The socket goes first, so a command arriving mid-shutdown is told there is
-   * no supervisor rather than being answered by one that is leaving. */
+  async #stopChild(unit: Supervised, child: Child): Promise<void> {
+    const within = async (work: Promise<unknown>): Promise<boolean> => {
+      let timer: ReturnType<typeof setTimeout> | undefined;
+      const deadline = new Promise<false>((resolve) => {
+        timer = setTimeout(() => resolve(false), this.#stopTimeoutMs);
+      });
+      try {
+        return await Promise.race([
+          work.then(
+            () => true,
+            () => false,
+          ),
+          deadline,
+        ]);
+      } finally {
+        clearTimeout(timer);
+      }
+    };
+
+    // Each stage is said out loud with how long the one before it took. A child
+    // that will not leave is a thing that happens on a machine nobody is
+    // watching, and what stage it was at is the whole of what can be known
+    // about it afterwards.
+    const startedAt = Date.now();
+    const say = (stage: string) => {
+      this.#log({
+        event: "stopping",
+        dir: unit.target.dir,
+        pid: child.pid,
+        stage,
+        in_ms: Date.now() - startedAt,
+      });
+    };
+
+    say("asked");
+    if (await within(askToStop(unit.target).then(() => child.exited))) {
+      say("exited");
+      return;
+    }
+    say("sigterm");
+    child.kill("SIGTERM");
+    if (await within(child.exited)) {
+      say("exited");
+      return;
+    }
+    say("sigkill");
+    child.kill("SIGKILL");
+    await child.exited;
+    say("exited");
+  }
+
+  /** Stop every child, stop restarting them, and give up the socket. */
   async stop(): Promise<void> {
     this.#leaving = true;
-    this.#listener?.stop(true);
-    this.#listener = undefined;
     for (const cancel of new Set(this.#waits)) cancel();
     await Promise.all(
       [...this.#units.values()].map(async (unit) => {
         const child = unit.child;
         if (child === undefined) return;
-        try {
-          await askToStop(unit.target);
-        } catch {
-          child.kill("SIGTERM");
-        }
-        await child.exited;
+        await this.#stopChild(unit, child);
       }),
     );
+    this.#listener?.stop(true);
+    this.#listener = undefined;
     this.#left?.();
     await this.#ran;
   }

package/src/instance/instance.ts

@@ -941,15 +941,25 @@ export class Instance {
     // they change rather than at exit, so there is nothing held back to flush;
     // the log's writer is synchronous for the same reason (§3.6).
     this.log.write("stopping", { instance: this.self });
-    // 5. let the resources go, the unix socket last. The pid and the lock go
-    // before it, because a client reads a refusing socket as this instance
-    // having finished leaving and a successor may claim what it sees free.
-    remove(this.paths.pidFile);
-    this.lock.release();
-    // Closing takes the path this process bound, and only that one: the stable
-    // address is a symlink nothing here touches, because a successor may have
-    // already pointed it at itself (§8.5).
-    await this.#transport.close();
+    // 5. let the resources go, the unix socket last. Closing takes the path
+    // this process bound, and only that one: the stable address is a symlink
+    // nothing here touches, because a successor may have already pointed it at
+    // itself (§8.5).
+    try {
+      await this.#transport.close();
+    } catch (cause) {
+      // A listener that could not be closed is worth saying, and is not worth
+      // holding the pid and the lock over: this process is leaving either way,
+      // and keeping them would leave a successor unable to start against a
+      // config home nothing is serving.
+      this.log.write("listener close failed", { instance: this.self, cause: String(cause) });
+    } finally {
+      // The pid and lock are the observable proof that this process is still
+      // leaving. Released only after every listener has finished closing, so a
+      // client cannot mistake an unreachable socket for a completed stop.
+      remove(this.paths.pidFile);
+      this.lock.release();
+    }
   }
 }
 

package/src/service/service.ts

@@ -171,20 +171,62 @@ function refuse(command: readonly string[], answer: RunResult): never {
   );
 }
 
+/** How long the supervisor gets to leave after being asked to, before the init
+ * system is told to kill it.
+ *
+ * The same budget the supervisor gives each of its own children, because that
+ * is what it spends: a supervisor that is going to leave has finished leaving
+ * by the time its last child has. */
+export const STOP_DEADLINE_MS = 10_000;
+
+/** How often the init system is asked again while waiting for a departure. */
+const DEPARTURE_POLL_MS = 100;
+
+/** Whether `gone` holds, waited for up to the deadline.
+ *
+ * Asked again rather than awaited: the supervisor belongs to the init system
+ * and not to this process, so there is no exit to wait on and the only account
+ * of whether it is still there is one that has to be fetched. */
+async function departed(gone: () => Promise<boolean>, deadlineMs: number): Promise<boolean> {
+  const end = Date.now() + deadlineMs;
+  for (;;) {
+    if (await gone()) return true;
+    const left = end - Date.now();
+    if (left <= 0) return false;
+    await Bun.sleep(Math.min(DEPARTURE_POLL_MS, left));
+  }
+}
+
+/** Whether the process that held `pid` is still there, asked of the kernel
+ * rather than of the init system.
+ *
+ * `kill -0` and not a report, because a unit that has been booted out is one
+ * launchd no longer says anything about — including whether the process it was
+ * running has actually gone. It goes through `Run` for the reason everything
+ * else here does: a test drives it without a process on this machine being
+ * signalled. */
+async function alive(pid: number, run: Run): Promise<boolean> {
+  return (await run(["kill", "-0", String(pid)])).code === 0;
+}
+
 export class LaunchdService implements Service {
   readonly kind = "launchd" as const;
   readonly unitFile: string;
   readonly #label: string;
   readonly #env: Env;
   readonly #domain: string;
+  readonly #stopDeadlineMs: number;
 
   /** The label is a parameter for `Run`'s reason: a test that drives this
    * machine's real launchd has to do it under a name that is not the one the
    * machine's own supervisor is registered under. Nothing in ccmsg passes it —
-   * `serviceFor` is where the name is settled. */
-  constructor(env: Env, label: string = LAUNCHD_LABEL) {
+   * `serviceFor` is where the name is settled. The deadline is a parameter for
+   * the same reason: a test drives the escalation without waiting out ten
+   * seconds of it. */
+  constructor(env: Env, label: string = LAUNCHD_LABEL, stopDeadlineMs = STOP_DEADLINE_MS) {
     this.#env = env;
     this.#label = label;
+    this.#stopDeadlineMs = stopDeadlineMs;
     const home = env["HOME"] ?? homedir();
     this.unitFile = join(home, "Library", "LaunchAgents", `${label}.plist`);
     this.#domain = `gui/${String(process.getuid?.() ?? 0)}`;
@@ -261,8 +303,33 @@ export class LaunchdService implements Service {
     return await this.state(run);
   }
 
+  /** Take the unit out of launchd, and stay until the supervisor has gone.
+   *
+   * `bootout` rather than a signal, because `KeepAlive` means a supervisor that
+   * is signalled is one launchd starts again: what the person asked for is a
+   * supervisor that stays stopped, which is systemd's `stop` and launchd's
+   * `bootout`. The file stays where it is — a unit launchd is not holding is
+   * what `start` already knows how to bootstrap, and taking the file away as
+   * well is `unregister`.
+   *
+   * The answer waits for the pid rather than being read off the command,
+   * because the moment the command returns is the moment the supervisor starts
+   * leaving and not the one it finishes at: a supervisor wedged in its own
+   * shutdown would otherwise be reported as gone by a `service stop` that had
+   * only asked. One that will not go inside the deadline is killed. */
   async stop(run: Run): Promise<ServiceState> {
-    await run(["launchctl", "kill", "SIGTERM", `${this.#domain}/${this.#label}`]);
+    const before = await this.#report(run);
+    // A refusal is launchd saying it is not holding this label, which is the
+    // state being asked for; `state` below is what reports otherwise.
+    await run(["launchctl", "bootout", `${this.#domain}/${this.#label}`]);
+    const held = before.service?.pid ?? null;
+    if (held !== null) {
+      const gone = async () => !(await alive(held, run));
+      if (!(await departed(gone, this.#stopDeadlineMs))) {
+        await run(["kill", "-KILL", String(held)]);
+        await departed(gone, this.#stopDeadlineMs);
+      }
+    }
     return await this.state(run);
   }
 
@@ -306,9 +373,11 @@ class SystemdService implements Service {
   readonly kind = "systemd" as const;
   readonly unitFile: string;
   readonly #env: Env;
+  readonly #stopDeadlineMs: number;
 
-  constructor(env: Env) {
+  constructor(env: Env, stopDeadlineMs = STOP_DEADLINE_MS) {
     this.#env = env;
+    this.#stopDeadlineMs = stopDeadlineMs;
     const home = env["HOME"] ?? homedir();
     const config = env["XDG_CONFIG_HOME"] ?? join(home, ".config");
     this.unitFile = join(config, "systemd", "user", SYSTEMD_UNIT);
@@ -374,8 +443,26 @@ class SystemdService implements Service {
     return await this.state(run);
   }
 
+  /** `LaunchdService.stop`'s reasoning, in systemd's vocabulary.
+   *
+   * `stop` here holds until the unit is inactive on its own, so the wait below
+   * usually settles on its first question; it is asked anyway because what the
+   * caller is promised is the departure, not the command having returned, and
+   * a `stop` that gave up on its own `TimeoutStopSec` returns having left the
+   * process behind. */
   async stop(run: Run): Promise<ServiceState> {
+    const before = await this.#report(run);
     await run(["systemctl", "--user", "stop", SYSTEMD_UNIT]);
+    const held = before.service?.pid ?? null;
+    if (held !== null) {
+      // The unit is still loaded here, so systemd is still the one that knows
+      // what became of the process it started.
+      const gone = async () => (await this.#report(run)).service?.pid !== held;
+      if (!(await departed(gone, this.#stopDeadlineMs))) {
+        await run(["systemctl", "--user", "kill", "--signal=SIGKILL", SYSTEMD_UNIT]);
+        await departed(gone, this.#stopDeadlineMs);
+      }
+    }
     return await this.state(run);
   }
 

package/src/transport/listener.ts

@@ -4,6 +4,9 @@
  * everything else, because a client reads "the unix socket refuses" as the
  * instance having finished leaving, and a successor may take the resources it
  * sees freed before that. */
+/** A Bun listener gets this long to settle after releasing its address. */
+export const STOP_DEADLINE_MS = 250;
+
 export interface Listener {
   readonly kind: "uds" | "ws";
   /** The socket path, or the bound `host:port` — resolved, so an ephemeral
@@ -29,11 +32,13 @@ export class Transport {
    * come before this one (§8.5 1-4: refuse new work, stop upstream watches,
    * tell the connections, settle what is persisted). */
   async close(): Promise<void> {
-    const ordered = [
-      ...this.#listeners.filter((l) => l.kind !== "uds"),
-      ...this.#listeners.filter((l) => l.kind === "uds"),
-    ];
+    const held = [...this.#listeners];
     this.#listeners.length = 0;
-    for (const listener of ordered) await listener.close();
+    // The order that matters is the unix socket coming last; among the rest
+    // there is none, and closing them one after another would add up their
+    // deadlines for no reason — two served listeners is a second of waiting
+    // that nothing is waiting for.
+    await Promise.all(held.filter((l) => l.kind !== "uds").map((l) => l.close()));
+    for (const listener of held.filter((l) => l.kind === "uds")) await listener.close();
   }
 }

package/src/transport/uds.ts

@@ -81,8 +81,11 @@ export function listenUds(options: UdsOptions): Listener {
   return {
     kind: "uds",
     address: options.path,
-    async close() {
+    close() {
+      // Nothing to wait on: a unix listener gives its address up inside the
+      // call, unlike the served WebSocket next door (Bun 1.3.13).
       server.stop(true);
+      return Promise.resolve();
     },
   };
 }

package/src/transport/ws.ts

@@ -3,7 +3,7 @@ import { BaseConn, type Conn, type ConnRegistry } from "./conn.ts";
 import { createDriver, type FrameHandler } from "./driver.ts";
 import { LineReader, WriteQueue } from "./framing.ts";
 import { type AuthorizedUpgrade, type EntryPolicy, OPEN } from "./entry.ts";
-import type { Listener } from "./listener.ts";
+import { type Listener, STOP_DEADLINE_MS } from "./listener.ts";
 
 /** What the upgrade hands the socket: whether it was let in as a peer. */
 interface UpgradeData {
@@ -186,8 +186,4 @@ export function entryPath(pathname: string, path: string): boolean {
   return pathname === path || pathname.endsWith(`/${path.replace(/^\//, "")}`);
 }
 
-/** How long the stop above waits before trusting the address over the promise.
- * Two orders of magnitude above the millisecond the release was measured at. */
-const STOP_DEADLINE_MS = 250;
-
 const NEWLINE = new Uint8Array([0x0a]);