@gajae-gateway/cli 0.1.1 → 0.1.2

package/src/restart-stack.ts

@@ -0,0 +1,361 @@
+import { randomUUID } from "node:crypto";
+import { mkdir, readFile, rename, stat, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import {
+	currentPlatform,
+	restartOrder,
+	restartStackCommands,
+	type ServicePlatform,
+	serviceSpecs,
+	systemdUnitName,
+} from "./services";
+
+/**
+ * `gajaeway ops restart-stack` (issue #54).
+ *
+ * A persona turn that restarts the gateway is killed with it, so running the
+ * ordered restart inline stopped between the gateway and its adapters and left
+ * the adapters on the old binary while the caller reported success. The
+ * sequence therefore runs in a detached supervisor (its own session, so the
+ * caller's process group and the service manager's job kill do not reach it),
+ * verifies every service against its deployed binary, and records a receipt
+ * under `$GAJAEWAY_HOME` that the next turn reads with `--status`.
+ */
+
+export type RestartStepResult = "pending" | "ok" | "stale" | "failed" | "skipped";
+export type RestartState = "queued" | "running" | "ok" | "stale" | "failed";
+
+export interface RestartStep {
+	readonly label: string;
+	/** Absent on systemd, where the gateway unit's restart carries its dependents. */
+	command?: readonly string[];
+	exitStatus?: number;
+	result: RestartStepResult;
+	pid?: number;
+	processStartedAt?: string;
+	binary?: string;
+	binaryModifiedAt?: string;
+	detail?: string;
+}
+
+export interface RestartReceipt {
+	readonly id: string;
+	state: RestartState;
+	readonly platform: ServicePlatform;
+	readonly requestedAt: string;
+	supervisorPid?: number;
+	finishedAt?: string;
+	readonly steps: RestartStep[];
+}
+
+/** A running service process as the service manager and `ps` report it. */
+export interface ServiceProcess {
+	readonly pid: number;
+	/** Epoch ms; `ps` reports whole seconds. */
+	readonly startedAt: number;
+	readonly binary: string;
+}
+
+export type CommandRunner = (command: readonly string[]) => number | PromiseLike<number>;
+export type ServiceProbe = (label: string) => Promise<ServiceProcess | undefined>;
+
+export interface RunRestartOptions {
+	readonly home: string;
+	readonly id: string;
+	readonly platform?: ServicePlatform;
+	readonly uid?: number;
+	readonly runner?: CommandRunner;
+	readonly probe?: ServiceProbe;
+	/** Epoch ms modification time of a deployed binary. */
+	readonly binaryModifiedAt?: (path: string) => Promise<number>;
+	readonly now?: () => number;
+	readonly sleep?: (ms: number) => Promise<void>;
+	/** How long one service may take to come back verified; covers the gateway's ordered shutdown. */
+	readonly verifyTimeoutMs?: number;
+	readonly pollMs?: number;
+}
+
+const RECEIPT_FILE = "restart-stack.json";
+const VERIFY_TIMEOUT_MS = 90_000;
+const POLL_MS = 1_000;
+
+export function restartReceiptPath(home: string): string {
+	return join(home, RECEIPT_FILE);
+}
+
+export async function readRestartReceipt(home: string): Promise<RestartReceipt | undefined> {
+	let raw: string;
+	try {
+		raw = await readFile(restartReceiptPath(home), "utf8");
+	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined;
+		throw error;
+	}
+	return JSON.parse(raw) as RestartReceipt;
+}
+
+async function writeReceipt(home: string, receipt: RestartReceipt): Promise<void> {
+	await mkdir(home, { recursive: true });
+	const path = restartReceiptPath(home);
+	const temporary = `${path}.${process.pid}.tmp`;
+	await writeFile(temporary, `${JSON.stringify(receipt, null, 2)}\n`, { mode: 0o600 });
+	await rename(temporary, path);
+}
+
+function newReceipt(id: string, platform: ServicePlatform, uid: number | undefined, now: number): RestartReceipt {
+	const commands = restartStackCommands(platform, uid);
+	return {
+		id,
+		state: "queued",
+		platform,
+		requestedAt: new Date(now).toISOString(),
+		steps: restartOrder().map((spec, index) => ({
+			label: spec.label,
+			...(platform === "darwin" ? { command: commands[index] } : index === 0 ? { command: commands[0] } : {}),
+			result: "pending",
+		})),
+	};
+}
+
+async function defaultCommandRunner(command: readonly string[]): Promise<number> {
+	const child = Bun.spawn([...command], { stdin: "ignore", stdout: "ignore", stderr: "ignore" });
+	return await child.exited;
+}
+
+async function capture(command: readonly string[]): Promise<string | undefined> {
+	const child = Bun.spawn([...command], {
+		stdin: "ignore",
+		stdout: "pipe",
+		stderr: "ignore",
+		env: { ...process.env, LC_ALL: "C" },
+	});
+	const output = await new Response(child.stdout).text();
+	return (await child.exited) === 0 ? output : undefined;
+}
+
+/** Reads the live pid from the service manager, then its start time and executable from `ps`. */
+export function defaultServiceProbe(platform: ServicePlatform): ServiceProbe {
+	return async (label) => {
+		const spec = serviceSpecs().find((candidate) => candidate.label === label);
+		if (spec === undefined) throw new Error(`unknown service label: ${label}`);
+		let pid: number | undefined;
+		if (platform === "darwin") {
+			const listed = await capture(["launchctl", "list", label]);
+			const match = listed?.match(/"PID"\s*=\s*(\d+);/);
+			pid = match ? Number(match[1]) : undefined;
+		} else {
+			const shown = await capture(["systemctl", "--user", "show", "-p", "MainPID", "--value", systemdUnitName(spec)]);
+			pid = shown === undefined ? undefined : Number(shown.trim());
+		}
+		if (pid === undefined || !Number.isInteger(pid) || pid <= 0) return undefined;
+		const lstart = await capture(["ps", "-o", "lstart=", "-p", String(pid)]);
+		const args = await capture(["ps", "-ww", "-o", "args=", "-p", String(pid)]);
+		if (lstart === undefined || args === undefined) return undefined;
+		const startedAt = Date.parse(lstart.trim());
+		if (!Number.isFinite(startedAt)) return undefined;
+		// The service definitions exec `<bin-dir>/<binary> <fixed args>`; stripping
+		// the known arguments keeps a bin dir containing spaces intact.
+		const suffix = spec.args.length > 0 ? ` ${spec.args.join(" ")}` : "";
+		const line = args.trim();
+		const binary = suffix && line.endsWith(suffix) ? line.slice(0, -suffix.length) : line;
+		return { pid, startedAt, binary };
+	};
+}
+
+async function defaultBinaryModifiedAt(path: string): Promise<number> {
+	return (await stat(path)).mtimeMs;
+}
+
+function floorSecond(ms: number): number {
+	return Math.floor(ms / 1_000) * 1_000;
+}
+
+/**
+ * Runs the ordered restart, verifying each service before the next command,
+ * and persists the receipt after every transition. The first service that is
+ * stale or missing aborts the remainder.
+ */
+export async function runRestartStack(options: RunRestartOptions): Promise<RestartReceipt> {
+	const platform = options.platform ?? currentPlatform();
+	const runner = options.runner ?? defaultCommandRunner;
+	const probe = options.probe ?? defaultServiceProbe(platform);
+	const modifiedAt = options.binaryModifiedAt ?? defaultBinaryModifiedAt;
+	const now = options.now ?? Date.now;
+	const sleep = options.sleep ?? ((ms: number) => Bun.sleep(ms));
+	const timeoutMs = options.verifyTimeoutMs ?? VERIFY_TIMEOUT_MS;
+	const pollMs = options.pollMs ?? POLL_MS;
+
+	const existing = await readRestartReceipt(options.home);
+	const receipt =
+		existing?.id === options.id && existing.state === "queued"
+			? existing
+			: newReceipt(options.id, platform, options.uid, now());
+	receipt.state = "running";
+	receipt.supervisorPid = process.pid;
+	await writeReceipt(options.home, receipt);
+
+	let issuedAt = now();
+	for (const step of receipt.steps) {
+		if (receipt.state !== "running") {
+			step.result = "skipped";
+			continue;
+		}
+		try {
+			if (step.command !== undefined) {
+				issuedAt = now();
+				const status = await runner(step.command);
+				step.exitStatus = status;
+				if (status !== 0) {
+					step.result = "failed";
+					step.detail = `${step.command.join(" ")} exited with status ${status}`;
+				}
+			}
+			if (step.result === "pending") await verifyStep(step, issuedAt);
+		} catch (error) {
+			step.result = "failed";
+			step.detail = error instanceof Error ? error.message : String(error);
+		}
+		if (step.result !== "ok") receipt.state = step.result === "stale" ? "stale" : "failed";
+		await writeReceipt(options.home, receipt);
+	}
+	if (receipt.state === "running") receipt.state = "ok";
+	receipt.finishedAt = new Date(now()).toISOString();
+	await writeReceipt(options.home, receipt);
+	return receipt;
+
+	async function verifyStep(step: RestartStep, since: number): Promise<void> {
+		const deadline = now() + timeoutMs;
+		for (;;) {
+			const running = await probe(step.label);
+			let binaryModifiedAt = 0;
+			if (running !== undefined) {
+				binaryModifiedAt = await modifiedAt(running.binary);
+				step.pid = running.pid;
+				step.processStartedAt = new Date(running.startedAt).toISOString();
+				step.binary = running.binary;
+				step.binaryModifiedAt = new Date(binaryModifiedAt).toISOString();
+				// `ps` has one-second resolution, so the bound is floored to match.
+				if (running.startedAt >= floorSecond(Math.max(binaryModifiedAt, since))) {
+					step.result = "ok";
+					delete step.detail;
+					return;
+				}
+			}
+			if (now() >= deadline) {
+				if (running === undefined) {
+					step.result = "failed";
+					step.detail = "no running process after restart";
+				} else {
+					step.result = "stale";
+					step.detail =
+						running.startedAt < binaryModifiedAt
+							? "process started before the deployed binary was modified"
+							: "process was not restarted by this sequence";
+				}
+				return;
+			}
+			await sleep(pollMs);
+		}
+	}
+}
+
+export interface LaunchRestartOptions {
+	readonly home: string;
+	readonly platform?: ServicePlatform;
+	readonly uid?: number;
+	/** Argv that re-enters the CLI; `--run <id>` is appended. */
+	readonly worker?: readonly string[];
+	readonly env?: NodeJS.ProcessEnv;
+}
+
+/** The argv that runs this CLI again, whether compiled or run from source. */
+export function cliCommand(): string[] {
+	return Bun.main.startsWith("/$bunfs/") ? [process.execPath] : [process.execPath, Bun.main];
+}
+
+function alive(pid: number | undefined): boolean {
+	if (pid === undefined) return false;
+	try {
+		process.kill(pid, 0);
+		return true;
+	} catch {
+		return false;
+	}
+}
+
+/**
+ * Queues a restart and hands it to a detached supervisor, returning as soon as
+ * the supervisor is spawned so the caller may be killed by the restart itself.
+ */
+export async function launchRestartStack(
+	options: LaunchRestartOptions,
+): Promise<{ readonly receipt: RestartReceipt; readonly supervisorPid: number }> {
+	const platform = options.platform ?? currentPlatform();
+	const existing = await readRestartReceipt(options.home);
+	if (existing?.state === "running" && alive(existing.supervisorPid))
+		throw new Error(`restart-stack ${existing.id} is still running (supervisor pid ${existing.supervisorPid})`);
+	const receipt = newReceipt(randomUUID(), platform, options.uid, Date.now());
+	await writeReceipt(options.home, receipt);
+	const worker = [...(options.worker ?? [...cliCommand(), "ops", "restart-stack"]), "--run", receipt.id];
+	const child = Bun.spawn(supervisorArgv(platform, worker, options.home, receipt.id), {
+		detached: true,
+		stdin: "ignore",
+		stdout: "ignore",
+		stderr: "ignore",
+		env: { ...(options.env ?? process.env), GAJAEWAY_HOME: options.home },
+	});
+	child.unref();
+	return { receipt, supervisorPid: child.pid };
+}
+
+/**
+ * A new session escapes the caller's process group, which is enough under
+ * launchd. Under systemd the caller usually lives in a service cgroup that the
+ * restart it requests may stop, so the supervisor is started as its own
+ * transient user unit instead.
+ */
+export function supervisorArgv(
+	platform: ServicePlatform,
+	worker: readonly string[],
+	home: string,
+	id: string,
+): string[] {
+	if (platform === "darwin") return [...worker];
+	return [
+		"systemd-run",
+		"--user",
+		"--collect",
+		"--quiet",
+		`--unit=gajaeway-restart-stack-${id}`,
+		`--setenv=GAJAEWAY_HOME=${home}`,
+		"--",
+		...worker,
+	];
+}
+
+/**
+ * The receipt's state, except that an unfinished sequence whose supervisor is
+ * gone is reported as interrupted rather than left looking in progress.
+ */
+export function effectiveRestartState(receipt: RestartReceipt): RestartState | "interrupted" {
+	if (receipt.state === "running" && !alive(receipt.supervisorPid)) return "interrupted";
+	return receipt.state;
+}
+
+export function renderRestartReceipt(receipt: RestartReceipt): string[] {
+	const lines = [
+		`restart-stack ${receipt.id}: ${effectiveRestartState(receipt)}`,
+		`requestedAt: ${receipt.requestedAt}`,
+		`finishedAt: ${receipt.finishedAt ?? "-"}`,
+	];
+	for (const step of receipt.steps) {
+		const facts = [
+			step.pid === undefined ? undefined : `pid=${step.pid}`,
+			step.processStartedAt === undefined ? undefined : `started=${step.processStartedAt}`,
+			step.binaryModifiedAt === undefined ? undefined : `binary=${step.binaryModifiedAt}`,
+			step.detail,
+		].filter((fact) => fact !== undefined);
+		lines.push(`${step.label}: ${step.result}${facts.length > 0 ? ` ${facts.join(" ")}` : ""}`);
+	}
+	return lines;
+}

package/src/services.ts

@@ -23,13 +23,20 @@ export interface ResolvePathOptions {
 	readonly loginPathRunner?: LoginPathRunner;
 }
 
+/** Service managers this CLI can write definitions for. */
+export type ServicePlatform = "darwin" | "linux";
+
 export interface InstallServicesOptions {
 	readonly binDir: string;
 	readonly launchAgentsDir?: string;
+	/** systemd user-unit directory; defaults to ${XDG_CONFIG_HOME:-$HOME/.config}/systemd/user. */
+	readonly unitDir?: string;
+	/** Service manager to generate for; defaults to the running platform. */
+	readonly platform?: ServicePlatform;
 	readonly env?: NodeJS.ProcessEnv;
 	/** Test seam for the `${SHELL} -lc` invocation used by the default path. */
 	readonly loginPathRunner?: LoginPathRunner;
-	/** Test seam for counting or inspecting the plist writes. */
+	/** Test seam for counting or inspecting the definition writes. */
 	readonly writeFile?: PlistWriter;
 }
 
@@ -38,25 +45,47 @@ interface ServiceSpec {
 	readonly label: string;
 	readonly binary: string;
 	readonly args: readonly string[];
+	/**
+	 * True for every process that holds a live connection to the gateway. A
+	 * gateway restart leaves such a process attached to the previous generation:
+	 * it neither dies nor loses messages, so no health check catches it, and the
+	 * symptom is replies arriving one beat late (issue #251). The service
+	 * definition, not an operator convention, has to restart it.
+	 */
+	readonly dependsOnGateway: boolean;
 }
 
+export const GATEWAY_UNIT = "gajaeway-gateway.service";
+
 const SERVICE_SPECS: readonly ServiceSpec[] = [
-	{ id: "gateway", label: "dev.gajaeway.gateway", binary: "gajaeway-gateway", args: ["daemon"] },
+	{
+		id: "gateway",
+		label: "dev.gajaeway.gateway",
+		binary: "gajaeway-gateway",
+		args: ["daemon"],
+		dependsOnGateway: false,
+	},
 	{
 		id: "adapter-discord",
 		label: "dev.gajaeway.adapter-discord",
 		binary: "gajaeway-discord",
 		args: [],
+		dependsOnGateway: true,
 	},
 	{
 		id: "adapter-slack",
 		label: "dev.gajaeway.adapter-slack",
 		binary: "gajaeway-slack",
 		args: [],
+		dependsOnGateway: true,
 	},
-	{ id: "admin", label: "dev.gajaeway.admin", binary: "gajaeway-admin", args: ["serve"] },
+	{ id: "admin", label: "dev.gajaeway.admin", binary: "gajaeway-admin", args: ["serve"], dependsOnGateway: true },
 ];
 
+export function serviceSpecs(): readonly ServiceSpec[] {
+	return SERVICE_SPECS;
+}
+
 function homeForEnvironment(env: NodeJS.ProcessEnv): string {
 	return env.GAJAEWAY_HOME || join(env.HOME || homedir(), ".gajaeway");
 }
@@ -161,6 +190,92 @@ export function renderLaunchAgent(spec: ServiceSpec, binDir: string, home: strin
 `;
 }
 
+/**
+ * systemd reads `Environment=` with its own quoting rules, so a value is wrapped
+ * in double quotes with backslashes and quotes escaped. PATH entries containing
+ * a space would otherwise silently split into two assignments.
+ */
+function unitEnvironment(name: string, value: string): string {
+	return `Environment="${name}=${value.replaceAll("\\", "\\\\").replaceAll('"', '\\"')}"`;
+}
+
+export function systemdUnitName(spec: ServiceSpec): string {
+	return `gajaeway-${spec.id}.service`;
+}
+
+export function renderSystemdUnit(spec: ServiceSpec, binDir: string, home: string, path: string): string {
+	const execStart = [join(binDir, spec.binary), ...spec.args].join(" ");
+	const unit = [
+		"[Unit]",
+		`Description=gajaeway ${spec.id}`,
+		// BindsTo restarts and stops this unit with the gateway; PartOf propagates a
+		// gateway restart even when this unit is started on its own; After orders
+		// the boot so the adapter connects to a listening socket.
+		...(spec.dependsOnGateway ? [`BindsTo=${GATEWAY_UNIT}`, `After=${GATEWAY_UNIT}`, `PartOf=${GATEWAY_UNIT}`] : []),
+		"",
+		"[Service]",
+		`ExecStart=${execStart}`,
+		`WorkingDirectory=${binDir}`,
+		unitEnvironment("GAJAEWAY_HOME", home),
+		unitEnvironment("PATH", path),
+		"Restart=always",
+		"RestartSec=2",
+		// Only the gateway: GJC daemon and session hosts share the gateway cgroup,
+		// so the default control-group kill would take unrelated sessions with it.
+		// The gateway exits on its own inside 25s (SHUTDOWN_DEADLINE_MS in the
+		// gateway main); the unit's stop window is pinned so that ceiling always fits.
+		...(spec.dependsOnGateway ? [] : ["KillMode=process", "TimeoutStopSec=30s"]),
+		"",
+		"[Install]",
+		// Enabling the gateway pulls the whole stack in; a dependent is never
+		// wanted by default.target on its own, because it cannot run without one.
+		`WantedBy=${spec.dependsOnGateway ? GATEWAY_UNIT : "default.target"}`,
+		"",
+	];
+	return unit.join("\n");
+}
+
+function systemdUnitDir(env: NodeJS.ProcessEnv, userHome: string): string {
+	const base = env.XDG_CONFIG_HOME || join(userHome, ".config");
+	return join(base, "systemd", "user");
+}
+
+/**
+ * The ordered commands that realign the whole stack.
+ *
+ * systemd expresses the dependency itself, so one restart of the gateway unit
+ * is the entire operation. launchd has no BindsTo/PartOf equivalent and
+ * `WatchPaths` does not restart an already-running job, so the ordering is
+ * explicit here instead — and `bootout` never appears, because removing a job
+ * leaves it with no automatic recovery.
+ */
+export function restartStackCommands(platform: ServicePlatform, uid?: number): readonly (readonly string[])[] {
+	if (platform === "linux") return [["systemctl", "--user", "restart", GATEWAY_UNIT]];
+	if (platform === "darwin") {
+		const target = uid ?? process.getuid?.() ?? 0;
+		return restartOrder().map((spec) => ["launchctl", "kickstart", "-k", `gui/${target}/${spec.label}`]);
+	}
+	throw new Error(`unsupported service platform: ${platform}`);
+}
+
+/** The gateway first, then every job that holds a connection to it. */
+export function restartOrder(): readonly ServiceSpec[] {
+	return [
+		...SERVICE_SPECS.filter((spec) => !spec.dependsOnGateway),
+		...SERVICE_SPECS.filter((spec) => spec.dependsOnGateway),
+	];
+}
+
+function requirePlatform(platform: string): ServicePlatform {
+	if (platform !== "darwin" && platform !== "linux") throw new Error(`unsupported service platform: ${platform}`);
+	return platform;
+}
+
+export function currentPlatform(): ServicePlatform {
+	if (process.platform === "darwin" || process.platform === "linux") return process.platform;
+	throw new Error(`unsupported service platform: ${process.platform}`);
+}
+
 async function readRuntime(home: string): Promise<RuntimeConfig | undefined> {
 	const configPath = join(home, "config.json");
 	let raw: string;
@@ -191,10 +306,14 @@ async function readRuntime(home: string): Promise<RuntimeConfig | undefined> {
 export async function installServices(options: InstallServicesOptions): Promise<readonly string[]> {
 	if (options.binDir.length === 0) throw new Error("services requires a non-empty --bin-dir DIR");
 	const env = options.env ?? process.env;
+	const platform = options.platform === undefined ? currentPlatform() : requirePlatform(options.platform);
 	const userHome = env.HOME || homedir();
 	const home = homeForEnvironment(env);
 	const binDir = expandHome(options.binDir, userHome);
-	const launchAgentsDir = expandHome(options.launchAgentsDir ?? join(userHome, "Library", "LaunchAgents"), userHome);
+	const targetDir =
+		platform === "darwin"
+			? expandHome(options.launchAgentsDir ?? join(userHome, "Library", "LaunchAgents"), userHome)
+			: expandHome(options.unitDir ?? systemdUnitDir(env, userHome), userHome);
 	const runtime = await readRuntime(home);
 	const path = await resolvePath({
 		binDir,
@@ -203,22 +322,26 @@ export async function installServices(options: InstallServicesOptions): Promise<
 		env,
 		loginPathRunner: options.loginPathRunner,
 	});
-	await mkdir(launchAgentsDir, { recursive: true, mode: 0o700 });
-	const plists = SERVICE_SPECS.map((spec) => ({
-		path: join(launchAgentsDir, `dev.gajaeway.${spec.id}.plist`),
-		contents: renderLaunchAgent(spec, binDir, home, path),
-	}));
+	await mkdir(targetDir, { recursive: true, mode: 0o700 });
+	const definitions = SERVICE_SPECS.map((spec) =>
+		platform === "darwin"
+			? {
+					path: join(targetDir, `dev.gajaeway.${spec.id}.plist`),
+					contents: renderLaunchAgent(spec, binDir, home, path),
+				}
+			: { path: join(targetDir, systemdUnitName(spec)), contents: renderSystemdUnit(spec, binDir, home, path) },
+	);
 	const write = options.writeFile;
-	for (const plist of plists) {
-		if (write) await write(plist.path, plist.contents);
+	for (const definition of definitions) {
+		if (write) await write(definition.path, definition.contents);
 		else {
-			await writeFile(plist.path, plist.contents, { encoding: "utf8", mode: 0o600 });
-			await chmod(plist.path, 0o600);
+			await writeFile(definition.path, definition.contents, { encoding: "utf8", mode: 0o600 });
+			await chmod(definition.path, 0o600);
 		}
 	}
-	return plists.map((plist) => plist.path);
+	return definitions.map((definition) => definition.path);
 }
 
 export function serviceUsage(): string {
-	return "usage: gajaeway services install|repair --bin-dir DIR [--launch-agents-dir DIR]";
+	return "usage: gajaeway services install|repair --bin-dir DIR [--launch-agents-dir DIR] [--unit-dir DIR] [--platform darwin|linux]";
 }