@tikoci/quickchr 0.4.7 → 0.5.0

package/src/lib/provisioning-window.ts

@@ -0,0 +1,356 @@
+/**
+ * The provisioning window: when quickchr may run provisioning against a guest,
+ * and what happens to an option that arrives after it has closed (#176).
+ *
+ * Provisioning expects the guest to hold RouterOS' default configuration — it
+ * creates the first account, flips device-mode, installs packages onto a factory
+ * image. Once the guest has booted, quickchr no longer knows what is in it, so the
+ * window closes and a provisioning option can no longer be honoured blindly.
+ *
+ * What it must never do is accept the option and drop it. Until #176 every
+ * provisioning option handed to `start()` on a machine past its first boot was
+ * parsed, accepted and discarded — no warning, no error, normal boot time, and the
+ * only way to find out was to read the value back. This module is the refusal:
+ * an option either applies, is already satisfied, or is rejected by name with the
+ * route that would work.
+ */
+
+import { getInstanceCredentials } from "./credentials.ts";
+import { shellQuote } from "./names.ts";
+import {
+	formatDeviceModeFlags,
+	formatDeviceModeSelection,
+	mergeDeviceModeOptions,
+	type ResolvedDeviceModeOptions,
+	resolveDeviceModeOptions,
+	shouldApplyDeviceMode,
+} from "./device-mode.ts";
+import {
+	type DeviceModeOptions,
+	type LicenseInput,
+	type MachineState,
+	type ProvisioningStep,
+	QuickCHRError,
+} from "./types.ts";
+
+/** The provisioning options `start()` accepts, in one shape. */
+export interface ProvisioningRequest {
+	installAllPackages?: boolean;
+	packages?: string[];
+	deviceMode?: DeviceModeOptions;
+	user?: { name: string; password: string };
+	disableAdmin?: boolean;
+	license?: LicenseInput;
+	secureLogin?: boolean;
+}
+
+/**
+ * Is the guest still in the state provisioning expects?
+ *
+ * Deliberately **not** `!state.lastStartedAt`. That field is stamped immediately
+ * after `spawnQemu`, before the guest is known to have booted and before
+ * provisioning runs, so it answers "has QEMU been launched" — a different question
+ * that happens to have the same answer most of the time. `clean()` is where the two
+ * come apart: it replaces the disk with the factory image, so the guest genuinely is
+ * fresh again, and gating on `lastStartedAt` left such a machine unable to be
+ * provisioned ever again. `clean()` now clears both fields, and this reads both:
+ * `provisioning` is the positive record, `lastStartedAt` the conservative fallback
+ * for a guest that booted at least once and may hold configuration quickchr did not
+ * put there — including machines created before `provisioning` existed.
+ */
+export function isProvisioningWindowOpen(state: MachineState): boolean {
+	return !state.provisioning && !state.lastStartedAt;
+}
+
+/** One provisioning option that was asked for on a machine past its first boot. */
+export interface ProvisioningAsk {
+	step: ProvisioningStep;
+	/** What was asked for, as the refusal should name it. */
+	description: string;
+	/** True when machine state already records this exact value, making it a no-op. */
+	satisfied: boolean;
+}
+
+/** A device-mode request flattened to `setting -> value`: `mode` plus one entry per
+ *  feature, normalized and order-independent. Empty when nothing is being asked for. */
+function deviceModeSelection(options?: DeviceModeOptions): Record<string, string> {
+	const resolved = resolveDeviceModeOptions(options);
+	if (!shouldApplyDeviceMode(resolved)) return {};
+	const selection: Record<string, string> = {};
+	if (resolved.mode) selection.mode = resolved.mode;
+	for (const [feature, value] of Object.entries(resolved.features)) {
+		selection[feature] = value;
+	}
+	return selection;
+}
+
+/** Does the applied record already carry every setting this request names?
+ *
+ *  **Subset, not equality**, because the record is cumulative and the request is not.
+ *  A device-mode update moves only the settings it names, so a machine that took
+ *  `--device-mode-enable container` and later `--device-mode-disable smb` has a record
+ *  of all three settings — and an exact comparison then refuses the very
+ *  `--device-mode-enable container` that a script has been passing on every start
+ *  since before the second change, naming a `quickchr set` command that is a no-op.
+ *  Every field the caller asked about matches; the fields they did not mention are
+ *  not theirs to match.
+ *
+ *  A field that is absent from the record, or present with a different value, is
+ *  still pending — this loosens what counts as equal, not what counts as applied. */
+function deviceModeSatisfiedBy(
+	applied: Record<string, string>,
+	requested: Record<string, string>,
+): boolean {
+	const entries = Object.entries(requested);
+	return entries.length > 0 && entries.every(([setting, value]) => applied[setting] === value);
+}
+
+function licenseLevelOf(license: LicenseInput): string {
+	return (typeof license === "string" ? license : license.level) ?? "p1";
+}
+
+/** Did the request's password actually reach the guest? The persisted `state.user`
+ *  carries a placeholder, never the password, so the comparison goes to the secret
+ *  store. An unreadable store answers "no": a name match alone would let a password
+ *  *change* pass as a no-op and be dropped in silence, which is the whole bug. */
+function userCredentialMatches(state: MachineState, user: { name: string; password: string }): boolean {
+	if (state.user?.name !== user.name) return false;
+	try {
+		const stored = getInstanceCredentials(state.name);
+		return stored?.user === user.name && stored.password === user.password;
+	} catch {
+		return false;
+	}
+}
+
+/**
+ * Which provisioning steps this request actually asks for, and which of them already
+ * landed on the current guest.
+ *
+ * **Satisfaction requires the step in `provisioning.steps`, not just a matching value
+ * in `machine.json`.** Most of those fields — `installAllPackages`, `packages`,
+ * `deviceMode`, `secureLogin` — are *desired config*, written at `add()` before
+ * anything ran. Comparing against them alone would call a first boot that threw
+ * half-way through "already applied" and drop the retry in silence, which is exactly
+ * the failure this module exists to prevent. The step record is the only evidence that
+ * a step ran, so it gates every row.
+ *
+ * Even then this is a claim about what quickchr applied, not about what is in the
+ * guest — quickchr cannot read the guest without booting it, which is the thing being
+ * decided. It exists so a caller that passes the same options on every start (the
+ * shape a script naturally takes) is not punished for asking for what it already got.
+ */
+export function classifyProvisioningRequest(
+	state: MachineState,
+	request: ProvisioningRequest,
+): ProvisioningAsk[] {
+	const asks: ProvisioningAsk[] = [];
+	const applied = new Set(state.provisioning?.steps ?? []);
+
+	if (request.installAllPackages) {
+		asks.push({
+			step: "packages",
+			description: "--install-all-packages",
+			satisfied: applied.has("packages") && state.installAllPackages === true,
+		});
+	} else if (request.packages?.length) {
+		const installed = new Set(state.packages ?? []);
+		asks.push({
+			step: "packages",
+			description: `packages ${request.packages.join(", ")}`,
+			satisfied: applied.has("packages") && request.packages.every((pkg) => installed.has(pkg)),
+		});
+	}
+
+	const requestedDeviceMode = deviceModeSelection(request.deviceMode);
+	if (Object.keys(requestedDeviceMode).length > 0) {
+		asks.push({
+			step: "deviceMode",
+			description: `device-mode (${formatDeviceModeSelection(resolveDeviceModeOptions(request.deviceMode))})`,
+			satisfied: applied.has("deviceMode")
+				&& deviceModeSatisfiedBy(deviceModeSelection(state.deviceMode), requestedDeviceMode),
+		});
+	}
+
+	if (request.license) {
+		const level = licenseLevelOf(request.license);
+		asks.push({
+			step: "license",
+			description: `license (${level})`,
+			satisfied: applied.has("license") && state.licenseLevel === level,
+		});
+	}
+
+	if (request.user) {
+		asks.push({
+			step: "user",
+			description: `user "${request.user.name}"`,
+			satisfied: applied.has("user") && userCredentialMatches(state, request.user),
+		});
+	}
+
+	if (request.disableAdmin) {
+		asks.push({
+			step: "disableAdmin",
+			description: "--disable-admin",
+			satisfied: applied.has("disableAdmin") && state.disableAdmin === true,
+		});
+	}
+
+	if (request.secureLogin === true) {
+		asks.push({
+			step: "secureLogin",
+			description: "managed login (--secure-login)",
+			satisfied: applied.has("secureLogin") && state.secureLogin === true,
+		});
+	}
+
+	return asks;
+}
+
+/**
+ * The device-mode record to persist after a post-boot apply.
+ *
+ * Cumulative, because the guest is: a device-mode update moves only the settings it
+ * names, so replacing the record with the request alone would leave `machine.json`
+ * claiming an earlier feature had never been asked for while the guest still had it.
+ *
+ * **But it only carries forward what actually ran.** `state.deviceMode` is desired
+ * config, written at `add()` before anything happens. A first boot that threw before
+ * the device-mode step leaves that intent sitting in state having never reached the
+ * guest — and folding it in here, then stamping `deviceMode` as applied, would make a
+ * later `start` treat settings the guest never received as already satisfied and drop
+ * them in silence. That is this module's own bug wearing a new hat, so it gets this
+ * module's own rule: the step record is the only evidence that a step ran.
+ */
+export function appliedDeviceModeRecord(
+	state: MachineState,
+	applied: ResolvedDeviceModeOptions,
+): DeviceModeOptions {
+	const priorRan = state.provisioning?.steps.includes("deviceMode") ?? false;
+	return mergeDeviceModeOptions(priorRan ? state.deviceMode : undefined, applied);
+}
+
+/**
+ * What the first boot of an `add()`-created machine should provision: the caller's
+ * options where given, the intent `add()` recorded where not.
+ *
+ * **`undefined` means "not given", and only `undefined` falls through.** An explicit
+ * `false` is an instruction — `--no-install-all-packages` on `start` overrides the
+ * `add`-time intent, as `--no-device-mode` does. The CLI used to send `false` for every
+ * absent boolean flag, so `false ?? existing.installAllPackages` discarded what `add`
+ * stored and a plain `quickchr start <name>` booted without it (#185). The CLI now
+ * sends `undefined` for an absent flag; this function is where the rule is enforced.
+ *
+ * `license` is not merged: quickchr never persists license credentials, so there is
+ * no stored intent to fall back to.
+ */
+export function pendingProvisioningFor(
+	existing: MachineState,
+	request: ProvisioningRequest,
+): ProvisioningRequest {
+	return {
+		installAllPackages: request.installAllPackages ?? existing.installAllPackages,
+		packages: request.packages?.length ? request.packages : (existing.packages.length > 0 ? existing.packages : undefined),
+		deviceMode: request.deviceMode ?? existing.deviceMode,
+		user: request.user ?? existing.user,
+		disableAdmin: request.disableAdmin ?? existing.disableAdmin,
+		license: request.license,
+		secureLogin: request.secureLogin ?? existing.secureLogin,
+	};
+}
+
+/** True when any step in this request would change something. */
+export function hasProvisioningRequest(request: ProvisioningRequest): boolean {
+	return !!(
+		request.installAllPackages ||
+		(request.packages?.length ?? 0) > 0 ||
+		shouldApplyDeviceMode(resolveDeviceModeOptions(request.deviceMode)) ||
+		request.license ||
+		request.user ||
+		request.disableAdmin ||
+		request.secureLogin === true
+	);
+}
+
+/** Where a step can still be applied once the window has closed. Every step has a
+ *  route — `clean()` resets the disk and reopens the window for all of them — but
+ *  the cheaper one is named first where it exists.
+ *
+ *  **Every route is a line you can paste and run**, with its caveat as a comment in
+ *  that line's own language — `#` for a shell command, `//` for the library API. This
+ *  is not cosmetic: the caveats used to be parenthetical prose appended to the command
+ *  (`quickchr set lab … (power-cycles the machine)`), and `(` opens a subshell, so the
+ *  advertised recovery was a bash syntax error. A route that cannot be run as printed
+ *  is a route that does not work, which is the whole thing this module exists to stop.
+ *
+ *  The `clean()` route says **repeat the original command**, not "start it again".
+ *  A refused request is never persisted, and `clean()` clears `user` and
+ *  `disableAdmin` because they are guest state, so a plain start after the reset
+ *  would quietly drop exactly the options that were just refused. (`packages`,
+ *  `deviceMode` and `secureLogin` are retained intent and would come back on their
+ *  own — repeating the command is simply correct for all of them.)
+ *
+ *  It also takes the request, not just the step, because the route depends on what
+ *  was asked for: `installPackage()` takes package names and cannot express
+ *  `--install-all-packages`, so that one goes through `clean()` as well. */
+function postBootRoute(ask: ProvisioningAsk, request: ProvisioningRequest, name: string): string {
+	// Every interpolation is quoted: `assertValidResourceName()` only guards names at
+	// creation, so a real machine can still be called `lab old`, and the advertised
+	// `quickchr set lab old …` addresses a machine called `lab`.
+	const quoted = shellQuote(name);
+	const replay = `quickchr clean ${quoted}   # resets the disk, then repeat the original start command`;
+	switch (ask.step) {
+		case "license":
+			return `quickchr set ${quoted} --license`;
+		case "deviceMode":
+			// Spelled out with the requested flags, because the route only helps if it
+			// can be run as printed — and `set` takes the same flag names `start` does.
+			return `quickchr set ${quoted} ${formatDeviceModeFlags(resolveDeviceModeOptions(request.deviceMode))}   # power-cycles the machine`;
+		case "packages":
+			// installPackage() has no install-all form — availablePackages() would have
+			// to be enumerated first — so the honest route for that request is a reset.
+			if (request.installAllPackages) return replay;
+			return `instance.installPackage(${JSON.stringify(request.packages ?? [])})   // library API; no CLI verb yet — tikoci/quickchr#24`;
+		default:
+			return replay;
+	}
+}
+
+/**
+ * Refuse a provisioning option that arrived after the window closed.
+ *
+ * Returns the asks that are already satisfied, so the caller can say so rather than
+ * staying silent about them; throws `PROVISIONING_WINDOW_CLOSED` naming every ask
+ * that is not.
+ */
+export function assertProvisioningWindow(
+	state: MachineState,
+	request: ProvisioningRequest,
+): ProvisioningAsk[] {
+	if (isProvisioningWindowOpen(state)) return [];
+
+	const asks = classifyProvisioningRequest(state, request);
+	const pending = asks.filter((ask) => !ask.satisfied);
+	if (pending.length === 0) return asks;
+
+	// Commands go on their own line, indented, so they can be copied without dragging
+	// the surrounding prose along with them.
+	const lines = pending.flatMap((ask) => [
+		`  ${ask.description} — apply it with:`,
+		`      ${postBootRoute(ask, request, state.name)}`,
+	]);
+	const satisfied = asks.filter((ask) => ask.satisfied);
+	if (satisfied.length > 0) {
+		lines.push(`  (already applied, unchanged: ${satisfied.map((a) => a.description).join("; ")})`);
+	}
+
+	throw new QuickCHRError(
+		"PROVISIONING_WINDOW_CLOSED",
+		`Machine "${state.name}" has already booted, so provisioning cannot run against it — ` +
+		"the guest may no longer hold the default configuration provisioning expects.\n" +
+		lines.join("\n") +
+		"\nTo provision from scratch, reset the disk to the factory image and reopen the window:\n" +
+		`      quickchr clean ${shellQuote(state.name)}   # then repeat the original start command — a refused request is not remembered`,
+	);
+}

package/src/lib/qemu.ts

@@ -10,9 +10,11 @@ import { join } from "node:path";
 import type { Arch, BootDiskFormat, MachineState, NetworkMode, NetworkConfig, PortMapping } from "./types.ts";
 import { QuickCHRError } from "./types.ts";
 import { detectAccel, requireQemu, requireFirmware, requireQemuImg } from "./platform.ts";
-import { buildHostfwdString } from "./network.ts";
+import { buildHostfwdString, deviceArgs } from "./network.ts";
 import { ensureDir } from "./state.ts";
 import { restGet } from "./rest.ts";
+import { apiLogin } from "./api-login.ts";
+import { isRouterOs6 } from "./versions.ts";
 import { classifyProbeError, recordBootProbe, serialLogEnabled, type BootProbeStats } from "./diagnostics.ts";
 
 export interface QemuLaunchConfig {
@@ -137,8 +139,15 @@ export async function buildQemuArgs(config: QemuLaunchConfig): Promise<string[]>
 	return args;
 }
 
-/** Build networking arguments from NetworkConfig array. */
-function buildNetworkArgs(
+/** Build networking arguments from a `NetworkConfig` array.
+ *
+ *  Exported because it is one of the two places a NIC reaches QEMU (the other is
+ *  `resolveAllNetworks()` in `network.ts`, whose output the `net.resolved` branch
+ *  below passes straight through). Testing it through {@link buildQemuArgs} would
+ *  require a QEMU binary on the host, which the unit tier does not have — and a
+ *  test that skips itself when QEMU is missing would assert nothing on CI, which
+ *  is how a MAC-less NIC got shipped in the first place (#154). */
+export function buildNetworkArgs(
 	args: string[],
 	ports: Record<string, PortMapping>,
 	networks: NetworkConfig[],
@@ -157,39 +166,39 @@ function buildNetworkArgs(
 		if (spec === "vmnet-shared") {
 			args.push(
 				"-netdev", `vmnet-shared,id=${id}`,
-				"-device", `virtio-net-pci,netdev=${id}`,
+				...deviceArgs(net),
 			);
 		} else if (typeof spec === "object" && spec.type === "vmnet-bridged") {
 			args.push(
 				"-netdev", `vmnet-bridged,id=${id},ifname=${spec.iface}`,
-				"-device", `virtio-net-pci,netdev=${id}`,
+				...deviceArgs(net),
 			);
 		} else if (typeof spec === "object" && spec.type === "socket-listen") {
 			args.push(
 				"-netdev", `socket,id=${id},listen=:${spec.port}`,
-				"-device", `virtio-net-pci,netdev=${id}`,
+				...deviceArgs(net),
 			);
 		} else if (typeof spec === "object" && spec.type === "socket-connect") {
 			args.push(
 				"-netdev", `socket,id=${id},connect=127.0.0.1:${spec.port}`,
-				"-device", `virtio-net-pci,netdev=${id}`,
+				...deviceArgs(net),
 			);
 		} else if (typeof spec === "object" && spec.type === "socket-mcast") {
 			args.push(
 				"-netdev", `socket,id=${id},mcast=${spec.group}:${spec.port}`,
-				"-device", `virtio-net-pci,netdev=${id}`,
+				...deviceArgs(net),
 			);
 		} else if (typeof spec === "object" && spec.type === "tap") {
 			args.push(
 				"-netdev", `tap,id=${id},ifname=${spec.ifname},script=no,downscript=no`,
-				"-device", `virtio-net-pci,netdev=${id}`,
+				...deviceArgs(net),
 			);
 		} else {
 			// Default: user-mode networking with port forwarding (only on the first user NIC)
 			const hostfwd = buildHostfwdString(ports);
 			args.push(
 				"-netdev", `user,id=${id}${hostfwd ? "," + hostfwd : ""}`,
-				"-device", `virtio-net-pci,netdev=${id}`,
+				...deviceArgs(net),
 			);
 		}
 	}
@@ -379,6 +388,20 @@ export async function spawnQemu(
 		// After spawn, close parent's copy; child process inherits its own copy.
 		const logFd = openSync(logPath, "a");
 
+		// A backgrounded QEMU outlives the CLI invocation that started it, so it must
+		// leave the caller's process group. `detached: true` is `setsid()` on POSIX;
+		// `unref()` alone is not enough, because it only covers a *voluntary* parent
+		// exit. It leaves QEMU in the caller's process group, where a group signal
+		// reaches it: Ctrl-C in the terminal, a shell `timeout`, a CI step teardown,
+		// an agent harness killing a stuck command (#159).
+		//
+		// Killing a run's VMs deliberately is by QEMU process name, not by group
+		// (`scripts/ci-file-watchdog.ts`, and the integration workflow's cleanup
+		// steps), so detaching costs no cleanup reach.
+		//
+		// Windows keeps its own spawn below for a different failure — a Job Object
+		// takes everything in it when the parent goes — and the two are not merged
+		// because only this one can be verified here.
 		let spawnedPid: number;
 
 		if (process.platform === "win32") {
@@ -394,6 +417,11 @@ export async function spawnQemu(
 				stdio: ["ignore", logFd, logFd],
 				windowsHide: true,
 			});
+			// A failed exec surfaces asynchronously here, as an `error` event with no
+			// pid — and an unhandled `error` event is a crash, not a thrown
+			// QuickCHRError. Nothing awaits the child, so this listener is what keeps
+			// a qemu binary that is missing, or cannot be executed, reportable.
+			child.on("error", () => { /* surfaced by the liveness check below */ });
 			closeSync(logFd);
 			if (child.pid === undefined) {
 				throw new QuickCHRError("SPAWN_FAILED", "Failed to spawn QEMU process on Windows");
@@ -402,13 +430,14 @@ export async function spawnQemu(
 			spawnedPid = child.pid;
 		} else {
 			const proc = Bun.spawn(spawnCmd, {
+				detached: true,
 				stdout: logFd,
 				stderr: logFd,
 				stdin: "ignore",
 			});
 			closeSync(logFd);
-			// unref() lets the parent Bun process exit without waiting for QEMU.
-			// QEMU becomes an orphan adopted by init/launchd and keeps running.
+			// unref() lets the parent exit without waiting for QEMU; detached keeps
+			// QEMU alive when the parent is signalled rather than exiting.
 			proc.unref();
 			spawnedPid = proc.pid;
 		}
@@ -455,10 +484,10 @@ export async function stopQemu(pid: number): Promise<boolean> {
 		return false; // Already dead
 	}
 
-	// Try graceful shutdown
+	// Terminate the host process (a guest power cut, not /system/shutdown)
 	process.kill(pid, "SIGTERM");
 
-	// Wait up to 5s for graceful exit
+	// Wait up to 5s for QEMU to exit before escalating
 	for (let i = 0; i < 50; i++) {
 		await Bun.sleep(100);
 		try {
@@ -573,3 +602,53 @@ export async function waitForBoot(
 
 	return false;
 }
+
+/** Which readiness signal a guest can give. REST everywhere it exists; the native-API
+ *  login on RouterOS 6, which has no REST and would otherwise run the REST probe to its
+ *  timeout against a guest that booted in seconds (#182). The one place the choice is
+ *  made — `waitForApiReady()` itself is not version-specific. */
+export function readinessProbeFor(version: string): "rest" | "api" {
+	return isRouterOs6(version) ? "api" : "rest";
+}
+
+/**
+ * Wait for the native API (guest 8728) to answer a `/login` — the readiness probe for a
+ * guest without REST. Same contract as {@link waitForBoot}: two consecutive answers,
+ * a credential refusal (`!trap`) counts as an answer, and every probe is classified into
+ * `stats` so a timeout explains itself.
+ */
+export async function waitForApiReady(
+	apiPort: number,
+	timeoutMs: number,
+	creds: { user: string; password: string },
+	stats?: BootProbeStats,
+): Promise<boolean> {
+	if (stats) stats.kind = "api";
+	const start = Date.now();
+	const deadline = start + timeoutMs;
+	let consecutiveReady = 0;
+
+	while (Date.now() < deadline) {
+		try {
+			const result = await apiLogin(apiPort, creds.user, creds.password, 3000);
+			if (result.outcome === "done") {
+				consecutiveReady++;
+				recordBootProbe(stats, Date.now() - start, "ready");
+			} else if (result.outcome === "trap") {
+				consecutiveReady++;
+				recordBootProbe(stats, Date.now() - start, "unauthorized", result.message);
+			} else {
+				consecutiveReady = 0;
+				recordBootProbe(stats, Date.now() - start, "other", `!fatal ${result.message}`);
+			}
+			if (consecutiveReady >= 2) return true;
+		} catch (e) {
+			consecutiveReady = 0;
+			const { outcome, detail } = classifyProbeError(e);
+			recordBootProbe(stats, Date.now() - start, outcome, detail);
+		}
+		await Bun.sleep(2000);
+	}
+
+	return false;
+}