@tikoci/quickchr 0.4.6 → 0.4.8

package/src/lib/provision.ts

@@ -67,12 +67,82 @@ async function waitForRest(
 	}
 }
 
+/** Wait until RouterOS actually accepts `auth`, returning how long that took.
+ *
+ * Deliberately *not* `waitForBoot()`: that one counts 401/403 as ready, because
+ * for a liveness probe an auth rejection still proves `www` is answering. Here
+ * a 401 is the thing being waited out, so it needs the opposite predicate.
+ *
+ * **This is hardening and an instrument, not a proven fix for #69.** The known
+ * symptom is a 401 on the first request made with freshly created credentials —
+ * four Windows CI legs across runs 35135008624 and 35386692051, always on the
+ * path that does no work between `createUser()` and that request. A local
+ * attempt to reproduce it found no window to close: on an Intel host against
+ * CHR 7.24.4 (x86 guest), a new user authenticated on attempt #1, 6/6, under
+ * both TCG and HVF. So the cause of the CI 401 is still open, and this wait
+ * does not claim to explain it.
+ *
+ * What it does buy: `createUser()` can no longer resolve while a `full`-group
+ * user's own credentials are rejected, and the outcome is measured either way.
+ * (Limited groups are exempt — see the table at the call site: a valid user
+ * without `rest-api` answers 401, which is indistinguishable from the symptom.) If the CI
+ * 401 is any kind of timing window, this closes it *and* the returned figure
+ * says how wide it was on that platform — which is the quantification #69 asks
+ * for. If it is not a timing window, the wait exhausts its budget and fails
+ * saying so, which is a far stronger signal than today's bare 401.
+ *
+ * Reports `attempts` alongside `elapsedMs`, and **`attempts` is the load-bearing
+ * number**. Elapsed time cannot tell a retry from a slow request: a first
+ * authentication with a new password costs a few hundred ms of ordinary request
+ * latency on an emulated guest, so an elapsed-only figure reads as a wait when
+ * nothing waited. `attempts === 1` means there was no window, whatever the
+ * clock says. (This is not hypothetical — it is the error the first pass at
+ * this investigation made; see `provisioning.instructions.md`.) */
+export async function waitForAuth(
+	httpPort: number,
+	auth: string,
+	timeoutMs: number = 30_000,
+): Promise<{ attempts: number; elapsedMs: number }> {
+	const url = `http://127.0.0.1:${httpPort}/rest/system/resource`;
+	const start = Date.now();
+	const deadline = start + timeoutMs;
+	let lastStatus: number | undefined;
+	let attempts = 0;
+
+	while (Date.now() < deadline) {
+		attempts++;
+		// Bound each request by what is left of the budget, so a stalled request
+		// cannot push the whole wait past the deadline it advertises.
+		const remaining = deadline - Date.now();
+		try {
+			const { status } = await restGet(url, auth, Math.min(5_000, remaining));
+			if (status >= 200 && status < 300) return { attempts, elapsedMs: Date.now() - start };
+			// 401/403 is the propagation window. 5xx also occurs briefly after a
+			// user-database change, and is equally transient, so both are polled
+			// out rather than one being singled out as "the" expected status.
+			lastStatus = status;
+		} catch { /* transport error mid-transition — keep polling */ }
+		await Bun.sleep(250);
+	}
+
+	throw new QuickCHRError(
+		"PROCESS_FAILED",
+		`Credentials were not accepted within ${timeoutMs}ms over ${attempts} attempt(s) ` +
+		`(last HTTP status: ${lastStatus ?? "no response"}) — ` +
+		"the user record exists but RouterOS is not authenticating it",
+	);
+}
+
 /** Create a user via the REST API. */
 export async function createUser(
 	httpPort: number,
 	name: string,
 	password: string,
 	group: string = "full",
+	logger?: ProgressLogger,
+	/** Budget for the post-visibility authentication wait. Injectable so tests
+	 *  can exercise the failure path without burning the 30 s default. */
+	authTimeoutMs: number = 30_000,
 ): Promise<void> {
 	await waitForRest(httpPort);
 
@@ -95,6 +165,7 @@ export async function createUser(
 	// Verify visibility (and group) so callers get deterministic behavior.
 	const expectedGroup = group.trim().toLowerCase();
 	const deadline = Date.now() + 30_000;
+	let visible = false;
 	while (Date.now() < deadline) {
 		try {
 			const user = await readUser(httpPort, auth, name);
@@ -110,7 +181,8 @@ export async function createUser(
 						`User "${name}" created with unexpected group (expected=${expectedGroup}, actual=${actualGroup})`,
 					);
 				}
-				return;
+				visible = true;
+				break;
 			}
 		} catch (e) {
 			// Rethrow deliberate group mismatch errors; retry transient HTTP failures
@@ -120,9 +192,58 @@ export async function createUser(
 		await Bun.sleep(500);
 	}
 
-	throw new QuickCHRError(
-		"PROCESS_FAILED",
-		`User "${name}" creation was acknowledged but did not become visible in RouterOS`,
+	if (!visible) {
+		throw new QuickCHRError(
+			"PROCESS_FAILED",
+			`User "${name}" creation was acknowledged but did not become visible in RouterOS`,
+		);
+	}
+
+	// Only the `full` group is gated, because only there does "the probe answers
+	// 2xx" mean "the credentials work". RouterOS separates authentication from
+	// authorization, and a correctly created user in a limited group cannot
+	// answer this probe at all. Measured on CHR 7.24.4 by creating one user per
+	// group and issuing GET /rest/system/resource as that user:
+	//
+	//   group policy                              | status
+	//   ------------------------------------------|---------------------------
+	//   local,ssh,winbox,read      (no rest-api)   | 401 Unauthorized
+	//   local,ssh,winbox,rest-api,write (no read)  | 500 std failure: not allowed (9)
+	//   local,winbox,rest-api,read (no web)        | 500 std failure: not allowed (9)
+	//   read / write / full (defaults)             | 200
+	//
+	// The 401 is byte-identical to #69's symptom, so no amount of polling can
+	// tell an unauthorized user from a credential that has not propagated, and the 500 is
+	// a permanent condition this wait would poll through to its deadline. Gating
+	// those would turn a correctly created user into a 30 s stall and a false
+	// "not authenticating" failure. `provision()` only ever creates `full`.
+	if (expectedGroup !== "full") {
+		logger?.debug(
+			`user "${name}" created in group "${group}" — authentication gate skipped: ` +
+			"only the full group is known to be REST-verifiable",
+		);
+		return;
+	}
+
+	// Deliberately outside the loop above. waitForAuth() throws a QuickCHRError,
+	// and that catch swallows every QuickCHRError except the group mismatch — so
+	// running it inside would discard the one diagnostic this wait exists to
+	// produce and report "did not become visible" for a user that plainly is.
+	//
+	// "Visible in /rest/user" was the whole done-when here, which is a weaker
+	// promise than every caller relies on: they use the credentials next.
+	const { attempts, elapsedMs } = await waitForAuth(
+		httpPort,
+		`Basic ${btoa(`${name}:${password}`)}`,
+		authTimeoutMs,
+	);
+	// attempts is the signal; elapsedMs alone cannot separate a retry from a slow
+	// first authentication on an emulated guest. The earlier attempts are only
+	// known to have not-succeeded — waitForAuth also polls through 5xx and
+	// transport errors — so they are not reported as rejections.
+	logger?.debug(
+		`user "${name}" accepted after ${attempts} attempt(s), ${elapsedMs}ms` +
+		(attempts > 1 ? ` — ${attempts - 1} earlier probe(s) did not succeed (#69)` : ""),
 	);
 }
 
@@ -609,7 +730,7 @@ export async function provision(
 	// REST provisioning path. The console fallback above always returns early,
 	// so reaching here means REST came up and we provision over it.
 	if (effectiveUser) {
-		await createUser(httpPort, effectiveUser.name, effectiveUser.password);
+		await createUser(httpPort, effectiveUser.name, effectiveUser.password, "full", log);
 		// Persist to secret store so resolveAuth() picks it up
 		saveInstanceCredentials(machineName, effectiveUser.name, effectiveUser.password);
 	}

package/src/lib/qemu.ts

@@ -10,9 +10,10 @@ import { join } from "node:path";
 import type { Arch, BootDiskFormat, MachineState, NetworkMode, NetworkConfig, PortMapping } from "./types.ts";
 import { QuickCHRError } from "./types.ts";
 import { detectAccel, requireQemu, requireFirmware, requireQemuImg } from "./platform.ts";
-import { buildHostfwdString } from "./network.ts";
+import { buildHostfwdString, deviceArgs } from "./network.ts";
 import { ensureDir } from "./state.ts";
 import { restGet } from "./rest.ts";
+import { classifyProbeError, recordBootProbe, serialLogEnabled, type BootProbeStats } from "./diagnostics.ts";
 
 export interface QemuLaunchConfig {
 	arch: Arch;
@@ -136,8 +137,15 @@ export async function buildQemuArgs(config: QemuLaunchConfig): Promise<string[]>
 	return args;
 }
 
-/** Build networking arguments from NetworkConfig array. */
-function buildNetworkArgs(
+/** Build networking arguments from a `NetworkConfig` array.
+ *
+ *  Exported because it is one of the two places a NIC reaches QEMU (the other is
+ *  `resolveAllNetworks()` in `network.ts`, whose output the `net.resolved` branch
+ *  below passes straight through). Testing it through {@link buildQemuArgs} would
+ *  require a QEMU binary on the host, which the unit tier does not have — and a
+ *  test that skips itself when QEMU is missing would assert nothing on CI, which
+ *  is how a MAC-less NIC got shipped in the first place (#154). */
+export function buildNetworkArgs(
 	args: string[],
 	ports: Record<string, PortMapping>,
 	networks: NetworkConfig[],
@@ -156,39 +164,39 @@ function buildNetworkArgs(
 		if (spec === "vmnet-shared") {
 			args.push(
 				"-netdev", `vmnet-shared,id=${id}`,
-				"-device", `virtio-net-pci,netdev=${id}`,
+				...deviceArgs(net),
 			);
 		} else if (typeof spec === "object" && spec.type === "vmnet-bridged") {
 			args.push(
 				"-netdev", `vmnet-bridged,id=${id},ifname=${spec.iface}`,
-				"-device", `virtio-net-pci,netdev=${id}`,
+				...deviceArgs(net),
 			);
 		} else if (typeof spec === "object" && spec.type === "socket-listen") {
 			args.push(
 				"-netdev", `socket,id=${id},listen=:${spec.port}`,
-				"-device", `virtio-net-pci,netdev=${id}`,
+				...deviceArgs(net),
 			);
 		} else if (typeof spec === "object" && spec.type === "socket-connect") {
 			args.push(
 				"-netdev", `socket,id=${id},connect=127.0.0.1:${spec.port}`,
-				"-device", `virtio-net-pci,netdev=${id}`,
+				...deviceArgs(net),
 			);
 		} else if (typeof spec === "object" && spec.type === "socket-mcast") {
 			args.push(
 				"-netdev", `socket,id=${id},mcast=${spec.group}:${spec.port}`,
-				"-device", `virtio-net-pci,netdev=${id}`,
+				...deviceArgs(net),
 			);
 		} else if (typeof spec === "object" && spec.type === "tap") {
 			args.push(
 				"-netdev", `tap,id=${id},ifname=${spec.ifname},script=no,downscript=no`,
-				"-device", `virtio-net-pci,netdev=${id}`,
+				...deviceArgs(net),
 			);
 		} else {
 			// Default: user-mode networking with port forwarding (only on the first user NIC)
 			const hostfwd = buildHostfwdString(ports);
 			args.push(
 				"-netdev", `user,id=${id}${hostfwd ? "," + hostfwd : ""}`,
-				"-device", `virtio-net-pci,netdev=${id}`,
+				...deviceArgs(net),
 			);
 		}
 	}
@@ -221,12 +229,20 @@ function buildChannelArgs(
 		qgaSock = `path=${join(machineDir, "qga.sock")}`;
 	}
 
+	// QUICKCHR_SERIAL_LOG=1 tees the serial console to <machineDir>/serial.log via
+	// the QEMU chardev `logfile=` option. Opt-in, never default: serial-console
+	// provisioning types the generated user password in cleartext (provision.ts
+	// provisionViaConsole), so this log is secret-bearing. It is the only record of
+	// what RouterOS printed when a boot never reaches REST — the socket chardev
+	// keeps no history, so connecting after the failure shows nothing (#79).
+	const serialLog = serialLogEnabled() ? `,logfile=${join(machineDir, "serial.log")},logappend=on` : "";
+
 	if (background) {
 		// Background: all channels on sockets/pipes
 		args.push(
 			"-chardev", `socket,id=monitor0,${monitorSock},server=on,wait=off`,
 			"-mon", "chardev=monitor0,mode=readline",
-			"-chardev", `socket,id=serial0,${serialSock},server=on,wait=off`,
+			"-chardev", `socket,id=serial0,${serialSock},server=on,wait=off${serialLog}`,
 			"-serial", "chardev:serial0",
 		);
 	} else {
@@ -370,6 +386,20 @@ export async function spawnQemu(
 		// After spawn, close parent's copy; child process inherits its own copy.
 		const logFd = openSync(logPath, "a");
 
+		// A backgrounded QEMU outlives the CLI invocation that started it, so it must
+		// leave the caller's process group. `detached: true` is `setsid()` on POSIX;
+		// `unref()` alone is not enough, because it only covers a *voluntary* parent
+		// exit. It leaves QEMU in the caller's process group, where a group signal
+		// reaches it: Ctrl-C in the terminal, a shell `timeout`, a CI step teardown,
+		// an agent harness killing a stuck command (#159).
+		//
+		// Killing a run's VMs deliberately is by QEMU process name, not by group
+		// (`scripts/ci-file-watchdog.ts`, and the integration workflow's cleanup
+		// steps), so detaching costs no cleanup reach.
+		//
+		// Windows keeps its own spawn below for a different failure — a Job Object
+		// takes everything in it when the parent goes — and the two are not merged
+		// because only this one can be verified here.
 		let spawnedPid: number;
 
 		if (process.platform === "win32") {
@@ -385,6 +415,11 @@ export async function spawnQemu(
 				stdio: ["ignore", logFd, logFd],
 				windowsHide: true,
 			});
+			// A failed exec surfaces asynchronously here, as an `error` event with no
+			// pid — and an unhandled `error` event is a crash, not a thrown
+			// QuickCHRError. Nothing awaits the child, so this listener is what keeps
+			// a qemu binary that is missing, or cannot be executed, reportable.
+			child.on("error", () => { /* surfaced by the liveness check below */ });
 			closeSync(logFd);
 			if (child.pid === undefined) {
 				throw new QuickCHRError("SPAWN_FAILED", "Failed to spawn QEMU process on Windows");
@@ -393,13 +428,14 @@ export async function spawnQemu(
 			spawnedPid = child.pid;
 		} else {
 			const proc = Bun.spawn(spawnCmd, {
+				detached: true,
 				stdout: logFd,
 				stderr: logFd,
 				stdin: "ignore",
 			});
 			closeSync(logFd);
-			// unref() lets the parent Bun process exit without waiting for QEMU.
-			// QEMU becomes an orphan adopted by init/launchd and keeps running.
+			// unref() lets the parent exit without waiting for QEMU; detached keeps
+			// QEMU alive when the parent is signalled rather than exiting.
 			proc.unref();
 			spawnedPid = proc.pid;
 		}
@@ -517,9 +553,11 @@ export async function waitForBoot(
 	httpPort: number,
 	timeoutMs: number = 120_000,
 	auth: string = `Basic ${btoa("admin:")}`,
+	stats?: BootProbeStats,
 ): Promise<boolean> {
 	const url = `http://127.0.0.1:${httpPort}/rest/system/resource`;
-	const deadline = Date.now() + timeoutMs;
+	const start = Date.now();
+	const deadline = start + timeoutMs;
 	let consecutiveReady = 0;
 
 	while (Date.now() < deadline) {
@@ -530,6 +568,7 @@ export async function waitForBoot(
 				// Auth rejected — the REST layer is up and responded, even though
 				// we cannot validate the body. Count as ready.
 				consecutiveReady++;
+				recordBootProbe(stats, Date.now() - start, "unauthorized", String(status));
 			} else if (status >= 200 && status < 300) {
 				// RouterOS REST may return wrong data (array body) briefly after boot —
 				// a startup race. Only accept once the body is the expected singleton object.
@@ -537,17 +576,24 @@ export async function waitForBoot(
 				try { parsed = JSON.parse(body); } catch { /* ignore */ }
 				if (parsed && typeof parsed === "object" && !Array.isArray(parsed) && "board-name" in (parsed as object)) {
 					consecutiveReady++;
+					recordBootProbe(stats, Date.now() - start, "ready");
 				} else {
 					consecutiveReady = 0; // wrong body — REST not fully initialized yet
+					recordBootProbe(stats, Date.now() - start, "wrong-body", body.slice(0, 80));
 				}
 			} else {
 				consecutiveReady = 0;
+				recordBootProbe(stats, Date.now() - start, "http-error", String(status));
 			}
 
 			if (consecutiveReady >= 2) return true;
-		} catch {
-			// ECONNRESET, connection refused, timeout — not ready yet
+		} catch (e) {
+			// ECONNRESET, connection refused, timeout — not ready yet. Which one it
+			// was is the whole diagnosis when a boot never completes (#79), so it is
+			// classified rather than swallowed.
 			consecutiveReady = 0;
+			const { outcome, detail } = classifyProbeError(e);
+			recordBootProbe(stats, Date.now() - start, outcome, detail);
 		}
 		await Bun.sleep(2000);
 	}