@tikoci/quickchr 0.4.6 → 0.4.7

package/src/lib/qemu.ts

@@ -13,6 +13,7 @@ import { detectAccel, requireQemu, requireFirmware, requireQemuImg } from "./pla
 import { buildHostfwdString } from "./network.ts";
 import { ensureDir } from "./state.ts";
 import { restGet } from "./rest.ts";
+import { classifyProbeError, recordBootProbe, serialLogEnabled, type BootProbeStats } from "./diagnostics.ts";
 
 export interface QemuLaunchConfig {
 	arch: Arch;
@@ -221,12 +222,20 @@ function buildChannelArgs(
 		qgaSock = `path=${join(machineDir, "qga.sock")}`;
 	}
 
+	// QUICKCHR_SERIAL_LOG=1 tees the serial console to <machineDir>/serial.log via
+	// the QEMU chardev `logfile=` option. Opt-in, never default: serial-console
+	// provisioning types the generated user password in cleartext (provision.ts
+	// provisionViaConsole), so this log is secret-bearing. It is the only record of
+	// what RouterOS printed when a boot never reaches REST — the socket chardev
+	// keeps no history, so connecting after the failure shows nothing (#79).
+	const serialLog = serialLogEnabled() ? `,logfile=${join(machineDir, "serial.log")},logappend=on` : "";
+
 	if (background) {
 		// Background: all channels on sockets/pipes
 		args.push(
 			"-chardev", `socket,id=monitor0,${monitorSock},server=on,wait=off`,
 			"-mon", "chardev=monitor0,mode=readline",
-			"-chardev", `socket,id=serial0,${serialSock},server=on,wait=off`,
+			"-chardev", `socket,id=serial0,${serialSock},server=on,wait=off${serialLog}`,
 			"-serial", "chardev:serial0",
 		);
 	} else {
@@ -517,9 +526,11 @@ export async function waitForBoot(
 	httpPort: number,
 	timeoutMs: number = 120_000,
 	auth: string = `Basic ${btoa("admin:")}`,
+	stats?: BootProbeStats,
 ): Promise<boolean> {
 	const url = `http://127.0.0.1:${httpPort}/rest/system/resource`;
-	const deadline = Date.now() + timeoutMs;
+	const start = Date.now();
+	const deadline = start + timeoutMs;
 	let consecutiveReady = 0;
 
 	while (Date.now() < deadline) {
@@ -530,6 +541,7 @@ export async function waitForBoot(
 				// Auth rejected — the REST layer is up and responded, even though
 				// we cannot validate the body. Count as ready.
 				consecutiveReady++;
+				recordBootProbe(stats, Date.now() - start, "unauthorized", String(status));
 			} else if (status >= 200 && status < 300) {
 				// RouterOS REST may return wrong data (array body) briefly after boot —
 				// a startup race. Only accept once the body is the expected singleton object.
@@ -537,17 +549,24 @@ export async function waitForBoot(
 				try { parsed = JSON.parse(body); } catch { /* ignore */ }
 				if (parsed && typeof parsed === "object" && !Array.isArray(parsed) && "board-name" in (parsed as object)) {
 					consecutiveReady++;
+					recordBootProbe(stats, Date.now() - start, "ready");
 				} else {
 					consecutiveReady = 0; // wrong body — REST not fully initialized yet
+					recordBootProbe(stats, Date.now() - start, "wrong-body", body.slice(0, 80));
 				}
 			} else {
 				consecutiveReady = 0;
+				recordBootProbe(stats, Date.now() - start, "http-error", String(status));
 			}
 
 			if (consecutiveReady >= 2) return true;
-		} catch {
-			// ECONNRESET, connection refused, timeout — not ready yet
+		} catch (e) {
+			// ECONNRESET, connection refused, timeout — not ready yet. Which one it
+			// was is the whole diagnosis when a boot never completes (#79), so it is
+			// classified rather than swallowed.
 			consecutiveReady = 0;
+			const { outcome, detail } = classifyProbeError(e);
+			recordBootProbe(stats, Date.now() - start, outcome, detail);
 		}
 		await Bun.sleep(2000);
 	}

package/src/lib/quickchr.ts

@@ -56,7 +56,8 @@ import { autoPruneIfOverCap } from "./cache.ts";
 import { resolveSetting } from "./settings.ts";
 import { buildQemuArgs, spawnQemu, stopQemu, cleanupQemuSockets, waitForBoot, extractWrapper, type QemuLaunchConfig } from "./qemu.ts";
 import { cleanDiskFiles, ensureConfiguredDisks, normalizeDiskOptions, parseSnapshotList, listSnapshots, formatDiskSize } from "./disk.ts";
-import { monitorCommand, serialStreams, qgaCommand, channelEndpoint } from "./channels.ts";
+import { monitorCommand, serialStreams, qgaCommand, channelEndpoint, channelFileExists, channelPath } from "./channels.ts";
+import { captureBootFailure, newBootProbeStats, type BootFailureReport, type BootProbeStats, type FailureTrigger } from "./diagnostics.ts";
 import { installPackages, installAllPackages, downloadAndListPackages, downloadPackages, findPackageFile, uploadPackages } from "./packages.ts";
 import { provision } from "./provision.ts";
 import { renewLicense, getLicenseInfo } from "./license.ts";
@@ -65,7 +66,7 @@ import { scpPush, scpPull } from "./scp.ts";
 import { deleteInstanceCredentials, credentialStorageLabel, getStoredCredentials, getInstanceCredentials, STORED_IN_SECRETS_PASSWORD } from "./credentials.ts";
 import { restExecute } from "./exec.ts";
 import { qgaExec } from "./qga.ts";
-import { consoleExec } from "./console.ts";
+import { consoleExec, CONSOLE_LOGIN_COST_MS } from "./console.ts";
 import { restRequest, restGet, restPost } from "./rest.ts";
 import { createNamedSocket, getNamedSocket, addSocketMember, removeSocketMember } from "./socket-registry.ts";
 import { createLogger, type ProgressLogger } from "./log.ts";
@@ -79,6 +80,7 @@ import {
 	waitForDeviceModeApi,
 } from "./device-mode.ts";
 import type { LicenseOptions } from "./types.ts";
+import type { GuestExec } from "./guest-snapshot.ts";
 import { toChrPorts } from "./network.ts";
 import { assertSufficientQuickchrStorage, formatQuickchrUsage, getQuickchrStorageReport } from "./storage.ts";
 import { existsSync, readdirSync, rmSync, copyFileSync, writeFileSync, unlinkSync, openSync, writeSync, closeSync, readFileSync } from "node:fs";
@@ -412,11 +414,11 @@ function createInstance(state: MachineState): ChrInstance {
 		captureInterface: process.platform === "darwin" ? "lo0" : "any",
 		tzspGatewayIp: "10.0.2.2",
 
-		async waitForBoot(timeoutMs?: number): Promise<boolean> {
+		async waitForBoot(timeoutMs?: number, stats?: BootProbeStats): Promise<boolean> {
 			// Use resolved credentials so waitForBoot can validate the response body
 			// on authenticated machines (post-provisioning, post-install reboots).
 			const auth = resolveAuth(state);
-			return waitForBoot(ports.http, timeoutMs, auth.header);
+			return waitForBoot(ports.http, timeoutMs, auth.header, stats);
 		},
 
 		async waitFor(condition: () => Promise<boolean>, timeoutMs = 30_000): Promise<boolean> {
@@ -487,18 +489,43 @@ function createInstance(state: MachineState): ChrInstance {
 				state.bootDiskFormat ?? (state.bootSize ? "qcow2" : "raw"),
 			);
 
-			// Clean up stored instance credentials (re-provisioned on next start)
+			// Every guest-side credential went with the old disk, so the credential
+			// facts in machine.json now describe a machine that no longer exists.
+			// Nothing re-provisions them either: a post-clean() start() takes the
+			// `_launchExisting(…, undefined)` path (lastStartedAt is set), so the
+			// erased account is never recreated. Left in place, `resolveAuth()` /
+			// `resolveCreds()` prefer `state.user` and authenticate every REST call,
+			// exec, and SCP as a deleted user — the confound #79 removes here.
+			// Clearing them restores the documented fallback: factory `admin` with an
+			// empty password, which is what a fresh CHR image actually answers to
+			// (anchored by the "clean() resets disk to factory defaults" integration
+			// test). `disableAdmin` goes with them because it is read as a live fact
+			// about the guest (buildDescriptor's `disableAdminLockout`) and a fresh
+			// image has admin enabled again. Provisioning *intent* — packages,
+			// deviceMode, secureLogin — is not guest state and survives.
 			deleteInstanceCredentials(state.name);
+			// The managed keypair authenticated to the account factory reset erased,
+			// so it is dead credential material sitting in the machine dir. Removing
+			// it also keeps a future re-provision viable: `ssh-keygen -f <path>` on an
+			// existing file exits 1 with "already exists" rather than overwriting
+			// (verified locally), so a leftover key would break `installSshKey()`.
+			rmSync(join(state.machineDir, "ssh"), { recursive: true, force: true });
 
 			// Update state
 			const current = loadMachine(state.name);
 			if (current) {
 				current.status = "stopped";
 				current.pid = undefined;
+				current.user = undefined;
+				current.managedSshKey = undefined;
+				current.disableAdmin = undefined;
 				saveMachine(current);
 			}
 			state.status = "stopped";
 			state.pid = undefined;
+			state.user = undefined;
+			state.managedSshKey = undefined;
+			state.disableAdmin = undefined;
 		},
 
 		async monitor(command: string): Promise<string> {
@@ -950,6 +977,183 @@ async function waitForPidExit(pid: number, timeoutMs: number): Promise<boolean>
 	return false;
 }
 
+/** Where boot-failure reports land: `<dataDir>/failures/`, deliberately outside
+ *  any machine directory so `remove()` — which every integration test calls from
+ *  its `finally` — cannot delete the evidence it was written to explain (#79). */
+function bootFailureReportDir(): string {
+	return join(getDataDir(), "failures");
+}
+
+/**
+ * How long one credential candidate gets inside a single guest-exec call.
+ *
+ * `timeoutMs` is a budget for the whole call, not per attempt: `captureGuestSnapshot()`
+ * sizes its own budget assuming one exec costs at most what it passed in, so N
+ * candidates × the full timeout would spend the snapshot's entire budget on the
+ * first query. Untried candidates therefore share it, and once a credential is
+ * known to work it gets all of it — which is every query after the first.
+ *
+ * The floor is {@link CONSOLE_LOGIN_COST_MS} rather than a round number, because
+ * a share below the cost of a login cannot succeed **by construction** — and a
+ * candidate that can never succeed also never becomes the working one, so every
+ * later query repeats the same doomed division. The floor used to be 5 s against
+ * a measured 11.4 s login, which is why forensics reported "console unreachable"
+ * for guests that were answering serial in 10 ms (#69). A share this large is
+ * affordable only because `captureGuestSnapshot()` adds a login allowance to its
+ * query budget until the console answers; see `GUEST_LOGIN_ALLOWANCE_MS`.
+ *
+ * Exported for tests: this is the arithmetic the #69/B10 fix turns on, and it is
+ * otherwise reachable only through a live serial socket.
+ */
+export function credentialShareMs(timeoutMs: number, candidateCount: number, haveWorking: boolean): number {
+	if (haveWorking) return timeoutMs;
+	return Math.max(CONSOLE_LOGIN_COST_MS, Math.floor(timeoutMs / Math.max(1, candidateCount)));
+}
+
+/** Serial-console executor for boot forensics, or null when there is no serial
+ *  channel to talk to (foreground runs, or QEMU never got far enough).
+ *
+ *  Which account works depends on how far the failed boot got: a machine that
+ *  timed out before provisioning — and any machine after `clean()` — is factory
+ *  fresh and answers only to `admin` with an empty password, while a relaunch of
+ *  an already-provisioned machine needs its stored credentials. Rather than
+ *  guess, candidates are tried in order and the first that reaches a prompt is
+ *  reused for the rest of the snapshot. */
+function bootFailureGuestExec(state: MachineState): { exec: GuestExec; users: string[] } | null {
+	if (!channelFileExists(channelPath(state.machineDir, "serial"))) return null;
+
+	// Order matters only for speed — a wrong candidate costs a ~15 s login timeout
+	// before the next is tried. Stored credentials are the strongest evidence the
+	// machine is provisioned (provisioning stores them on success), so they lead;
+	// factory admin comes next because a cleaned machine has neither (`clean()`
+	// clears both the stored credentials and `state.user`), leaving `state.user`
+	// as the weakest case: a user configured on the machine but never provisioned
+	// onto the guest.
+	const candidates: { user: string; password: string }[] = [];
+	// Guarded even though secrets.ts already swallows unreadable/corrupt stores:
+	// this runs on the BOOT_TIMEOUT path *before* captureBootFailure(), so
+	// anything thrown here would replace the forensics report with an unrelated
+	// error — the exact failure mode the diagnostics modules exist to avoid.
+	let stored: { user: string; password: string } | null = null;
+	try {
+		stored = getInstanceCredentials(state.name);
+	} catch { /* no stored credentials — fall through to the other candidates */ }
+	if (stored) candidates.push(stored);
+	candidates.push({ user: "admin", password: "" });
+	if (state.user && state.user.name !== "admin" && state.user.password !== STORED_IN_SECRETS_PASSWORD) {
+		candidates.push({ user: state.user.name, password: state.user.password });
+	}
+
+	let working: { user: string; password: string } | undefined;
+	const exec: GuestExec = async (command, timeoutMs) => {
+		const order = working ? [working, ...candidates.filter((c) => c !== working)] : candidates;
+		const deadline = Date.now() + timeoutMs;
+		const share = credentialShareMs(timeoutMs, order.length, working !== undefined);
+		let lastError: unknown = new Error("no credential candidates");
+		for (const cred of order) {
+			const remaining = deadline - Date.now();
+			if (remaining <= 0) break;
+			try {
+				const { output, framed } = await consoleExec(
+					state.machineDir, command, cred.user, cred.password, Math.min(share, remaining), state.portBase,
+				);
+				working = cred;
+				return { output, framed };
+			} catch (e) {
+				lastError = e;
+			}
+		}
+		throw lastError instanceof Error ? lastError : new Error(String(lastError));
+	};
+
+	return { exec, users: candidates.map((c) => c.user) };
+}
+
+/**
+ * Capture the boot-failure evidence set for a machine that is **already
+ * running** — a request that failed after the machine was declared REST-ready.
+ *
+ * Same instruments as the BOOT_TIMEOUT path (guest snapshot, per-port slirp
+ * classification, monitor state, QEMU liveness, embedded logs), because #69's
+ * one-line `ECONNRESET` has never been enough to say whether the reset came from
+ * the client, RouterOS `www`, or slirp — and every one of those instruments
+ * answers a different part of that. What it does *not* share is the boot
+ * budget's excuse for the machine looking healthy: {@link FailureTrigger} is
+ * required here, so the report always records which request died and what
+ * changed just before it.
+ *
+ * Does not stop, remove, or otherwise touch the machine — the caller owns its
+ * lifecycle and may well want to keep testing against it. Never throws;
+ * `captureBootFailure()` swallows its own failures.
+ *
+ * One trace it does leave, and the reason that matters more here than on the
+ * boot path: the guest snapshot logs in over the serial console, and a RouterOS
+ * console login survives the host socket closing. A boot-timeout capture is
+ * followed by `stop()`/`remove()`, so nothing outlives it; here the machine
+ * keeps running. Harmless — `consoleExec()` `/quit`s a session belonging to a
+ * different user before it logs in — but a later console interaction on this
+ * machine starts from an already-logged-in prompt, not a `Login:` one.
+ */
+export async function captureRunningFailure(
+	state: MachineState,
+	phase: string,
+	trigger: FailureTrigger,
+): Promise<BootFailureReport> {
+	// Approximate, and labelled as such in the field docs: lastStartedAt is
+	// stamped at spawn and lastBootMs measured from just before it, so their sum
+	// is REST-ready to within the few ms between those two statements. Both are
+	// absent on a machine that never completed a timed boot, in which case the
+	// field is simply omitted rather than guessed at.
+	const readyAt = state.lastStartedAt !== undefined && state.lastBootMs !== undefined
+		? Date.parse(state.lastStartedAt) + state.lastBootMs
+		: undefined;
+	const guest = bootFailureGuestExec(state);
+	return await captureBootFailure({
+		name: state.name,
+		machineDir: state.machineDir,
+		arch: state.arch,
+		accel: state.lastAccel ?? "unknown",
+		pid: state.pid,
+		httpPort: state.ports.http?.host,
+		portBase: state.portBase,
+		phase,
+		trigger: {
+			...trigger,
+			sinceReadyMs: trigger.sinceReadyMs
+				?? (readyAt !== undefined && Number.isFinite(readyAt) ? Date.now() - readyAt : undefined),
+		},
+		reportDir: bootFailureReportDir(),
+		monitorQuery: (cmd) => monitorCommand(state.machineDir, cmd, 3000, state.portBase),
+		forwards: Object.values(state.ports),
+		guestExec: guest?.exec,
+		guestUser: guest?.users.join(" → "),
+	});
+}
+
+/** Tear down a machine that failed to boot, and describe what was done.
+ *
+ *  Default is full cleanup — a failed `start()` must not leave a half-dead
+ *  machine behind for the next call to trip over. QUICKCHR_PRESERVE_ON_FAILURE=1
+ *  inverts that for **local** debugging: the QEMU process is still stopped (it is
+ *  wedged, and its port block must be released), but the machine directory
+ *  survives so it can be inspected by hand.
+ *
+ *  Evidence does not depend on that flag, and CI deliberately does not set it —
+ *  `captureBootFailure()` has already written its report outside the machine dir
+ *  with qemu.log and serial.log embedded, so cleanup cannot destroy it (#79). */
+async function cleanupAfterBootFailure(
+	instance: ChrInstance,
+	name: string,
+	failure: BootFailureReport,
+): Promise<string> {
+	try { await instance.stop(); } catch { /* ignore */ }
+	if (failure.preserved) {
+		return ` Machine "${name}" was preserved for diagnosis (QUICKCHR_PRESERVE_ON_FAILURE=1) — remove it with \`quickchr remove ${name}\`.`;
+	}
+	try { await instance.remove(); } catch { /* ignore */ }
+	return ` Machine "${name}" has been cleaned up automatically.`;
+}
+
 /** Wait for CHR to boot with periodic progress status updates every 20s. */
 async function waitForBootWithProgress(
 	instance: ChrInstance,
@@ -1569,7 +1773,8 @@ export class QuickCHR {
 		const bootTimeout = defaultBootTimeout(arch, opts.installAllPackages, accel) + (opts.timeoutExtra ?? 0);
 
 		// Always wait for boot in background mode — the JSDoc promises "REST-ready".
-		let booted = await instance.waitForBoot(bootTimeout);
+		const probe = newBootProbeStats();
+		let booted = await instance.waitForBoot(bootTimeout, probe);
 
 		// Hardware-accelerated runners (nested-KVM on CI, HVF on virtualized macOS
 		// runners) occasionally produce a single QEMU process that boots but never
@@ -1586,22 +1791,35 @@ export class QuickCHR {
 			({ pid } = await spawnQemu(qemuArgs, machineDir, spawnInBackground, wrapper));
 			state.pid = pid;
 			saveMachine(state);
-			booted = await instance.waitForBoot(bootTimeout);
+			booted = await instance.waitForBoot(bootTimeout, probe);
 		}
 
 		if (!booted) {
-			let qemuLogTail = "";
-			try {
-				const logPath = join(machineDir, "qemu.log");
-				if (existsSync(logPath)) qemuLogTail = `\nqemu.log:\n${readFileSync(logPath, "utf-8").slice(-1200)}`;
-			} catch { /* ignore */ }
-			try { await instance.stop(); } catch { /* ignore */ }
-			try { await instance.remove(); } catch { /* ignore */ }
+			const guest = bootFailureGuestExec(state);
+			const failure = await captureBootFailure({
+				name,
+				machineDir,
+				arch,
+				accel,
+				bootTimeoutMs: bootTimeout,
+				pid,
+				httpPort: instance.ports.http,
+				portBase: state.portBase,
+				qemuArgs,
+				probe,
+				phase: "start",
+				reportDir: bootFailureReportDir(),
+				monitorQuery: (cmd) => monitorCommand(machineDir, cmd, 3000, state.portBase),
+				forwards: Object.values(state.ports),
+				guestExec: guest?.exec,
+				guestUser: guest?.users.join(" → "),
+			});
 			throw new QuickCHRError(
 				"BOOT_TIMEOUT",
 				`CHR did not respond within ${bootTimeout / 1000}s (accel=${accel})` +
 				(hasProvisioning ? " — provisioning could not run." : ".") +
-				` Machine "${name}" has been cleaned up automatically.${qemuLogTail}`,
+				await cleanupAfterBootFailure(instance, name, failure) +
+				`\n${failure.summary}`,
 			);
 		}
 
@@ -1882,20 +2100,34 @@ export class QuickCHR {
 
 		// Always wait for boot in background mode — start() promises "REST-ready".
 		const bootTimeout = defaultBootTimeout(state.arch, provisioningOpts?.installAllPackages, accel);
-		const booted = await instance.waitForBoot(bootTimeout);
+		const probe = newBootProbeStats();
+		const booted = await instance.waitForBoot(bootTimeout, probe);
 		if (!booted) {
-			let qemuLogTail = "";
-			try {
-				const logPath = join(state.machineDir, "qemu.log");
-				if (existsSync(logPath)) qemuLogTail = `\nqemu.log:\n${readFileSync(logPath, "utf-8").slice(-1200)}`;
-			} catch { /* ignore */ }
-			try { await instance.stop(); } catch { /* ignore */ }
-			try { await instance.remove(); } catch { /* ignore */ }
+			const guest = bootFailureGuestExec(state);
+			const failure = await captureBootFailure({
+				name: state.name,
+				machineDir: state.machineDir,
+				arch: state.arch,
+				accel,
+				bootTimeoutMs: bootTimeout,
+				pid,
+				httpPort: instance.ports.http,
+				portBase: state.portBase,
+				qemuArgs,
+				probe,
+				phase: "_launchExisting",
+				reportDir: bootFailureReportDir(),
+				monitorQuery: (cmd) => monitorCommand(state.machineDir, cmd, 3000, state.portBase),
+				forwards: Object.values(state.ports),
+				guestExec: guest?.exec,
+				guestUser: guest?.users.join(" → "),
+			});
 			throw new QuickCHRError(
 				"BOOT_TIMEOUT",
 				`CHR did not respond within ${bootTimeout / 1000}s (accel=${accel})` +
 				(hasProvisioning ? " — provisioning could not run." : ".") +
-				` Machine "${state.name}" has been cleaned up automatically.${qemuLogTail}`,
+				await cleanupAfterBootFailure(instance, state.name, failure) +
+				`\n${failure.summary}`,
 			);
 		}
 

package/src/lib/types.ts

@@ -3,6 +3,7 @@
  */
 
 import type { ProgressLogger } from "./log.ts";
+import type { BootProbeStats } from "./diagnostics.ts";
 
 // --- Version & Architecture ---
 
@@ -598,7 +599,10 @@ export interface ChrInstance {
 	 *  After `QuickCHR.start()` with provisioning options (installAllPackages, license, etc.),
 	 *  the returned instance is already REST-reachable — this returns immediately.
 	 *  Call this only when starting without provisioning, or as a safety check after a manual stop/restart. */
-	waitForBoot(timeoutMs?: number): Promise<boolean>;
+	/** @param stats Optional out-param; each REST readiness probe is classified
+	 *  into it (see diagnostics.ts BootProbeStats) so a failed wait can explain
+	 *  whether the forwarded port ever answered. */
+	waitForBoot(timeoutMs?: number, stats?: BootProbeStats): Promise<boolean>;
 	/** Poll a condition until it returns true or the timeout elapses.
 	 *  Retries every 2 s, swallowing errors from the condition function.
 	 *  Returns `true` when the condition passes, `false` on timeout.
@@ -784,6 +788,16 @@ export type ErrorCode =
 	| "QGA_UNSUPPORTED"
 	| "QGA_TIMEOUT"
 	| "DOWNLOAD_FAILED"
+	// A transfer that received nothing for DOWNLOAD_STALL_MS. Retriable, and
+	// retried before this surfaces — a wedged socket usually moves on the next
+	// attempt (#116).
+	| "DOWNLOAD_STALLED"
+	// A transfer that kept moving but could not finish inside a budget derived
+	// from its content-length and COLD_DOWNLOAD_FLOOR_BYTES_PER_S. Terminal on
+	// purpose: retrying re-downloads from zero on a link already measured slower
+	// than the floor, which is the "one slow transfer becomes three" behavior
+	// #116 exists to remove.
+	| "DOWNLOAD_TOO_SLOW"
 	| "MACHINE_EXISTS"
 	| "MACHINE_NOT_FOUND"
 	| "MACHINE_RUNNING"

package/src/lib/versions.ts

@@ -4,7 +4,7 @@
 
 import type { Arch, Channel } from "./types.ts";
 import { CHANNELS, QuickCHRError } from "./types.ts";
-import { fetchResilient } from "./net.ts";
+import { fetchResilient, isConnectionFailure } from "./net.ts";
 
 const UPGRADE_BASE = "https://upgrade.mikrotik.com/routeros/NEWESTa7";
 const DOWNLOAD_BASE = "https://download.mikrotik.com/routeros";
@@ -65,35 +65,112 @@ export function provisioningSupportHint(minimumVersion = MIN_PROVISION_VERSION):
 	return `Use --channel long-term or --version ${minimumVersion}+ for ${PROVISIONING_FEATURE_SUMMARY}, or keep the older version without provisioning options.`;
 }
 
-/** Fetch the latest version for a given channel from MikroTik's upgrade server. */
-export async function resolveVersion(channel: Channel): Promise<string> {
-	const url = `${UPGRADE_BASE}.${channel}`;
+/** Attempts for a connection-class failure while resolving a channel version. */
+export const VERSION_RESOLVE_MAX_ATTEMPTS = 3;
+
+/** Backoff base: attempt N waits N × this before N+1. Mirrors `downloadToFile`. */
+const VERSION_RESOLVE_RETRY_MS = 2000;
+
+export interface ResolveVersionOptions {
+	/** Attempts for connection-class failures. Defaults to {@link VERSION_RESOLVE_MAX_ATTEMPTS}. */
+	maxAttempts?: number;
+	/**
+	 * Override the backoff base. A test lever, like `downloadToFile`'s `stallMs`:
+	 * the real backoff spends 6 s of wall clock exhausting the default attempts.
+	 * Production callers should not set it.
+	 */
+	retryDelayMs?: number;
+}
 
-	const response = await fetchResilient(url);
-	if (!response.ok) {
-		throw new QuickCHRError(
-			"DOWNLOAD_FAILED",
-			`Failed to fetch version for channel "${channel}": HTTP ${response.status}`,
-		);
-	}
+/** A whole count of at least 1, or `fallback` for anything that isn't one. */
+function positiveInt(value: number | undefined, fallback: number): number {
+	if (value === undefined || !Number.isFinite(value)) return fallback;
+	return Math.max(1, Math.floor(value));
+}
+
+/** A finite, non-negative delay in ms, or `fallback` for anything that isn't one. */
+function nonNegativeMs(value: number | undefined, fallback: number): number {
+	if (value === undefined || !Number.isFinite(value)) return fallback;
+	return Math.max(0, value);
+}
 
-	const text = await response.text();
-	// Response format: "7.22.1 1774276515" (version + unix timestamp)
-	const version = text.trim().split(/\s+/)[0]?.trim();
-	if (!version || !isValidVersion(version)) {
-		throw new QuickCHRError(
-			"INVALID_VERSION",
-			`Unexpected version format for channel "${channel}": "${text.trim()}"`,
-		);
+/**
+ * Fetch the latest version for a given channel from MikroTik's upgrade server.
+ *
+ * Retried on a connection-class failure only — a transient refusal at
+ * `upgrade.mikrotik.com` used to fail a whole `quickchr start`, since this sits
+ * on the critical path of `start()`, the wizard and the CLI while only the
+ * download path had a retry policy (#121). An HTTP answer is the server's verdict
+ * and stays terminal on the first attempt: a 404 for a bad channel must fail fast.
+ */
+export async function resolveVersion(
+	channel: Channel,
+	opts: ResolveVersionOptions = {},
+): Promise<string> {
+	const url = `${UPGRADE_BASE}.${channel}`;
+	// Sanitized, not just clamped: these are public options, and `Math.max(1, NaN)`
+	// is NaN — the loop would never run, and the failure would read "all NaN
+	// attempts: undefined" instead of doing anything.
+	const maxAttempts = positiveInt(opts.maxAttempts, VERSION_RESOLVE_MAX_ATTEMPTS);
+	const retryDelayMs = nonNegativeMs(opts.retryDelayMs, VERSION_RESOLVE_RETRY_MS);
+	let lastError: unknown;
+
+	for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+		let text: string;
+		try {
+			const response = await fetchResilient(url);
+			if (!response.ok) {
+				// An HTTP answer is the server's verdict. Thrown as a QuickCHRError so
+				// the catch below can tell it from a transport failure and let it out.
+				throw new QuickCHRError(
+					"DOWNLOAD_FAILED",
+					`Failed to fetch version for channel "${channel}": HTTP ${response.status}`,
+				);
+			}
+			// Inside the try: the body can fail *after* the headers arrive (a reset
+			// mid-response), which is a transport failure like any other and must be
+			// retried rather than escaping the loop unwrapped.
+			text = await response.text();
+		} catch (err) {
+			// Only "could not reach the host at all" is retried. An HTTP verdict, an
+			// abort (the caller gave up) or a real bug surfaces immediately.
+			if (err instanceof QuickCHRError) throw err;
+			if (!isConnectionFailure(err)) throw err;
+			lastError = err;
+			if (attempt < maxAttempts) await Bun.sleep(attempt * retryDelayMs);
+			continue;
+		}
+
+		// Parsing is terminal: the server answered, and answering with something
+		// unparseable is not a condition another attempt can improve.
+		// Response format: "7.22.1 1774276515" (version + unix timestamp)
+		const version = text.trim().split(/\s+/)[0]?.trim();
+		if (!version || !isValidVersion(version)) {
+			throw new QuickCHRError(
+				"INVALID_VERSION",
+				`Unexpected version format for channel "${channel}": "${text.trim()}"`,
+			);
+		}
+
+		return version;
 	}
 
-	return version;
+	// Says "1 attempt" rather than "all 1 attempts" — these land in CI logs.
+	const tries = maxAttempts === 1 ? "1 attempt" : `all ${maxAttempts} attempts`;
+	throw new QuickCHRError(
+		"DOWNLOAD_FAILED",
+		`Failed to fetch version for channel "${channel}" on ${tries}: ` +
+			`${lastError instanceof Error ? lastError.message : String(lastError)}`,
+		`Check network access to ${new URL(url).hostname}, or pass an explicit --version to skip channel resolution.`,
+	);
 }
 
 /** Fetch latest versions for all channels in parallel. */
-export async function resolveAllVersions(): Promise<Record<Channel, string>> {
+export async function resolveAllVersions(
+	opts: ResolveVersionOptions = {},
+): Promise<Record<Channel, string>> {
 	const results = await Promise.all(
-		CHANNELS.map(async (ch) => [ch, await resolveVersion(ch)] as const),
+		CHANNELS.map(async (ch) => [ch, await resolveVersion(ch, opts)] as const),
 	);
 	return Object.fromEntries(results) as Record<Channel, string>;
 }