@alfe.ai/gateway 0.10.6 → 0.10.7

package/dist/node-remediation.js

@@ -1,4 +1,4 @@
-import { p as supportsOpenClawNode, u as OPENCLAW_NODE_REQUIREMENT } from "./runtime-process.js";
+import { w as supportsOpenClawNode, y as OPENCLAW_NODE_REQUIREMENT } from "./runtime-upgrade-recovery.js";
 import { mkdir, readFile, unlink, writeFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import { homedir } from "node:os";

package/dist/{runtime-process.js → runtime-upgrade-recovery.js}

@@ -1,8 +1,13 @@
-import { l as captureRuntimeCrash, u as captureRuntimeErrorOutput } from "./sentry.js";
+import { h as logger$1, i as captureCliFailure, l as captureRuntimeCrash, u as captureRuntimeErrorOutput } from "./sentry.js";
 import { createRequire } from "node:module";
+import { lstat, mkdir, readFile, readdir, rename, rm, symlink, unlink, writeFile } from "node:fs/promises";
+import { dirname, join, relative } from "node:path";
+import { homedir } from "node:os";
+import { existsSync } from "node:fs";
+import { execFile, spawn } from "node:child_process";
+import { promisify } from "node:util";
 import pino from "pino";
 import crypto from "crypto";
-import { spawn } from "node:child_process";
 import stream, { Readable } from "stream";
 import util, { format } from "util";
 import http from "http";
@@ -52,6 +57,9 @@ var __require = /* @__PURE__ */ createRequire(import.meta.url);
 *   - the daemon `runtime.update` default (this package)
 *   - managed provisioning (`services/compute/src/lib/provisioner.ts` installs the
 *     Node.js 24 channel before the CLI consumes this pin)
+*   - the dashboard/fleet upgrade ceiling: `services/agents` mirrors this literal
+*     as `PINNED_OPENCLAW_VERSION` in `version-catalog.service.ts` (its test fails
+*     on drift) — bump both in the same change
 *
 * Pinning (rather than floating `npm install -g openclaw` → `latest`) is deliberate:
 * OpenClaw runs on every managed agent, so a `latest` that changes runtime semantics —
@@ -100,6 +108,67 @@ function supportsOpenClawNode(nodeVersion) {
 function parseOpenClawVersion(output) {
 	return /(?:^|\s)(\d{4}\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?)(?:\s|$)/u.exec(output.trim())?.[1];
 }
+/**
+* Compare two EXACT OpenClaw `YYYY.M.P` versions numerically. Anything else —
+* a pre-release/build suffix (`2026.9.5-rc.2`, `2026.9.5+build`), a fourth
+* segment (`2026.9.5.1`), a branded `--version` line — is `undefined`, never
+* "equal": callers treat `undefined` as "not a validated release".
+*/
+function compareOpenClawVersions(a, b) {
+	const parse = (value) => {
+		const match = /^(\d{4})\.(\d+)\.(\d+)$/u.exec(value.trim());
+		return match ? [
+			Number(match[1]),
+			Number(match[2]),
+			Number(match[3])
+		] : void 0;
+	};
+	const left = parse(a);
+	const right = parse(b);
+	if (!left || !right) return void 0;
+	for (let i = 0; i < 3; i++) if (left[i] !== right[i]) return left[i] - right[i];
+	return 0;
+}
+/**
+* OpenClaw 2026.8.1 dropped the 2026.6.x `register ?? activate` plugin-export
+* alias: from 2026.8 a plugin whose default export has no `register` function
+* fails to load ("missing register/activate export") and `doctor --fix` exits
+* non-zero. `@alfe.ai/*` plugins export `register` from alfe manifest 1.1.50
+* (#373); anything installed from an older manifest is activate-only.
+*/
+const OPENCLAW_REGISTER_EXPORT_REQUIRED_FROM = "2026.8.0";
+/** Whether `version` is an OpenClaw that only loads `register`-exporting plugins. */
+function openClawRequiresRegisterExport(version) {
+	if (!version) return false;
+	const cmp = compareOpenClawVersions(version, OPENCLAW_REGISTER_EXPORT_REQUIRED_FROM);
+	return cmp !== void 0 && cmp >= 0;
+}
+/**
+* The OpenClaw version a `runtime.update` actually installs. The pin is the
+* ceiling: a requested version above it (the dashboard offered npm `latest`,
+* 2026.9.7, to Dana on 2026-10-02) or an unparseable one is clamped to the pin,
+* because an unvalidated runtime is a fleet outage with no Alfe deploy. Versions
+* at or below the pin pass through (operator-chosen rollback targets).
+*/
+function resolveOpenClawUpgradeTarget(requested) {
+	if (!requested) return {
+		version: PINNED_OPENCLAW_VERSION,
+		clamped: false
+	};
+	const cmp = compareOpenClawVersions(requested, PINNED_OPENCLAW_VERSION);
+	if (cmp === void 0 || cmp > 0) return {
+		version: PINNED_OPENCLAW_VERSION,
+		clamped: true
+	};
+	if (cmp === 0) return {
+		version: PINNED_OPENCLAW_VERSION,
+		clamped: false
+	};
+	return {
+		version: requested.trim(),
+		clamped: false
+	};
+}
 //#endregion
 //#region ../../node_modules/.pnpm/dotenv@17.2.2/node_modules/dotenv/package.json
 var require_package = /* @__PURE__ */ __commonJSMin(((exports, module) => {
@@ -15832,4 +15901,396 @@ var RuntimeProcess = class {
 	}
 };
 //#endregion
-export { ErrorLineDetector as a, createLogger as c, PINNED_OPENCLAW_VERSION as d, parseOpenClawVersion as f, __require as h, CaptureThrottle as i, OPENCLAW_MANAGED_NODE_MAJOR as l, __commonJSMin as m, OPENCLAW_EXTERNAL_SUPERVISOR_ENV as n, STRICT_ERROR_CAPTURE_PROFILE as o, supportsOpenClawNode as p, RuntimeProcess as r, toFingerprintKey as s, DRAIN_GRACE_MS as t, OPENCLAW_NODE_REQUIREMENT as u };
+//#region src/runtime-upgrade-recovery.ts
+/**
+* Surviving an OS-level restart of the daemon during an OpenClaw upgrade.
+*
+* Incident (Dana, prod, 2026-10-02): `apt-daily-upgrade` upgraded libssl and
+* needrestart restarted `alfe-gateway` while the daemon's `npm install -g
+* openclaw@…` was mid-reify. systemd SIGKILLed everything in the unit's cgroup,
+* npm included, leaving a partial `/usr/lib/node_modules/openclaw` (no
+* `openclaw.mjs`), the intact old tree retired to `.openclaw-<hash>` and the bin
+* symlink retired to `/usr/bin/.openclaw-<hash>`. The restarted daemon had no
+* record that an upgrade had been running, so the agent stayed without a
+* runtime (and still reported `running`).
+*
+* Two defences live here:
+*   1. **Cgroup isolation** — on a systemd host the long, state-mutating
+*      commands (npm install, doctor --fix) run inside a transient scope
+*      (`systemd-run --scope`), so stopping/restarting the daemon's unit no
+*      longer kills them. `--scope` executes the command as the caller's own
+*      child (stdio, exit code and environment are unchanged), it only moves it
+*      into its own cgroup. Off systemd (self-hosted macOS, containers) the
+*      commands run directly, as before.
+*   2. **Durable marker + boot recovery** — `~/.alfe/runtime-upgrade.json` is
+*      written before the package is replaced and removed when the transaction
+*      finishes. A marker found at boot means the daemon died mid-upgrade:
+*        - phase `installing` (doctor never started, state not migrated): put the
+*          PREVIOUS version back — first by restoring npm's retired
+*          `.openclaw-*` sibling (no network), else `npm install -g
+*          openclaw@<previous>` (or the pin when the previous is unknown);
+*        - phase `migrating` (doctor started; rollback is unsafe): make sure the
+*          TARGET package is installed and re-run the idempotent doctor.
+*      Recovery runs under the exclusive RuntimeGate + CLI lock before the first
+*      runtime start, is bounded to {@link MAX_RECOVERY_ATTEMPTS} boots, and is
+*      reported to Sentry on failure.
+*/
+const execFileAsync = promisify(execFile);
+const SYSTEMD_RUN_PATHS = ["/usr/bin/systemd-run", "/bin/systemd-run"];
+/**
+* Whether this daemon runs as a systemd service that can start transient
+* scopes. `INVOCATION_ID` is set by systemd for every unit it starts, so its
+* absence means "not under systemd" (an interactive/self-hosted daemon, a
+* container) and the commands run directly.
+*/
+function detectUpgradeIsolation(env = process.env, uid = process.getuid?.(), hasSystemdRun = () => SYSTEMD_RUN_PATHS.some((path) => existsSync(path)), platform = process.platform) {
+	if (platform !== "linux") return void 0;
+	if (!env.INVOCATION_ID || !hasSystemdRun()) return void 0;
+	if (uid === 0) return { manager: "system" };
+	return env.XDG_RUNTIME_DIR ? { manager: "user" } : void 0;
+}
+function upgradeScopeUnitName(now = Date.now()) {
+	return `alfe-openclaw-upgrade-${String(now)}`;
+}
+/** Wrap a command so it runs in its own transient scope (own cgroup). */
+function wrapInUpgradeScope(isolation, unit, command, args) {
+	if (!isolation) return {
+		command,
+		args
+	};
+	return {
+		command: "systemd-run",
+		args: [
+			...isolation.manager === "user" ? ["--user"] : [],
+			"--scope",
+			"--quiet",
+			"--collect",
+			`--unit=${unit}`,
+			"--",
+			command,
+			...args
+		]
+	};
+}
+function defaultRuntimeUpgradeMarkerPath() {
+	return join(homedir(), ".alfe", "runtime-upgrade.json");
+}
+async function writeRuntimeUpgradeMarker(path, marker) {
+	await mkdir(dirname(path), {
+		recursive: true,
+		mode: 448
+	});
+	const tmp = `${path}.${String(process.pid)}.tmp`;
+	await writeFile(tmp, JSON.stringify(marker, null, 2) + "\n", { mode: 384 });
+	await rename(tmp, path);
+}
+async function readRuntimeUpgradeMarker(path) {
+	let raw;
+	try {
+		raw = await readFile(path, "utf8");
+	} catch (err) {
+		if (err.code === "ENOENT") return void 0;
+		throw err;
+	}
+	const parsed = JSON.parse(raw);
+	if (parsed?.runtime !== "openclaw" || parsed.phase !== "installing" && parsed.phase !== "migrating" || typeof parsed.targetVersion !== "string") throw new Error(`Runtime upgrade marker ${path} is malformed`);
+	return parsed;
+}
+async function clearRuntimeUpgradeMarker(path) {
+	await rm(path, { force: true });
+}
+const NPM_INSTALL_TIMEOUT_MS = 3e5;
+const DOCTOR_TIMEOUT_MS = 9e5;
+const DOCTOR_MAX_BUFFER_BYTES = 16 * 1024 * 1024;
+const SCOPE_WAIT_POLL_MS = 5e3;
+const SCOPE_WAIT_MAX_MS = 20 * 6e4;
+/**
+* `doctor --fix` runs the state/config migration. On 2026.9.x `--json` puts
+* doctor into READ-ONLY lint mode and it refuses the combination outright
+* (exit 2: "doctor --json runs read-only lint checks and cannot be combined
+* with --repair, --fix, or --force"), so the migration must run without it.
+*/
+const OPENCLAW_DOCTOR_FIX_ARGS = [
+	"doctor",
+	"--fix",
+	"--non-interactive",
+	"--yes"
+];
+async function installedOpenClawVersion(run, env) {
+	try {
+		const { stdout } = await run("openclaw", ["--version"], {
+			timeout: 3e4,
+			env
+		});
+		return parseOpenClawVersion(stdout);
+	} catch {
+		return;
+	}
+}
+/** Wait for the interrupted upgrade's scopes to finish so recovery never races a still-running npm/doctor. */
+async function waitForScopes(run, isolation, units, sleep, now) {
+	if (!isolation || units.length === 0) return;
+	const deadline = now() + SCOPE_WAIT_MAX_MS;
+	for (const unit of units) for (;;) {
+		let active = false;
+		try {
+			await run("systemctl", [
+				...isolation.manager === "user" ? ["--user"] : [],
+				"is-active",
+				"--quiet",
+				`${unit}.scope`
+			], { timeout: 1e4 });
+			active = true;
+		} catch {
+			active = false;
+		}
+		if (!active) break;
+		if (now() >= deadline) throw new Error(`upgrade scope ${unit}.scope is still running after ${String(SCOPE_WAIT_MAX_MS / 6e4)} min`);
+		logger$1.info({ unit }, "Interrupted runtime upgrade still running in its scope — waiting");
+		await sleep(SCOPE_WAIT_POLL_MS);
+	}
+}
+/**
+* Put npm's retired `.openclaw-*` tree for `version` back in place, with its
+* bin link. npm's reify renames the live package to `.<name>-<rand>` before
+* extracting and only deletes it on success, so a killed reify leaves the old
+* tree intact next to the partial one. Returns true when `openclaw --version`
+* then reports `version`.
+*/
+async function restoreRetiredOpenClawTree(run, env, version) {
+	const root = (await run("npm", ["root", "-g"], {
+		timeout: 3e4,
+		env
+	})).stdout.trim();
+	const prefix = (await run("npm", ["prefix", "-g"], {
+		timeout: 3e4,
+		env
+	})).stdout.trim();
+	if (!root.startsWith("/") || !prefix.startsWith("/")) return false;
+	let candidate;
+	for (const entry of await readdir(root).catch(() => [])) {
+		if (!entry.startsWith(".openclaw-")) continue;
+		const dir = join(root, entry);
+		try {
+			const pkg = JSON.parse(await readFile(join(dir, "package.json"), "utf8"));
+			if (pkg.name !== "openclaw" || pkg.version !== version) continue;
+			const bin = typeof pkg.bin === "string" ? pkg.bin : pkg.bin?.openclaw;
+			if (typeof bin !== "string" || !existsSync(join(dir, bin))) continue;
+			candidate = {
+				dir,
+				bin
+			};
+			break;
+		} catch {
+			continue;
+		}
+	}
+	if (!candidate) return false;
+	const live = join(root, "openclaw");
+	if (existsSync(live)) {
+		const aside = join(root, `.openclaw-alfe-partial-${String(Date.now())}`);
+		await rename(live, aside);
+		await rm(aside, {
+			recursive: true,
+			force: true
+		});
+	}
+	await rename(candidate.dir, live);
+	const binLink = join(prefix, "bin", "openclaw");
+	await lstat(binLink).then(() => unlink(binLink), () => void 0);
+	await symlink(relative(dirname(binLink), join(live, candidate.bin)), binLink);
+	logger$1.info({
+		restored: candidate.dir,
+		version
+	}, "Restored the retired OpenClaw package tree");
+	return await installedOpenClawVersion(run, env) === version;
+}
+/**
+* Finish or undo an OpenClaw upgrade the daemon did not live to complete.
+* Never throws; the caller holds the exclusive RuntimeGate and the CLI lock.
+*/
+async function recoverInterruptedRuntimeUpgrade(options = {}) {
+	const markerPath = options.markerPath ?? defaultRuntimeUpgradeMarkerPath();
+	const run = options.runCommand ?? (async (command, args, commandOptions) => {
+		const result = await execFileAsync(command, args, commandOptions);
+		return {
+			stdout: result.stdout,
+			stderr: result.stderr
+		};
+	});
+	const env = {
+		...process.env,
+		...OPENCLAW_EXTERNAL_SUPERVISOR_ENV
+	};
+	const sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+	const now = options.now ?? Date.now;
+	let marker;
+	try {
+		marker = await readRuntimeUpgradeMarker(markerPath);
+	} catch (err) {
+		captureCliFailure("openclaw-upgrade-recovery", err);
+		logger$1.error({ err: err instanceof Error ? err.message : String(err) }, "Discarding unreadable runtime upgrade marker");
+		await clearRuntimeUpgradeMarker(markerPath).catch(() => void 0);
+		return { status: "none" };
+	}
+	if (!marker) return { status: "none" };
+	const attempts = (marker.recoveryAttempts ?? 0) + 1;
+	const phase = marker.phase;
+	if (attempts > 3) {
+		const error = `Gave up recovering the interrupted OpenClaw upgrade to ${marker.targetVersion} after ${String(3)} boots`;
+		captureCliFailure("openclaw-upgrade-recovery", new Error(error));
+		logger$1.error({ marker }, error);
+		await clearRuntimeUpgradeMarker(markerPath).catch(() => void 0);
+		return {
+			status: "failed",
+			phase,
+			error,
+			gaveUp: true
+		};
+	}
+	try {
+		await writeRuntimeUpgradeMarker(markerPath, {
+			...marker,
+			recoveryAttempts: attempts
+		});
+	} catch (err) {
+		const error = `Could not record the recovery attempt in ${markerPath}: ${err instanceof Error ? err.message : String(err)}`;
+		captureCliFailure("openclaw-upgrade-recovery", new Error(error, { cause: err }));
+		logger$1.error({ err: error }, "Not recovering the interrupted OpenClaw upgrade — marker is not writable");
+		return {
+			status: "failed",
+			phase,
+			error,
+			gaveUp: false
+		};
+	}
+	logger$1.warn({
+		marker,
+		attempt: attempts
+	}, "Found an interrupted OpenClaw runtime upgrade — recovering before the runtime starts");
+	let scopeSequence = 0;
+	const scoped = (command, args) => {
+		scopeSequence += 1;
+		return wrapInUpgradeScope(options.isolation, `${upgradeScopeUnitName(now())}-recovery-${String(scopeSequence)}`, command, args);
+	};
+	try {
+		await waitForScopes(run, options.isolation, marker.scopeUnits ?? [], sleep, now);
+		const installed = await installedOpenClawVersion(run, env);
+		const target = marker.previousVersion;
+		if (phase === "installing" && target !== void 0 && installed !== marker.targetVersion) {
+			let method = "already-in-place";
+			if (installed !== target) {
+				if (await restoreRetiredOpenClawTree(run, env, target).catch((err) => {
+					logger$1.warn({ err: err instanceof Error ? err.message : String(err) }, "Restoring the retired OpenClaw tree failed — reinstalling");
+					return false;
+				})) method = "restored-retired-tree";
+				else {
+					const install = scoped("npm", [
+						"install",
+						"-g",
+						`openclaw@${target}`
+					]);
+					await run(install.command, install.args, {
+						timeout: NPM_INSTALL_TIMEOUT_MS,
+						env
+					});
+					method = "reinstalled";
+				}
+				const verified = await installedOpenClawVersion(run, env);
+				if (verified !== target) throw new Error(`OpenClaw did not verify after recovery: expected ${target}, resolved ${verified ?? "nothing"}`);
+			}
+			await clearRuntimeUpgradeMarker(markerPath);
+			logger$1.info({
+				version: target,
+				method
+			}, "Recovered interrupted OpenClaw upgrade (pre-migration): previous runtime restored");
+			return {
+				status: "recovered",
+				phase,
+				version: target,
+				method
+			};
+		}
+		let method = phase === "migrating" ? "doctor-rerun" : "rolled-forward";
+		if (installed !== marker.targetVersion) {
+			const install = scoped("npm", [
+				"install",
+				"-g",
+				`openclaw@${marker.targetVersion}`
+			]);
+			await run(install.command, install.args, {
+				timeout: NPM_INSTALL_TIMEOUT_MS,
+				env
+			});
+			const verified = await installedOpenClawVersion(run, env);
+			if (verified !== marker.targetVersion) throw new Error(`OpenClaw did not verify after recovery: expected ${marker.targetVersion}, resolved ${verified ?? "nothing"}`);
+			method = `${method}-after-reinstall`;
+		}
+		const doctor = scoped("openclaw", [...OPENCLAW_DOCTOR_FIX_ARGS]);
+		await run(doctor.command, doctor.args, {
+			timeout: DOCTOR_TIMEOUT_MS,
+			env,
+			maxBuffer: DOCTOR_MAX_BUFFER_BYTES
+		});
+		await clearRuntimeUpgradeMarker(markerPath);
+		logger$1.info({
+			version: marker.targetVersion,
+			method
+		}, "Recovered interrupted OpenClaw upgrade (migration completed)");
+		return {
+			status: "recovered",
+			phase,
+			version: marker.targetVersion,
+			method
+		};
+	} catch (err) {
+		const error = err instanceof Error ? err.message : String(err);
+		captureCliFailure("openclaw-upgrade-recovery", err instanceof Error ? err : new Error(error));
+		logger$1.error({
+			err: error,
+			phase,
+			attempt: attempts
+		}, "Recovering the interrupted OpenClaw upgrade failed — will retry on the next daemon start");
+		return {
+			status: "failed",
+			phase,
+			error,
+			gaveUp: false
+		};
+	}
+}
+/**
+* Start boot-time recovery when a marker exists. Returns `undefined` (and does
+* nothing) without one. Otherwise the exclusive RuntimeGate hold is RESERVED
+* SYNCHRONOUSLY before this returns: `RuntimeGate.suspendExclusive` sets its
+* exclusive flag and depth before its first await, so a `requestStart()` issued
+* right after this call folds into our `resume()` instead of spawning a runtime
+* whose binary may be missing. Order: gate, then CLI lock (the documented
+* lock-ordering invariant). The returned promise never rejects.
+*/
+function startBootUpgradeRecovery(options) {
+	if (!existsSync(options.markerPath)) return void 0;
+	const reservation = options.gate.suspendExclusive();
+	return (async () => {
+		try {
+			await reservation;
+			const outcome = await options.cliLock.run(() => options.recover(options.markerPath));
+			logger$1.info({ outcome }, "Interrupted runtime upgrade recovery finished");
+			if (outcome.status === "recovered") options.onRecovered?.(outcome.version);
+			return outcome;
+		} catch (err) {
+			const error = err instanceof Error ? err.message : String(err);
+			logger$1.error({ err: error }, "Interrupted runtime upgrade recovery threw");
+			return {
+				status: "failed",
+				phase: "installing",
+				error,
+				gaveUp: false
+			};
+		} finally {
+			await options.gate.resume();
+		}
+	})();
+}
+//#endregion
+export { resolveOpenClawUpgradeTarget as C, __require as E, parseOpenClawVersion as S, __commonJSMin as T, createLogger as _, recoverInterruptedRuntimeUpgrade as a, PINNED_OPENCLAW_VERSION as b, wrapInUpgradeScope as c, OPENCLAW_EXTERNAL_SUPERVISOR_ENV as d, RuntimeProcess as f, toFingerprintKey as g, STRICT_ERROR_CAPTURE_PROFILE as h, detectUpgradeIsolation as i, writeRuntimeUpgradeMarker as l, ErrorLineDetector as m, clearRuntimeUpgradeMarker as n, startBootUpgradeRecovery as o, CaptureThrottle as p, defaultRuntimeUpgradeMarkerPath as r, upgradeScopeUnitName as s, OPENCLAW_DOCTOR_FIX_ARGS as t, DRAIN_GRACE_MS as u, OPENCLAW_MANAGED_NODE_MAJOR as v, supportsOpenClawNode as w, openClawRequiresRegisterExport as x, OPENCLAW_NODE_REQUIREMENT as y };