@alfe.ai/gateway 0.10.5 → 0.10.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/health.js +848 -415
- package/dist/node-remediation.js +1 -1
- package/dist/{runtime-process.js → runtime-upgrade-recovery.js} +524 -7
- package/dist/runtime-upgrade.js +327 -18
- package/dist/sentry.js +16 -9
- package/dist/src/index.d.ts +10 -3
- package/dist/src/index.js +1 -1
- package/package.json +4 -4
package/dist/node-remediation.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { w as supportsOpenClawNode, y as OPENCLAW_NODE_REQUIREMENT } from "./runtime-upgrade-recovery.js";
|
|
2
2
|
import { mkdir, readFile, unlink, writeFile } from "node:fs/promises";
|
|
3
3
|
import { dirname, join } from "node:path";
|
|
4
4
|
import { homedir } from "node:os";
|
|
@@ -1,8 +1,13 @@
|
|
|
1
|
-
import { l as captureRuntimeCrash, u as captureRuntimeErrorOutput } from "./sentry.js";
|
|
1
|
+
import { h as logger$1, i as captureCliFailure, l as captureRuntimeCrash, u as captureRuntimeErrorOutput } from "./sentry.js";
|
|
2
2
|
import { createRequire } from "node:module";
|
|
3
|
+
import { lstat, mkdir, readFile, readdir, rename, rm, symlink, unlink, writeFile } from "node:fs/promises";
|
|
4
|
+
import { dirname, join, relative } from "node:path";
|
|
5
|
+
import { homedir } from "node:os";
|
|
6
|
+
import { existsSync } from "node:fs";
|
|
7
|
+
import { execFile, spawn } from "node:child_process";
|
|
8
|
+
import { promisify } from "node:util";
|
|
3
9
|
import pino from "pino";
|
|
4
10
|
import crypto from "crypto";
|
|
5
|
-
import { spawn } from "node:child_process";
|
|
6
11
|
import stream, { Readable } from "stream";
|
|
7
12
|
import util, { format } from "util";
|
|
8
13
|
import http from "http";
|
|
@@ -52,6 +57,9 @@ var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
|
52
57
|
* - the daemon `runtime.update` default (this package)
|
|
53
58
|
* - managed provisioning (`services/compute/src/lib/provisioner.ts` installs the
|
|
54
59
|
* Node.js 24 channel before the CLI consumes this pin)
|
|
60
|
+
* - the dashboard/fleet upgrade ceiling: `services/agents` mirrors this literal
|
|
61
|
+
* as `PINNED_OPENCLAW_VERSION` in `version-catalog.service.ts` (its test fails
|
|
62
|
+
* on drift) — bump both in the same change
|
|
55
63
|
*
|
|
56
64
|
* Pinning (rather than floating `npm install -g openclaw` → `latest`) is deliberate:
|
|
57
65
|
* OpenClaw runs on every managed agent, so a `latest` that changes runtime semantics —
|
|
@@ -100,6 +108,67 @@ function supportsOpenClawNode(nodeVersion) {
|
|
|
100
108
|
function parseOpenClawVersion(output) {
|
|
101
109
|
return /(?:^|\s)(\d{4}\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?)(?:\s|$)/u.exec(output.trim())?.[1];
|
|
102
110
|
}
|
|
111
|
+
/**
|
|
112
|
+
* Compare two EXACT OpenClaw `YYYY.M.P` versions numerically. Anything else —
|
|
113
|
+
* a pre-release/build suffix (`2026.9.5-rc.2`, `2026.9.5+build`), a fourth
|
|
114
|
+
* segment (`2026.9.5.1`), a branded `--version` line — is `undefined`, never
|
|
115
|
+
* "equal": callers treat `undefined` as "not a validated release".
|
|
116
|
+
*/
|
|
117
|
+
function compareOpenClawVersions(a, b) {
|
|
118
|
+
const parse = (value) => {
|
|
119
|
+
const match = /^(\d{4})\.(\d+)\.(\d+)$/u.exec(value.trim());
|
|
120
|
+
return match ? [
|
|
121
|
+
Number(match[1]),
|
|
122
|
+
Number(match[2]),
|
|
123
|
+
Number(match[3])
|
|
124
|
+
] : void 0;
|
|
125
|
+
};
|
|
126
|
+
const left = parse(a);
|
|
127
|
+
const right = parse(b);
|
|
128
|
+
if (!left || !right) return void 0;
|
|
129
|
+
for (let i = 0; i < 3; i++) if (left[i] !== right[i]) return left[i] - right[i];
|
|
130
|
+
return 0;
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* OpenClaw 2026.8.1 dropped the 2026.6.x `register ?? activate` plugin-export
|
|
134
|
+
* alias: from 2026.8 a plugin whose default export has no `register` function
|
|
135
|
+
* fails to load ("missing register/activate export") and `doctor --fix` exits
|
|
136
|
+
* non-zero. `@alfe.ai/*` plugins export `register` from alfe manifest 1.1.50
|
|
137
|
+
* (#373); anything installed from an older manifest is activate-only.
|
|
138
|
+
*/
|
|
139
|
+
const OPENCLAW_REGISTER_EXPORT_REQUIRED_FROM = "2026.8.0";
|
|
140
|
+
/** Whether `version` is an OpenClaw that only loads `register`-exporting plugins. */
|
|
141
|
+
function openClawRequiresRegisterExport(version) {
|
|
142
|
+
if (!version) return false;
|
|
143
|
+
const cmp = compareOpenClawVersions(version, OPENCLAW_REGISTER_EXPORT_REQUIRED_FROM);
|
|
144
|
+
return cmp !== void 0 && cmp >= 0;
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* The OpenClaw version a `runtime.update` actually installs. The pin is the
|
|
148
|
+
* ceiling: a requested version above it (the dashboard offered npm `latest`,
|
|
149
|
+
* 2026.9.7, to Dana on 2026-10-02) or an unparseable one is clamped to the pin,
|
|
150
|
+
* because an unvalidated runtime is a fleet outage with no Alfe deploy. Versions
|
|
151
|
+
* at or below the pin pass through (operator-chosen rollback targets).
|
|
152
|
+
*/
|
|
153
|
+
function resolveOpenClawUpgradeTarget(requested) {
|
|
154
|
+
if (!requested) return {
|
|
155
|
+
version: PINNED_OPENCLAW_VERSION,
|
|
156
|
+
clamped: false
|
|
157
|
+
};
|
|
158
|
+
const cmp = compareOpenClawVersions(requested, PINNED_OPENCLAW_VERSION);
|
|
159
|
+
if (cmp === void 0 || cmp > 0) return {
|
|
160
|
+
version: PINNED_OPENCLAW_VERSION,
|
|
161
|
+
clamped: true
|
|
162
|
+
};
|
|
163
|
+
if (cmp === 0) return {
|
|
164
|
+
version: PINNED_OPENCLAW_VERSION,
|
|
165
|
+
clamped: false
|
|
166
|
+
};
|
|
167
|
+
return {
|
|
168
|
+
version: requested.trim(),
|
|
169
|
+
clamped: false
|
|
170
|
+
};
|
|
171
|
+
}
|
|
103
172
|
//#endregion
|
|
104
173
|
//#region ../../node_modules/.pnpm/dotenv@17.2.2/node_modules/dotenv/package.json
|
|
105
174
|
var require_package = /* @__PURE__ */ __commonJSMin(((exports, module) => {
|
|
@@ -15413,6 +15482,14 @@ const BACKOFF_INITIAL_MS = 1e3;
|
|
|
15413
15482
|
const BACKOFF_MAX_MS = 3e4;
|
|
15414
15483
|
const STABLE_UPTIME_MS = 6e4;
|
|
15415
15484
|
/**
|
|
15485
|
+
* Output markers (lower-cased substrings) that distinguish the planned exit-78
|
|
15486
|
+
* from any other EX_CONFIG. 2026.9.5 raises the same startup race two ways:
|
|
15487
|
+
* "OpenClaw migration inputs changed during startup", and
|
|
15488
|
+
* `throwStartupMigrationGuardRejected`'s "OpenClaw startup migrations were
|
|
15489
|
+
* skipped because the selected config changed during startup… Retry startup".
|
|
15490
|
+
*/
|
|
15491
|
+
const MIGRATION_RESTART_MARKERS = ["migration inputs changed during startup", "selected config changed during startup"];
|
|
15492
|
+
/**
|
|
15416
15493
|
* Alfe owns supervision and version rollout; OpenClaw must not self-manage.
|
|
15417
15494
|
*
|
|
15418
15495
|
* `OPENCLAW_NODE_UPDATE_RESPAWNED=1` also disables 2026.9.x's node-runtime
|
|
@@ -15446,6 +15523,17 @@ var RuntimeProcess = class {
|
|
|
15446
15523
|
backoffMs = BACKOFF_INITIAL_MS;
|
|
15447
15524
|
lastStartTime = 0;
|
|
15448
15525
|
restartTimer = null;
|
|
15526
|
+
/** Whether the CURRENT run printed one of {@link MIGRATION_RESTART_MARKERS}. Reset on every spawn. */
|
|
15527
|
+
sawMigrationInputsChanged = false;
|
|
15528
|
+
/** Consecutive exit-78 migration restarts treated as planned (see {@link MAX_CONSECUTIVE_MIGRATION_RESTARTS}). */
|
|
15529
|
+
consecutiveMigrationRestarts = 0;
|
|
15530
|
+
/**
|
|
15531
|
+
* Daemon shutdown latch. Once set, `start()` / `resume()` / `restart()` are
|
|
15532
|
+
* permanent no-ops: an in-flight reconcile's final gate resume (or a held
|
|
15533
|
+
* suspension released by the drain loop) must not respawn the runtime after
|
|
15534
|
+
* shutdown stopped it.
|
|
15535
|
+
*/
|
|
15536
|
+
shutdownLatched = false;
|
|
15449
15537
|
ringBuffer = new OutputRingBuffer();
|
|
15450
15538
|
detector = new ErrorLineDetector();
|
|
15451
15539
|
throttle = new CaptureThrottle(STABLE_UPTIME_MS);
|
|
@@ -15492,9 +15580,10 @@ var RuntimeProcess = class {
|
|
|
15492
15580
|
* Start the runtime process.
|
|
15493
15581
|
*/
|
|
15494
15582
|
start() {
|
|
15495
|
-
if (this.stopped || this.child !== null) return;
|
|
15583
|
+
if (this.stopped || this.shutdownLatched || this.child !== null) return;
|
|
15496
15584
|
const { command, args } = this.resolveCommand();
|
|
15497
15585
|
this.lastStartTime = Date.now();
|
|
15586
|
+
this.sawMigrationInputsChanged = false;
|
|
15498
15587
|
log.info({
|
|
15499
15588
|
runtime: this.options.runtime,
|
|
15500
15589
|
command,
|
|
@@ -15555,8 +15644,18 @@ var RuntimeProcess = class {
|
|
|
15555
15644
|
}, "Runtime stopped (expected)");
|
|
15556
15645
|
return;
|
|
15557
15646
|
}
|
|
15558
|
-
|
|
15559
|
-
|
|
15647
|
+
const uptime = Date.now() - this.lastStartTime;
|
|
15648
|
+
if (uptime >= 6e4) this.consecutiveMigrationRestarts = 0;
|
|
15649
|
+
const isGraceful = !spawnError && code === 0 && signal == null;
|
|
15650
|
+
const isPlannedMigrationRestart = !spawnError && signal == null && code === 78 && this.sawMigrationInputsChanged && this.consecutiveMigrationRestarts < 1;
|
|
15651
|
+
if (isGraceful || isPlannedMigrationRestart) {
|
|
15652
|
+
if (isPlannedMigrationRestart) {
|
|
15653
|
+
this.consecutiveMigrationRestarts += 1;
|
|
15654
|
+
log.info({
|
|
15655
|
+
runtime: this.options.runtime,
|
|
15656
|
+
code
|
|
15657
|
+
}, "Runtime asked for a restart after its startup migration inputs changed — restarting (planned, not a crash)");
|
|
15658
|
+
} else log.info({
|
|
15560
15659
|
runtime: this.options.runtime,
|
|
15561
15660
|
code
|
|
15562
15661
|
}, "Runtime exited gracefully — restarting");
|
|
@@ -15573,7 +15672,6 @@ var RuntimeProcess = class {
|
|
|
15573
15672
|
spawnError: spawnError?.message,
|
|
15574
15673
|
backoffMs: this.backoffMs
|
|
15575
15674
|
}, "Runtime crashed — scheduling restart with backoff");
|
|
15576
|
-
const uptime = Date.now() - this.lastStartTime;
|
|
15577
15675
|
if (this.detectorFlushTimer) {
|
|
15578
15676
|
clearTimeout(this.detectorFlushTimer);
|
|
15579
15677
|
this.detectorFlushTimer = null;
|
|
@@ -15619,6 +15717,10 @@ var RuntimeProcess = class {
|
|
|
15619
15717
|
stream,
|
|
15620
15718
|
line
|
|
15621
15719
|
});
|
|
15720
|
+
if (!this.sawMigrationInputsChanged) {
|
|
15721
|
+
const lower = line.toLowerCase();
|
|
15722
|
+
if (MIGRATION_RESTART_MARKERS.some((marker) => lower.includes(marker))) this.sawMigrationInputsChanged = true;
|
|
15723
|
+
}
|
|
15622
15724
|
for (const block of this.detector.feed(stream, line)) this.emitErrorBlock(block);
|
|
15623
15725
|
if (this.detectorFlushTimer) {
|
|
15624
15726
|
clearTimeout(this.detectorFlushTimer);
|
|
@@ -15690,6 +15792,14 @@ var RuntimeProcess = class {
|
|
|
15690
15792
|
});
|
|
15691
15793
|
}
|
|
15692
15794
|
/**
|
|
15795
|
+
* Final stop for daemon shutdown: latches so nothing can start the runtime
|
|
15796
|
+
* again (see {@link shutdownLatched}), then stops it.
|
|
15797
|
+
*/
|
|
15798
|
+
async shutdown() {
|
|
15799
|
+
this.shutdownLatched = true;
|
|
15800
|
+
await this.stop();
|
|
15801
|
+
}
|
|
15802
|
+
/**
|
|
15693
15803
|
* Stop the runtime, but first wait for any in-flight chat turn to finish
|
|
15694
15804
|
* (up to a grace deadline). This is the entry point for NON-crash restarts —
|
|
15695
15805
|
* a mid-turn SIGTERM turns a healthy reply into an "unknown error" for the
|
|
@@ -15744,8 +15854,10 @@ var RuntimeProcess = class {
|
|
|
15744
15854
|
* the runtime back after a mutating reconcile branch has finished.
|
|
15745
15855
|
*/
|
|
15746
15856
|
resume() {
|
|
15857
|
+
if (this.shutdownLatched) return;
|
|
15747
15858
|
this.stopped = false;
|
|
15748
15859
|
this.backoffMs = BACKOFF_INITIAL_MS;
|
|
15860
|
+
this.consecutiveMigrationRestarts = 0;
|
|
15749
15861
|
this.start();
|
|
15750
15862
|
}
|
|
15751
15863
|
/**
|
|
@@ -15766,6 +15878,7 @@ var RuntimeProcess = class {
|
|
|
15766
15878
|
else await this.stop();
|
|
15767
15879
|
this.stopped = false;
|
|
15768
15880
|
this.backoffMs = BACKOFF_INITIAL_MS;
|
|
15881
|
+
this.consecutiveMigrationRestarts = 0;
|
|
15769
15882
|
this.start();
|
|
15770
15883
|
}
|
|
15771
15884
|
/**
|
|
@@ -15774,6 +15887,410 @@ var RuntimeProcess = class {
|
|
|
15774
15887
|
get isRunning() {
|
|
15775
15888
|
return this.child !== null && !this.child.killed;
|
|
15776
15889
|
}
|
|
15890
|
+
/**
|
|
15891
|
+
* Whether the runtime is live OR about to be: a child exists, or a restart
|
|
15892
|
+
* timer (crash backoff / graceful / migration restart) is pending and will
|
|
15893
|
+
* spawn one. This — not {@link isRunning} — is what the RuntimeGate must
|
|
15894
|
+
* check before a mutation: during the restart window the child is null but
|
|
15895
|
+
* the pending timer would fire `start()` mid-mutation, putting a second
|
|
15896
|
+
* SQLite writer on ~/.openclaw/state. `stop()` cancels the timer, and the
|
|
15897
|
+
* gate's `resume()` brings the runtime back.
|
|
15898
|
+
*/
|
|
15899
|
+
get isActive() {
|
|
15900
|
+
return !this.stopped && (this.child !== null || this.restartTimer !== null);
|
|
15901
|
+
}
|
|
15777
15902
|
};
|
|
15778
15903
|
//#endregion
|
|
15779
|
-
|
|
15904
|
+
//#region src/runtime-upgrade-recovery.ts
|
|
15905
|
+
/**
|
|
15906
|
+
* Surviving an OS-level restart of the daemon during an OpenClaw upgrade.
|
|
15907
|
+
*
|
|
15908
|
+
* Incident (Dana, prod, 2026-10-02): `apt-daily-upgrade` upgraded libssl and
|
|
15909
|
+
* needrestart restarted `alfe-gateway` while the daemon's `npm install -g
|
|
15910
|
+
* openclaw@…` was mid-reify. systemd SIGKILLed everything in the unit's cgroup,
|
|
15911
|
+
* npm included, leaving a partial `/usr/lib/node_modules/openclaw` (no
|
|
15912
|
+
* `openclaw.mjs`), the intact old tree retired to `.openclaw-<hash>` and the bin
|
|
15913
|
+
* symlink retired to `/usr/bin/.openclaw-<hash>`. The restarted daemon had no
|
|
15914
|
+
* record that an upgrade had been running, so the agent stayed without a
|
|
15915
|
+
* runtime (and still reported `running`).
|
|
15916
|
+
*
|
|
15917
|
+
* Two defences live here:
|
|
15918
|
+
* 1. **Cgroup isolation** — on a systemd host the long, state-mutating
|
|
15919
|
+
* commands (npm install, doctor --fix) run inside a transient scope
|
|
15920
|
+
* (`systemd-run --scope`), so stopping/restarting the daemon's unit no
|
|
15921
|
+
* longer kills them. `--scope` executes the command as the caller's own
|
|
15922
|
+
* child (stdio, exit code and environment are unchanged), it only moves it
|
|
15923
|
+
* into its own cgroup. Off systemd (self-hosted macOS, containers) the
|
|
15924
|
+
* commands run directly, as before.
|
|
15925
|
+
* 2. **Durable marker + boot recovery** — `~/.alfe/runtime-upgrade.json` is
|
|
15926
|
+
* written before the package is replaced and removed when the transaction
|
|
15927
|
+
* finishes. A marker found at boot means the daemon died mid-upgrade:
|
|
15928
|
+
* - phase `installing` (doctor never started, state not migrated): put the
|
|
15929
|
+
* PREVIOUS version back — first by restoring npm's retired
|
|
15930
|
+
* `.openclaw-*` sibling (no network), else `npm install -g
|
|
15931
|
+
* openclaw@<previous>` (or the pin when the previous is unknown);
|
|
15932
|
+
* - phase `migrating` (doctor started; rollback is unsafe): make sure the
|
|
15933
|
+
* TARGET package is installed and re-run the idempotent doctor.
|
|
15934
|
+
* Recovery runs under the exclusive RuntimeGate + CLI lock before the first
|
|
15935
|
+
* runtime start, is bounded to {@link MAX_RECOVERY_ATTEMPTS} boots, and is
|
|
15936
|
+
* reported to Sentry on failure.
|
|
15937
|
+
*/
|
|
15938
|
+
const execFileAsync = promisify(execFile);
|
|
15939
|
+
const SYSTEMD_RUN_PATHS = ["/usr/bin/systemd-run", "/bin/systemd-run"];
|
|
15940
|
+
/**
|
|
15941
|
+
* Whether this daemon runs as a systemd service that can start transient
|
|
15942
|
+
* scopes. `INVOCATION_ID` is set by systemd for every unit it starts, so its
|
|
15943
|
+
* absence means "not under systemd" (an interactive/self-hosted daemon, a
|
|
15944
|
+
* container) and the commands run directly.
|
|
15945
|
+
*/
|
|
15946
|
+
function detectUpgradeIsolation(env = process.env, uid = process.getuid?.(), hasSystemdRun = () => SYSTEMD_RUN_PATHS.some((path) => existsSync(path)), platform = process.platform) {
|
|
15947
|
+
if (platform !== "linux") return void 0;
|
|
15948
|
+
if (!env.INVOCATION_ID || !hasSystemdRun()) return void 0;
|
|
15949
|
+
if (uid === 0) return { manager: "system" };
|
|
15950
|
+
return env.XDG_RUNTIME_DIR ? { manager: "user" } : void 0;
|
|
15951
|
+
}
|
|
15952
|
+
function upgradeScopeUnitName(now = Date.now()) {
|
|
15953
|
+
return `alfe-openclaw-upgrade-${String(now)}`;
|
|
15954
|
+
}
|
|
15955
|
+
/** Wrap a command so it runs in its own transient scope (own cgroup). */
|
|
15956
|
+
function wrapInUpgradeScope(isolation, unit, command, args) {
|
|
15957
|
+
if (!isolation) return {
|
|
15958
|
+
command,
|
|
15959
|
+
args
|
|
15960
|
+
};
|
|
15961
|
+
return {
|
|
15962
|
+
command: "systemd-run",
|
|
15963
|
+
args: [
|
|
15964
|
+
...isolation.manager === "user" ? ["--user"] : [],
|
|
15965
|
+
"--scope",
|
|
15966
|
+
"--quiet",
|
|
15967
|
+
"--collect",
|
|
15968
|
+
`--unit=${unit}`,
|
|
15969
|
+
"--",
|
|
15970
|
+
command,
|
|
15971
|
+
...args
|
|
15972
|
+
]
|
|
15973
|
+
};
|
|
15974
|
+
}
|
|
15975
|
+
function defaultRuntimeUpgradeMarkerPath() {
|
|
15976
|
+
return join(homedir(), ".alfe", "runtime-upgrade.json");
|
|
15977
|
+
}
|
|
15978
|
+
async function writeRuntimeUpgradeMarker(path, marker) {
|
|
15979
|
+
await mkdir(dirname(path), {
|
|
15980
|
+
recursive: true,
|
|
15981
|
+
mode: 448
|
|
15982
|
+
});
|
|
15983
|
+
const tmp = `${path}.${String(process.pid)}.tmp`;
|
|
15984
|
+
await writeFile(tmp, JSON.stringify(marker, null, 2) + "\n", { mode: 384 });
|
|
15985
|
+
await rename(tmp, path);
|
|
15986
|
+
}
|
|
15987
|
+
async function readRuntimeUpgradeMarker(path) {
|
|
15988
|
+
let raw;
|
|
15989
|
+
try {
|
|
15990
|
+
raw = await readFile(path, "utf8");
|
|
15991
|
+
} catch (err) {
|
|
15992
|
+
if (err.code === "ENOENT") return void 0;
|
|
15993
|
+
throw err;
|
|
15994
|
+
}
|
|
15995
|
+
const parsed = JSON.parse(raw);
|
|
15996
|
+
if (parsed?.runtime !== "openclaw" || parsed.phase !== "installing" && parsed.phase !== "migrating" || typeof parsed.targetVersion !== "string") throw new Error(`Runtime upgrade marker ${path} is malformed`);
|
|
15997
|
+
return parsed;
|
|
15998
|
+
}
|
|
15999
|
+
async function clearRuntimeUpgradeMarker(path) {
|
|
16000
|
+
await rm(path, { force: true });
|
|
16001
|
+
}
|
|
16002
|
+
const NPM_INSTALL_TIMEOUT_MS = 3e5;
|
|
16003
|
+
const DOCTOR_TIMEOUT_MS = 9e5;
|
|
16004
|
+
const DOCTOR_MAX_BUFFER_BYTES = 16 * 1024 * 1024;
|
|
16005
|
+
const SCOPE_WAIT_POLL_MS = 5e3;
|
|
16006
|
+
const SCOPE_WAIT_MAX_MS = 20 * 6e4;
|
|
16007
|
+
/**
|
|
16008
|
+
* `doctor --fix` runs the state/config migration. On 2026.9.x `--json` puts
|
|
16009
|
+
* doctor into READ-ONLY lint mode and it refuses the combination outright
|
|
16010
|
+
* (exit 2: "doctor --json runs read-only lint checks and cannot be combined
|
|
16011
|
+
* with --repair, --fix, or --force"), so the migration must run without it.
|
|
16012
|
+
*/
|
|
16013
|
+
const OPENCLAW_DOCTOR_FIX_ARGS = [
|
|
16014
|
+
"doctor",
|
|
16015
|
+
"--fix",
|
|
16016
|
+
"--non-interactive",
|
|
16017
|
+
"--yes"
|
|
16018
|
+
];
|
|
16019
|
+
async function installedOpenClawVersion(run, env) {
|
|
16020
|
+
try {
|
|
16021
|
+
const { stdout } = await run("openclaw", ["--version"], {
|
|
16022
|
+
timeout: 3e4,
|
|
16023
|
+
env
|
|
16024
|
+
});
|
|
16025
|
+
return parseOpenClawVersion(stdout);
|
|
16026
|
+
} catch {
|
|
16027
|
+
return;
|
|
16028
|
+
}
|
|
16029
|
+
}
|
|
16030
|
+
/** Wait for the interrupted upgrade's scopes to finish so recovery never races a still-running npm/doctor. */
|
|
16031
|
+
async function waitForScopes(run, isolation, units, sleep, now) {
|
|
16032
|
+
if (!isolation || units.length === 0) return;
|
|
16033
|
+
const deadline = now() + SCOPE_WAIT_MAX_MS;
|
|
16034
|
+
for (const unit of units) for (;;) {
|
|
16035
|
+
let active = false;
|
|
16036
|
+
try {
|
|
16037
|
+
await run("systemctl", [
|
|
16038
|
+
...isolation.manager === "user" ? ["--user"] : [],
|
|
16039
|
+
"is-active",
|
|
16040
|
+
"--quiet",
|
|
16041
|
+
`${unit}.scope`
|
|
16042
|
+
], { timeout: 1e4 });
|
|
16043
|
+
active = true;
|
|
16044
|
+
} catch {
|
|
16045
|
+
active = false;
|
|
16046
|
+
}
|
|
16047
|
+
if (!active) break;
|
|
16048
|
+
if (now() >= deadline) throw new Error(`upgrade scope ${unit}.scope is still running after ${String(SCOPE_WAIT_MAX_MS / 6e4)} min`);
|
|
16049
|
+
logger$1.info({ unit }, "Interrupted runtime upgrade still running in its scope — waiting");
|
|
16050
|
+
await sleep(SCOPE_WAIT_POLL_MS);
|
|
16051
|
+
}
|
|
16052
|
+
}
|
|
16053
|
+
/**
|
|
16054
|
+
* Put npm's retired `.openclaw-*` tree for `version` back in place, with its
|
|
16055
|
+
* bin link. npm's reify renames the live package to `.<name>-<rand>` before
|
|
16056
|
+
* extracting and only deletes it on success, so a killed reify leaves the old
|
|
16057
|
+
* tree intact next to the partial one. Returns true when `openclaw --version`
|
|
16058
|
+
* then reports `version`.
|
|
16059
|
+
*/
|
|
16060
|
+
async function restoreRetiredOpenClawTree(run, env, version) {
|
|
16061
|
+
const root = (await run("npm", ["root", "-g"], {
|
|
16062
|
+
timeout: 3e4,
|
|
16063
|
+
env
|
|
16064
|
+
})).stdout.trim();
|
|
16065
|
+
const prefix = (await run("npm", ["prefix", "-g"], {
|
|
16066
|
+
timeout: 3e4,
|
|
16067
|
+
env
|
|
16068
|
+
})).stdout.trim();
|
|
16069
|
+
if (!root.startsWith("/") || !prefix.startsWith("/")) return false;
|
|
16070
|
+
let candidate;
|
|
16071
|
+
for (const entry of await readdir(root).catch(() => [])) {
|
|
16072
|
+
if (!entry.startsWith(".openclaw-")) continue;
|
|
16073
|
+
const dir = join(root, entry);
|
|
16074
|
+
try {
|
|
16075
|
+
const pkg = JSON.parse(await readFile(join(dir, "package.json"), "utf8"));
|
|
16076
|
+
if (pkg.name !== "openclaw" || pkg.version !== version) continue;
|
|
16077
|
+
const bin = typeof pkg.bin === "string" ? pkg.bin : pkg.bin?.openclaw;
|
|
16078
|
+
if (typeof bin !== "string" || !existsSync(join(dir, bin))) continue;
|
|
16079
|
+
candidate = {
|
|
16080
|
+
dir,
|
|
16081
|
+
bin
|
|
16082
|
+
};
|
|
16083
|
+
break;
|
|
16084
|
+
} catch {
|
|
16085
|
+
continue;
|
|
16086
|
+
}
|
|
16087
|
+
}
|
|
16088
|
+
if (!candidate) return false;
|
|
16089
|
+
const live = join(root, "openclaw");
|
|
16090
|
+
if (existsSync(live)) {
|
|
16091
|
+
const aside = join(root, `.openclaw-alfe-partial-${String(Date.now())}`);
|
|
16092
|
+
await rename(live, aside);
|
|
16093
|
+
await rm(aside, {
|
|
16094
|
+
recursive: true,
|
|
16095
|
+
force: true
|
|
16096
|
+
});
|
|
16097
|
+
}
|
|
16098
|
+
await rename(candidate.dir, live);
|
|
16099
|
+
const binLink = join(prefix, "bin", "openclaw");
|
|
16100
|
+
await lstat(binLink).then(() => unlink(binLink), () => void 0);
|
|
16101
|
+
await symlink(relative(dirname(binLink), join(live, candidate.bin)), binLink);
|
|
16102
|
+
logger$1.info({
|
|
16103
|
+
restored: candidate.dir,
|
|
16104
|
+
version
|
|
16105
|
+
}, "Restored the retired OpenClaw package tree");
|
|
16106
|
+
return await installedOpenClawVersion(run, env) === version;
|
|
16107
|
+
}
|
|
16108
|
+
/**
|
|
16109
|
+
* Finish or undo an OpenClaw upgrade the daemon did not live to complete.
|
|
16110
|
+
* Never throws; the caller holds the exclusive RuntimeGate and the CLI lock.
|
|
16111
|
+
*/
|
|
16112
|
+
async function recoverInterruptedRuntimeUpgrade(options = {}) {
|
|
16113
|
+
const markerPath = options.markerPath ?? defaultRuntimeUpgradeMarkerPath();
|
|
16114
|
+
const run = options.runCommand ?? (async (command, args, commandOptions) => {
|
|
16115
|
+
const result = await execFileAsync(command, args, commandOptions);
|
|
16116
|
+
return {
|
|
16117
|
+
stdout: result.stdout,
|
|
16118
|
+
stderr: result.stderr
|
|
16119
|
+
};
|
|
16120
|
+
});
|
|
16121
|
+
const env = {
|
|
16122
|
+
...process.env,
|
|
16123
|
+
...OPENCLAW_EXTERNAL_SUPERVISOR_ENV
|
|
16124
|
+
};
|
|
16125
|
+
const sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
16126
|
+
const now = options.now ?? Date.now;
|
|
16127
|
+
let marker;
|
|
16128
|
+
try {
|
|
16129
|
+
marker = await readRuntimeUpgradeMarker(markerPath);
|
|
16130
|
+
} catch (err) {
|
|
16131
|
+
captureCliFailure("openclaw-upgrade-recovery", err);
|
|
16132
|
+
logger$1.error({ err: err instanceof Error ? err.message : String(err) }, "Discarding unreadable runtime upgrade marker");
|
|
16133
|
+
await clearRuntimeUpgradeMarker(markerPath).catch(() => void 0);
|
|
16134
|
+
return { status: "none" };
|
|
16135
|
+
}
|
|
16136
|
+
if (!marker) return { status: "none" };
|
|
16137
|
+
const attempts = (marker.recoveryAttempts ?? 0) + 1;
|
|
16138
|
+
const phase = marker.phase;
|
|
16139
|
+
if (attempts > 3) {
|
|
16140
|
+
const error = `Gave up recovering the interrupted OpenClaw upgrade to ${marker.targetVersion} after ${String(3)} boots`;
|
|
16141
|
+
captureCliFailure("openclaw-upgrade-recovery", new Error(error));
|
|
16142
|
+
logger$1.error({ marker }, error);
|
|
16143
|
+
await clearRuntimeUpgradeMarker(markerPath).catch(() => void 0);
|
|
16144
|
+
return {
|
|
16145
|
+
status: "failed",
|
|
16146
|
+
phase,
|
|
16147
|
+
error,
|
|
16148
|
+
gaveUp: true
|
|
16149
|
+
};
|
|
16150
|
+
}
|
|
16151
|
+
try {
|
|
16152
|
+
await writeRuntimeUpgradeMarker(markerPath, {
|
|
16153
|
+
...marker,
|
|
16154
|
+
recoveryAttempts: attempts
|
|
16155
|
+
});
|
|
16156
|
+
} catch (err) {
|
|
16157
|
+
const error = `Could not record the recovery attempt in ${markerPath}: ${err instanceof Error ? err.message : String(err)}`;
|
|
16158
|
+
captureCliFailure("openclaw-upgrade-recovery", new Error(error, { cause: err }));
|
|
16159
|
+
logger$1.error({ err: error }, "Not recovering the interrupted OpenClaw upgrade — marker is not writable");
|
|
16160
|
+
return {
|
|
16161
|
+
status: "failed",
|
|
16162
|
+
phase,
|
|
16163
|
+
error,
|
|
16164
|
+
gaveUp: false
|
|
16165
|
+
};
|
|
16166
|
+
}
|
|
16167
|
+
logger$1.warn({
|
|
16168
|
+
marker,
|
|
16169
|
+
attempt: attempts
|
|
16170
|
+
}, "Found an interrupted OpenClaw runtime upgrade — recovering before the runtime starts");
|
|
16171
|
+
let scopeSequence = 0;
|
|
16172
|
+
const scoped = (command, args) => {
|
|
16173
|
+
scopeSequence += 1;
|
|
16174
|
+
return wrapInUpgradeScope(options.isolation, `${upgradeScopeUnitName(now())}-recovery-${String(scopeSequence)}`, command, args);
|
|
16175
|
+
};
|
|
16176
|
+
try {
|
|
16177
|
+
await waitForScopes(run, options.isolation, marker.scopeUnits ?? [], sleep, now);
|
|
16178
|
+
const installed = await installedOpenClawVersion(run, env);
|
|
16179
|
+
const target = marker.previousVersion;
|
|
16180
|
+
if (phase === "installing" && target !== void 0 && installed !== marker.targetVersion) {
|
|
16181
|
+
let method = "already-in-place";
|
|
16182
|
+
if (installed !== target) {
|
|
16183
|
+
if (await restoreRetiredOpenClawTree(run, env, target).catch((err) => {
|
|
16184
|
+
logger$1.warn({ err: err instanceof Error ? err.message : String(err) }, "Restoring the retired OpenClaw tree failed — reinstalling");
|
|
16185
|
+
return false;
|
|
16186
|
+
})) method = "restored-retired-tree";
|
|
16187
|
+
else {
|
|
16188
|
+
const install = scoped("npm", [
|
|
16189
|
+
"install",
|
|
16190
|
+
"-g",
|
|
16191
|
+
`openclaw@${target}`
|
|
16192
|
+
]);
|
|
16193
|
+
await run(install.command, install.args, {
|
|
16194
|
+
timeout: NPM_INSTALL_TIMEOUT_MS,
|
|
16195
|
+
env
|
|
16196
|
+
});
|
|
16197
|
+
method = "reinstalled";
|
|
16198
|
+
}
|
|
16199
|
+
const verified = await installedOpenClawVersion(run, env);
|
|
16200
|
+
if (verified !== target) throw new Error(`OpenClaw did not verify after recovery: expected ${target}, resolved ${verified ?? "nothing"}`);
|
|
16201
|
+
}
|
|
16202
|
+
await clearRuntimeUpgradeMarker(markerPath);
|
|
16203
|
+
logger$1.info({
|
|
16204
|
+
version: target,
|
|
16205
|
+
method
|
|
16206
|
+
}, "Recovered interrupted OpenClaw upgrade (pre-migration): previous runtime restored");
|
|
16207
|
+
return {
|
|
16208
|
+
status: "recovered",
|
|
16209
|
+
phase,
|
|
16210
|
+
version: target,
|
|
16211
|
+
method
|
|
16212
|
+
};
|
|
16213
|
+
}
|
|
16214
|
+
let method = phase === "migrating" ? "doctor-rerun" : "rolled-forward";
|
|
16215
|
+
if (installed !== marker.targetVersion) {
|
|
16216
|
+
const install = scoped("npm", [
|
|
16217
|
+
"install",
|
|
16218
|
+
"-g",
|
|
16219
|
+
`openclaw@${marker.targetVersion}`
|
|
16220
|
+
]);
|
|
16221
|
+
await run(install.command, install.args, {
|
|
16222
|
+
timeout: NPM_INSTALL_TIMEOUT_MS,
|
|
16223
|
+
env
|
|
16224
|
+
});
|
|
16225
|
+
const verified = await installedOpenClawVersion(run, env);
|
|
16226
|
+
if (verified !== marker.targetVersion) throw new Error(`OpenClaw did not verify after recovery: expected ${marker.targetVersion}, resolved ${verified ?? "nothing"}`);
|
|
16227
|
+
method = `${method}-after-reinstall`;
|
|
16228
|
+
}
|
|
16229
|
+
const doctor = scoped("openclaw", [...OPENCLAW_DOCTOR_FIX_ARGS]);
|
|
16230
|
+
await run(doctor.command, doctor.args, {
|
|
16231
|
+
timeout: DOCTOR_TIMEOUT_MS,
|
|
16232
|
+
env,
|
|
16233
|
+
maxBuffer: DOCTOR_MAX_BUFFER_BYTES
|
|
16234
|
+
});
|
|
16235
|
+
await clearRuntimeUpgradeMarker(markerPath);
|
|
16236
|
+
logger$1.info({
|
|
16237
|
+
version: marker.targetVersion,
|
|
16238
|
+
method
|
|
16239
|
+
}, "Recovered interrupted OpenClaw upgrade (migration completed)");
|
|
16240
|
+
return {
|
|
16241
|
+
status: "recovered",
|
|
16242
|
+
phase,
|
|
16243
|
+
version: marker.targetVersion,
|
|
16244
|
+
method
|
|
16245
|
+
};
|
|
16246
|
+
} catch (err) {
|
|
16247
|
+
const error = err instanceof Error ? err.message : String(err);
|
|
16248
|
+
captureCliFailure("openclaw-upgrade-recovery", err instanceof Error ? err : new Error(error));
|
|
16249
|
+
logger$1.error({
|
|
16250
|
+
err: error,
|
|
16251
|
+
phase,
|
|
16252
|
+
attempt: attempts
|
|
16253
|
+
}, "Recovering the interrupted OpenClaw upgrade failed — will retry on the next daemon start");
|
|
16254
|
+
return {
|
|
16255
|
+
status: "failed",
|
|
16256
|
+
phase,
|
|
16257
|
+
error,
|
|
16258
|
+
gaveUp: false
|
|
16259
|
+
};
|
|
16260
|
+
}
|
|
16261
|
+
}
|
|
16262
|
+
/**
|
|
16263
|
+
* Start boot-time recovery when a marker exists. Returns `undefined` (and does
|
|
16264
|
+
* nothing) without one. Otherwise the exclusive RuntimeGate hold is RESERVED
|
|
16265
|
+
* SYNCHRONOUSLY before this returns: `RuntimeGate.suspendExclusive` sets its
|
|
16266
|
+
* exclusive flag and depth before its first await, so a `requestStart()` issued
|
|
16267
|
+
* right after this call folds into our `resume()` instead of spawning a runtime
|
|
16268
|
+
* whose binary may be missing. Order: gate, then CLI lock (the documented
|
|
16269
|
+
* lock-ordering invariant). The returned promise never rejects.
|
|
16270
|
+
*/
|
|
16271
|
+
function startBootUpgradeRecovery(options) {
|
|
16272
|
+
if (!existsSync(options.markerPath)) return void 0;
|
|
16273
|
+
const reservation = options.gate.suspendExclusive();
|
|
16274
|
+
return (async () => {
|
|
16275
|
+
try {
|
|
16276
|
+
await reservation;
|
|
16277
|
+
const outcome = await options.cliLock.run(() => options.recover(options.markerPath));
|
|
16278
|
+
logger$1.info({ outcome }, "Interrupted runtime upgrade recovery finished");
|
|
16279
|
+
if (outcome.status === "recovered") options.onRecovered?.(outcome.version);
|
|
16280
|
+
return outcome;
|
|
16281
|
+
} catch (err) {
|
|
16282
|
+
const error = err instanceof Error ? err.message : String(err);
|
|
16283
|
+
logger$1.error({ err: error }, "Interrupted runtime upgrade recovery threw");
|
|
16284
|
+
return {
|
|
16285
|
+
status: "failed",
|
|
16286
|
+
phase: "installing",
|
|
16287
|
+
error,
|
|
16288
|
+
gaveUp: false
|
|
16289
|
+
};
|
|
16290
|
+
} finally {
|
|
16291
|
+
await options.gate.resume();
|
|
16292
|
+
}
|
|
16293
|
+
})();
|
|
16294
|
+
}
|
|
16295
|
+
//#endregion
|
|
16296
|
+
export { resolveOpenClawUpgradeTarget as C, __require as E, parseOpenClawVersion as S, __commonJSMin as T, createLogger as _, recoverInterruptedRuntimeUpgrade as a, PINNED_OPENCLAW_VERSION as b, wrapInUpgradeScope as c, OPENCLAW_EXTERNAL_SUPERVISOR_ENV as d, RuntimeProcess as f, toFingerprintKey as g, STRICT_ERROR_CAPTURE_PROFILE as h, detectUpgradeIsolation as i, writeRuntimeUpgradeMarker as l, ErrorLineDetector as m, clearRuntimeUpgradeMarker as n, startBootUpgradeRecovery as o, CaptureThrottle as p, defaultRuntimeUpgradeMarkerPath as r, upgradeScopeUnitName as s, OPENCLAW_DOCTOR_FIX_ARGS as t, DRAIN_GRACE_MS as u, OPENCLAW_MANAGED_NODE_MAJOR as v, supportsOpenClawNode as w, openClawRequiresRegisterExport as x, OPENCLAW_NODE_REQUIREMENT as y };
|