@senpi/runtime 3.0.180 → 3.0.182

package/dist/runtime/auto-update/coordinator.js

@@ -1,10 +1,13 @@
 import { runCommand } from "./command-runner.js";
-import { evaluateGatewayRestartCapability } from "./gateway-restart.js";
+import { evaluateGatewayRestartCapability, evaluateModernRestartCapability, triggerGatewayRestartViaMutate, } from "./gateway-restart.js";
 import { classifyUpdate, parseSemver } from "./policy.js";
 import { evaluatePreflight } from "./preflight.js";
 import { resolveConfiguredSpec } from "./resolve-package-name.js";
 import { AutoUpdateStateStore } from "./state-store.js";
 import { createDefaultAutoUpdateState, } from "./types.js";
+import { consumeInstallResults, describeInstallResult, installResultLevel, } from "./install-results.js";
+import { buildExactUpdateSpec, clearUpdateMarker, readUpdateMarker, writeUpdateMarker } from "./update-marker.js";
+import { eligibleAt as computeEligibleAt, mustForce } from "./update-schedule.js";
 import { NpmVersionSource } from "./version-source.js";
 import { EVENT_NAME, eventAttrs } from "../../utils/event-catalog.js";
 /**
@@ -22,6 +25,50 @@ const CHECK_NOW_REASON = {
     disabledInConfig: "Auto-update is turned off in config; no check was run.",
     coordinatorStopped: "The auto-update coordinator is not running; no check was run.",
 };
+/**
+ * How long a requested restart may take to load its target before the
+ * awaiting-restart state is treated as failed (the 3.0.171 latch fix). A 10.x
+ * restart-to-apply measured about 71 s on Box A, with starts up to 93 s; 30 min
+ * leaves room for the gateway's own deferral (forced after 300 s) and a slow
+ * offline install.
+ */
+export const RESTART_LOAD_TIMEOUT_MS = 30 * 60_000;
+/**
+ * Restarts requested for one target that may fail to load before the box
+ * stops trying that version and waits for a newer one. Bounds the loop of a
+ * version that can never install (a blocked scan, a yanked version, a wrapper
+ * without the marker step): without it an idle box would restart about every
+ * 80 min forever.
+ */
+export const MAX_APPLY_ATTEMPTS = 3;
+/**
+ * How often a 10.x box that is `pending-idle` re-reads its apply gate (the
+ * spread time and the idle gate) between registry polls. The idle gate has no
+ * change notification, so the change is caught by this re-read; when the gate
+ * opens, an ordinary check runs at once instead of up to one poll interval
+ * (60 min by default) later. The re-read itself is local: no registry call,
+ * no `onTickComplete`.
+ */
+export const PENDING_IDLE_RECHECK_MS = 3 * 60_000;
+/**
+ * The only idle-gate blockers the 24 h forced restart may override: a busy
+ * user (`user-run`, `recent-message`). Every other blocker holds it, money
+ * first: an order call in flight (the restart would cut it), an open trade
+ * approval (it would be cancelled), a running `senpi deploy` (it would leave a
+ * funded wallet with no runtime), a queued user message, and a gate that
+ * threw (it cannot rule any of those out).
+ */
+const FORCE_MAY_OVERRIDE = new Set(["user-run", "recent-message"]);
+/** Whether every blocker in `blockers` is one the forced restart may override. */
+function onlyOverridable(blockers) {
+    return blockers.every((b) => FORCE_MAY_OVERRIDE.has(b));
+}
+/** Whether `loaded` is at or past `target` (major.minor.patch; both must parse). */
+function isAtLeast(loaded, target) {
+    if (!parseSemver(loaded) || !parseSemver(target))
+        return false;
+    return classifyUpdate(loaded, target) === "none";
+}
 /** Cap a registry-supplied string before it reaches a log line or an event body. */
 function summarizeForLog(value, maxLength = 120) {
     const collapsed = value.replace(/\s+/g, " ").trim();
@@ -49,6 +96,8 @@ export class AutoUpdateCoordinator {
     versionSource;
     runUpdateCommand;
     timer = null;
+    /** 10.x `pending-idle` gate re-read; see {@link PENDING_IDLE_RECHECK_MS}. */
+    idleRecheckTimer = null;
     running = false;
     stopped = true;
     runToken = 0;
@@ -58,8 +107,24 @@ export class AutoUpdateCoordinator {
     /** Spec the last resolve used. Seeded in the constructor, so the snapshot
      * reports a line before the first tick. */
     updateSpec;
+    /** Why the modern path is still waiting; reset every tick. See {@link AutoUpdateSnapshot.pendingBlockers}. */
+    pendingBlockers = [];
+    now;
+    rand;
+    loadedVersion;
+    /**
+     * 10.x: `<version>|<sorted blockers>` of the last `auto_update.deferred`, so the event fires on
+     * entering `pending-idle` and on a change only. Cleared when a tick ends outside `pending-idle` or fails.
+     * In memory: a new gateway process reports a wait that is still on once.
+     */
+    lastDeferredKey = null;
+    /** The install-results report the last {@link start} began; see {@link whenInstallResultsReported}. */
+    installResultsReport = Promise.resolve();
     constructor(options) {
         this.options = options;
+        this.now = options.now ?? Date.now;
+        this.rand = options.rand ?? Math.random;
+        this.loadedVersion = options.loadedVersion ?? options.currentVersion;
         this.store = new AutoUpdateStateStore(options.stateDir, options.logger);
         this.versionSource = options.versionSource ?? new NpmVersionSource();
         this.runUpdateCommand = options.runUpdateCommand ?? defaultRunUpdateCommand;
@@ -79,6 +144,8 @@ export class AutoUpdateCoordinator {
             this.stopped = true;
             this.snapshot = this.buildSnapshot("disabled", "auto-update-disabled");
             logger.info(`[senpi-auto-update] disabled by config (pluginEnabled=${String(this.options.enabled)}, configEnabled=${String(config.enabled)})`);
+            // Operator markers and the boot floor still install on a box with auto-update off.
+            this.startInstallResultsReport(null);
             return;
         }
         logger.info(`[senpi-auto-update] starting | localVersion=${this.options.currentVersion} package=${this.options.packageName} spec=${this.resolveUpdateSpec()} mode=${config.mode} pollInterval=${config.pollIntervalMinutes}m startupDelay=${config.startupDelaySeconds}s failureBackoff=${config.failureBackoffMinutes}m maxBackoff=${config.maxBackoffMinutes}m`);
@@ -90,17 +157,188 @@ export class AutoUpdateCoordinator {
         logger.info(`[senpi-auto-update] preflight result | ok=${String(preflight.ok)} effectiveMode=${preflight.effectiveMode} reasons=${preflight.reasons.join(",") || "none"}`);
         this.state = await this.store.load(this.options.currentVersion, preflight.effectiveMode);
         this.state.effectiveMode = preflight.effectiveMode;
+        const reconciled = await this.reconcileLoadedVersion(this.state);
         try {
             await this.store.save(this.state);
         }
         catch (error) {
             this.options.logger.warn("[senpi-auto-update] failed to persist initial state", error);
         }
-        this.snapshot = this.buildSnapshot("healthy", "started");
+        this.startInstallResultsReport(this.state);
+        this.snapshot = this.buildSnapshot("healthy", reconciled ?? "started");
         const delayMs = config.startupDelaySeconds * 1000;
         logger.info(`[senpi-auto-update] first check scheduled | delayMs=${delayMs} nextCheckAt=${new Date(Date.now() + delayMs).toISOString()}`);
         this.scheduleNext(delayMs, this.runToken);
     }
+    /**
+     * Startup half of the latch fix: compare what this process loaded with what
+     * a previous process asked for.
+     *
+     * When the loaded version reaches the requested restart's target, the update
+     * took effect: the pending restart and pending update are cleared, a stale
+     * install marker is removed, and on a 10.x host `auto_update.applied` fires
+     * (nothing was installed before this boot there). On 5.x `applied` already
+     * fired when the restart was requested, so it is not counted twice.
+     *
+     * A target the loaded version has not reached is left for the tick, which
+     * expires it once the request is older than {@link RESTART_LOAD_TIMEOUT_MS}.
+     *
+     * @returns `updated:<v>` when a requested update is now loaded, else `undefined`.
+     */
+    async reconcileLoadedVersion(state) {
+        const loaded = this.loadedVersion;
+        let action;
+        const pending = state.pendingRestart;
+        if (pending && isAtLeast(loaded, pending.toVersion)) {
+            action = `updated:${pending.toVersion}`;
+            this.options.logger.info(`[senpi-auto-update] update to v${pending.toVersion} is loaded (requested ${pending.requestedAt}, mode ${pending.mode})`);
+            // A 5.x pending restart already reported `applied` when it was requested.
+            if (this.options.modernHost && pending.mode === "restart-to-apply") {
+                // How long the apply took: request to this boot (stop + wrapper install + start), and
+                // first seen to this boot (spread + idle wait + apply). Read before pendingUpdate is cleared.
+                const now = this.now();
+                const sinceMs = (iso) => {
+                    const ms = iso ? Date.parse(iso) : Number.NaN;
+                    return Number.isFinite(ms) ? Math.max(0, now - ms) : undefined;
+                };
+                const firstSeenAt = state.pendingUpdate?.firstSeenAt;
+                const pendingMs = sinceMs(firstSeenAt);
+                this.emitEvent(EVENT_NAME.autoUpdateApplied, "info", {
+                    body: `Update applied: ${pending.fromVersion} -> v${pending.toVersion} (${pending.mode})`,
+                    attributes: eventAttrs({
+                        "senpi.auto_update.from_version": pending.fromVersion,
+                        "senpi.auto_update.to_version": pending.toVersion,
+                        "senpi.auto_update.restart_mode": pending.mode,
+                        "senpi.auto_update.requested_at": pending.requestedAt,
+                        "senpi.auto_update.first_seen_at": pendingMs === undefined ? undefined : firstSeenAt,
+                        "senpi.auto_update.apply_ms": sinceMs(pending.requestedAt),
+                        "senpi.auto_update.pending_ms": pendingMs,
+                    }),
+                });
+            }
+            state.pendingRestart = undefined;
+        }
+        if (state.pendingUpdate && isAtLeast(loaded, state.pendingUpdate.version)) {
+            state.pendingUpdate = undefined;
+        }
+        if (state.applyFailures && isAtLeast(loaded, state.applyFailures.version)) {
+            state.applyFailures = undefined;
+        }
+        if (state.lastAppliedVersion && isAtLeast(loaded, state.lastAppliedVersion)) {
+            state.lastLoggedAwaitingRestartVersion = undefined;
+        }
+        const markerPath = this.options.updateMarkerPath;
+        if (markerPath) {
+            const marker = await readUpdateMarker(markerPath);
+            if (marker && isAtLeast(loaded, marker.targetVersion)) {
+                await this.clearMarkerQuietly();
+            }
+        }
+        return action;
+    }
+    /**
+     * Resolves when the install-results report begun by the last {@link start} has finished.
+     * `start` does not wait for it: the report waits for the OTel SDK first.
+     */
+    whenInstallResultsReported() {
+        return this.installResultsReport;
+    }
+    /**
+     * 10.x: start reporting the wrapper's install outcomes in the background. `liveState` is the
+     * running coordinator's state (the ids are saved with it); `null` when auto-update is
+     * disabled, which loads only the ids and patches only them.
+     */
+    startInstallResultsReport(liveState) {
+        if (!this.options.modernHost || !this.options.installResultsPath)
+            return;
+        this.installResultsReport = this.reportInstallResults(this.options.installResultsPath, liveState);
+    }
+    /**
+     * One `auto_update.install_result` per entry not reported before (`install-results.ts`), then
+     * the file is deleted. Runs from {@link start}, which only the gateway's plugin service calls,
+     * so a CLI process never consumes the file. Never throws: telemetry must not break the
+     * coordinator.
+     */
+    async reportInstallResults(path, liveState) {
+        try {
+            const reportedIds = liveState
+                ? liveState.reportedInstallResultIds
+                : (await this.store.load(this.options.currentVersion, this.options.config.mode)).reportedInstallResultIds;
+            const result = await consumeInstallResults({
+                path,
+                reportedIds: Array.isArray(reportedIds) ? reportedIds : [],
+                emit: (entry) => this.emitInstallResult(entry),
+                persistReportedIds: async (ids) => {
+                    if (!liveState)
+                        return this.store.patch({ reportedInstallResultIds: ids });
+                    // The running coordinator saves this same object on every tick; saving it whole
+                    // never writes back an older copy.
+                    liveState.reportedInstallResultIds = ids;
+                    await this.store.save(liveState);
+                },
+                telemetry: this.options.installResultsTelemetry,
+                logger: this.options.logger,
+            });
+            if (result.emitted > 0 || result.skipped > 0) {
+                this.options.logger.info(`[senpi-auto-update] reported ${result.emitted} wrapper install result(s)` +
+                    (result.skipped > 0 ? ` (${result.skipped} already reported)` : ""));
+            }
+        }
+        catch (error) {
+            this.options.logger.warn(`[senpi-auto-update] reading the wrapper's install results failed | err=${formatErr(error)}`);
+        }
+    }
+    emitInstallResult(e) {
+        const opt = (v) => (v === null ? undefined : v);
+        this.emitEvent(EVENT_NAME.autoUpdateInstallResult, installResultLevel(e.outcome), {
+            body: describeInstallResult(e),
+            attributes: eventAttrs({
+                "senpi.auto_update.source": opt(e.source),
+                "senpi.auto_update.outcome": e.outcome,
+                "senpi.auto_update.spec": opt(e.spec),
+                "senpi.auto_update.from_version": opt(e.fromVersion),
+                "senpi.auto_update.to_version": opt(e.toVersion),
+                "senpi.auto_update.target_version": opt(e.targetVersion),
+                "senpi.auto_update.duration_ms": opt(e.durationMs),
+                "senpi.auto_update.reason": opt(e.reason),
+                "senpi.auto_update.ended_at": opt(e.at),
+                "senpi.auto_update.result_id": opt(e.id),
+                "senpi.auto_update.openclaw_version": opt(e.openclawVersion),
+            }),
+        });
+    }
+    /**
+     * 10.x: the box waits (`pending-idle:<v>`). `auto_update.deferred` on entering the wait and when
+     * the target or the sorted blocker set changes; an unchanged re-check or poll is silent.
+     */
+    noteDeferred(pending, latest, blockers) {
+        const sorted = blockerList(blockers);
+        const key = `${latest}|${sorted}`;
+        if (key === this.lastDeferredKey)
+            return;
+        this.lastDeferredKey = key;
+        this.emitEvent(EVENT_NAME.autoUpdateDeferred, "info", {
+            body: `Restart to apply v${latest} deferred: waiting for ${sorted || "nothing"} (eligible at ${pending.eligibleAt})`,
+            attributes: eventAttrs({
+                "senpi.auto_update.to_version": latest,
+                "senpi.auto_update.blockers": sorted,
+                "senpi.auto_update.eligible_at": pending.eligibleAt,
+                "senpi.auto_update.first_seen_at": pending.firstSeenAt,
+            }),
+        });
+    }
+    /** Remove the install marker; a failure is logged, never thrown (the wrapper deletes it too). */
+    async clearMarkerQuietly() {
+        const markerPath = this.options.updateMarkerPath;
+        if (!markerPath)
+            return;
+        try {
+            await clearUpdateMarker(markerPath);
+        }
+        catch (error) {
+            this.options.logger.warn(`[senpi-auto-update] failed to clear the update marker | path=${markerPath} err=${formatErr(error)}`);
+        }
+    }
     /** Cancel the next scheduled poll and prevent in-flight ticks from rescheduling. */
     async stop() {
         this.runToken += 1;
@@ -109,6 +347,7 @@ export class AutoUpdateCoordinator {
             clearTimeout(this.timer);
             this.timer = null;
         }
+        this.clearIdleRecheck();
     }
     /** Return the latest point-in-time snapshot for health/state gateway RPCs. */
     getSnapshot() {
@@ -154,9 +393,7 @@ export class AutoUpdateCoordinator {
         // way to tell a completed check from a failed one is whether it counted a
         // failure. Reporting "ran" either way tells an operator testing a config
         // change that it worked when nothing reached the registry.
-        // Baseline on live state, not the snapshot: `runTick` builds the healthy
-        // snapshot before `tick` zeroes the counter, so after a recovery the
-        // snapshot still carries the old total and 0 -> 1 reads as no change.
+        // Baseline on live state: it is what `tick` changes.
         const failuresBefore = this.state.consecutiveFailures;
         await this.tick(this.runToken);
         if (this.state.consecutiveFailures > failuresBefore) {
@@ -194,15 +431,24 @@ export class AutoUpdateCoordinator {
             return;
         }
         this.running = true;
+        this.clearIdleRecheck();
         this.options.logger.info(`[senpi-auto-update] tick started | localVersion=${this.options.currentVersion}`);
         const state = this.state;
         let nextDelayMs = null;
+        let recheckIdle = false;
         try {
             await this.runTick(state);
             state.consecutiveFailures = 0;
             state.lastFailureAt = undefined;
             state.lastFailureReason = undefined;
             await this.persistState(state);
+            // `runTick` built the snapshot before the reset above; rebuild it so it
+            // carries the state that was just written.
+            this.snapshot = this.buildSnapshot(this.snapshot.health, this.snapshot.lastAction, this.snapshot.lastError);
+            recheckIdle = this.options.modernHost === true && this.snapshot.lastAction.startsWith("pending-idle:");
+            // The wait ended (restart requested, no update, a restart failure): the next wait is new.
+            if (!recheckIdle)
+                this.lastDeferredKey = null;
             nextDelayMs = this.options.config.pollIntervalMinutes * 60_000;
             this.options.logger.info(`[senpi-auto-update] tick succeeded | nextDelayMs=${nextDelayMs}`);
         }
@@ -211,6 +457,8 @@ export class AutoUpdateCoordinator {
             state.lastFailureAt = new Date().toISOString();
             state.lastFailureReason = formatErr(error);
             await this.persistState(state);
+            // A failed tick ends the wait it was in: the next wait is reported as new.
+            this.lastDeferredKey = null;
             const base = this.options.config.failureBackoffMinutes * 60_000;
             const max = this.options.config.maxBackoffMinutes * 60_000;
             const backoff = Math.min(max, base * Math.max(1, state.consecutiveFailures));
@@ -247,8 +495,64 @@ export class AutoUpdateCoordinator {
                 this.options.logger.info(`[senpi-auto-update] scheduling next tick after onTickComplete | nextDelayMs=${nextDelayMs}`);
                 this.scheduleNext(nextDelayMs, token);
             }
+            if (recheckIdle)
+                this.scheduleIdleRecheck(token);
         }
     }
+    /** Re-read the apply gate of a `pending-idle` box after {@link PENDING_IDLE_RECHECK_MS}. */
+    scheduleIdleRecheck(token) {
+        if (this.stopped || token !== this.runToken)
+            return;
+        this.clearIdleRecheck();
+        this.idleRecheckTimer = setTimeout(() => {
+            this.idleRecheckTimer = null;
+            this.recheckPendingIdle(token);
+        }, PENDING_IDLE_RECHECK_MS);
+    }
+    clearIdleRecheck() {
+        if (this.idleRecheckTimer) {
+            clearTimeout(this.idleRecheckTimer);
+            this.idleRecheckTimer = null;
+        }
+    }
+    /**
+     * Run a check now when the gate that held the pending update has opened,
+     * else re-read it again later. A check already in flight re-arms this
+     * itself when it ends `pending-idle`.
+     */
+    recheckPendingIdle(token) {
+        if (this.running || this.stopped || token !== this.runToken)
+            return;
+        const pending = this.state.pendingUpdate;
+        if (!pending)
+            return;
+        const now = this.now();
+        const gate = this.applyGate(now, Date.parse(pending.eligibleAt));
+        if (!gate.open) {
+            // Still waiting; a blocker set that changed since the last poll is a new deferred.
+            this.noteDeferred(pending, pending.version, gate.blockers);
+            this.scheduleIdleRecheck(token);
+            return;
+        }
+        this.options.logger.info(`[senpi-auto-update] pending v${pending.version}: apply gate open; checking now`);
+        void this.tick(token);
+    }
+    /**
+     * The 10.x apply gate: wait for `eligibleAt` (blocker `spread`), then for an
+     * idle box, unless {@link mustForce} applies and every blocker is one the
+     * forced restart may override.
+     */
+    applyGate(now, eligibleMs) {
+        if (Number.isFinite(eligibleMs) && now < eligibleMs) {
+            return { open: false, forced: false, blockers: ["spread"] };
+        }
+        const decision = this.readIdle(now);
+        // Forcing overrides a busy user, never money in motion (FORCE_MAY_OVERRIDE).
+        const forced = !decision.idle &&
+            onlyOverridable(decision.blockers) &&
+            mustForce(now, Number.isFinite(eligibleMs) ? eligibleMs : now);
+        return { open: decision.idle || forced, forced, blockers: [...decision.blockers] };
+    }
     /**
      * Emit a catalogued `auto_update.*` event, guarded so neither a throwing sink nor a bad
      * attribute shape can disturb the tick — telemetry must never break auto-update itself.
@@ -274,6 +578,7 @@ export class AutoUpdateCoordinator {
     /** Core update logic: fetch latest version, classify, and take action. */
     async runTick(state) {
         const { logger } = this.options;
+        this.pendingBlockers = [];
         // Stamped before the tag is resolved so a rejected tag reads as "checked
         // just now and failed", not as "stopped polling a while ago".
         state.lastCheckedAt = new Date().toISOString();
@@ -324,29 +629,311 @@ export class AutoUpdateCoordinator {
                 "senpi.auto_update.will_update": willUpdate,
             }),
         });
+        // A pending 10.x update lives only while an applicable update is seen, so
+        // a dist-tag rollback or a mode change does not leave a stale eligibility
+        // time that would let the next release skip the spread.
         if (updateType === "none") {
+            state.pendingUpdate = undefined;
             this.snapshot = this.buildSnapshot("healthy", `no-update:${latest}`);
             return;
         }
         if (updateType === "major") {
+            state.pendingUpdate = undefined;
             await this.handleMajorAvailable(state, latest, spec);
             return;
         }
         if (!this.isAllowedByPolicy(updateType)) {
+            state.pendingUpdate = undefined;
             logger.info(`[senpi-auto-update] update blocked by policy | updateType=${updateType} remoteVersion=${latest}`);
             this.snapshot = this.buildSnapshot("healthy", `policy-blocked:${updateType}:${latest}`);
             return;
         }
         if (state.effectiveMode !== "auto-apply") {
+            state.pendingUpdate = undefined;
             await this.logUpdateAvailable(state, latest, updateType, spec);
             return;
         }
-        if (state.lastAppliedVersion === latest) {
-            this.handleAlreadyAppliedAwaitingRestart(state, latest);
+        if (state.applyFailures && state.applyFailures.version !== latest) {
+            // A newer release after failed applies of an older one starts a new
+            // episode with a fresh spread wait. Keeping the old `eligibleAt` (long
+            // past) would restart every such box on its first idle tick, and a
+            // fleet-wide apply failure makes that the whole fleet at once.
+            state.applyFailures = undefined;
+            state.pendingUpdate = undefined;
+        }
+        // The latch fix: a persisted target equal to `latest` means a restart was
+        // requested for it and the loaded version is still behind (classifyUpdate
+        // above said so). That is trusted only while the request is recent.
+        const target = state.pendingRestart?.toVersion ?? state.lastAppliedVersion;
+        if (target === latest) {
+            if (this.restartStillPending(state) || this.awaitingManualRestart(state)) {
+                this.handleAlreadyAppliedAwaitingRestart(state, latest);
+                return;
+            }
+            await this.expireUnloadedUpdate(state, latest);
+        }
+        if ((state.applyFailures?.count ?? 0) >= MAX_APPLY_ATTEMPTS) {
+            this.handleParkedUpdate(state, latest);
+            return;
+        }
+        if (this.options.modernHost) {
+            await this.runRestartToApply(state, latest, spec);
             return;
         }
         await this.runUpdateTransaction(state, latest, updateType, spec);
     }
+    /**
+     * Whether the restart requested for the persisted target may still land:
+     * its request time (the pending restart's, else the last restart trigger,
+     * else the last install) is under {@link RESTART_LOAD_TIMEOUT_MS} old. With
+     * no time on record the persisted version is not trusted at all.
+     */
+    restartStillPending(state) {
+        // The latest of the recorded times: an older trigger for a previous
+        // version must not expire a fresh install.
+        const times = [state.pendingRestart?.requestedAt, state.lastRestartTriggerAt, state.lastSuccessAt]
+            .map((iso) => (iso ? Date.parse(iso) : Number.NaN))
+            .filter((ms) => Number.isFinite(ms));
+        if (times.length === 0)
+            return false;
+        return this.now() - Math.max(...times) < RESTART_LOAD_TIMEOUT_MS;
+    }
+    /**
+     * The 5.x latch, as before R2: once this flow installed `latest` (an
+     * install or restart time is on record), the box waits for a restart and
+     * never runs the in-process install and config-bump restart for that
+     * version again, however long it waits. 5.x has no idle gate, so a retry
+     * would restart the gateway over an order call or an open approval. A
+     * restart trigger that threw is latched too (the warning asks for a manual
+     * restart). A persisted applied version with no time at all is still not
+     * trusted. 10.x expires its wait after 30 min instead: its retries are
+     * idle-gated.
+     */
+    awaitingManualRestart(state) {
+        if (this.options.modernHost)
+            return false;
+        return [state.pendingRestart?.requestedAt, state.lastRestartTriggerAt, state.lastSuccessAt].some((iso) => typeof iso === "string" && Number.isFinite(Date.parse(iso)));
+    }
+    /**
+     * {@link MAX_APPLY_ATTEMPTS} restarts for `latest` did not load it. Stop
+     * trying this version: no install and no restart until a newer one ships
+     * (or a restart loads it some other way). Reported, not thrown, so the poll
+     * keeps its normal cadence.
+     */
+    handleParkedUpdate(state, latest) {
+        const count = state.applyFailures?.count ?? 0;
+        const message = `update to v${latest} did not load after ${count} restarts; not retrying this version. ` +
+            `The gateway stays on v${this.loadedVersion} until a newer release ships or an operator installs it`;
+        if (state.lastLoggedAwaitingRestartVersion !== `parked:${latest}`) {
+            this.options.logger.warn(`[senpi-auto-update] ${message}`);
+            state.lastLoggedAwaitingRestartVersion = `parked:${latest}`;
+        }
+        this.snapshot = this.buildSnapshot("degraded", `update-not-loaded:${latest}`, message);
+    }
+    /**
+     * The requested update did not load. Drop the awaiting-restart state and the
+     * marker, report `auto_update.apply_failed`, and fail this tick (so the
+     * backoff applies). The next tick starts over: a new install and restart on
+     * 5.x, a new idle-gated restart request on 10.x. The pending update keeps
+     * its eligibility time, so the box does not wait out its spread again.
+     *
+     * @throws Always.
+     */
+    async expireUnloadedUpdate(state, target) {
+        const loaded = this.loadedVersion;
+        const requestedAt = state.pendingRestart?.requestedAt ?? state.lastRestartTriggerAt ?? "never";
+        const message = `update to v${target} did not load: the gateway still runs v${loaded} ` +
+            `(restart requested ${requestedAt}); retrying from the start`;
+        this.emitEvent(EVENT_NAME.autoUpdateApplyFailed, "warn", {
+            body: `Update to v${target} did not load; loaded v${loaded}`,
+            attributes: eventAttrs({
+                "senpi.auto_update.to_version": target,
+                "senpi.auto_update.loaded_version": loaded,
+            }),
+        });
+        this.options.logger.warn(`[senpi-auto-update] ${message}`);
+        state.lastAppliedVersion = undefined;
+        state.pendingRestart = undefined;
+        state.lastLoggedAwaitingRestartVersion = undefined;
+        state.applyFailures = {
+            version: target,
+            count: (state.applyFailures?.version === target ? state.applyFailures.count : 0) + 1,
+        };
+        await this.clearMarkerQuietly();
+        this.snapshot = this.buildSnapshot("degraded", `update-not-loaded:${target}`, message);
+        throw new Error(message);
+    }
+    /**
+     * The OpenClaw 10.x apply path (restart-to-apply). Nothing is installed in
+     * this process: 10.10 refuses `plugins update` while the gateway runs.
+     *
+     * 1. Record the pending update once per episode (`firstSeenAt`,
+     *    `eligibleAt` = first seen + spread offset + jitter).
+     * 2. Before `eligibleAt`, wait (`pending-idle:<v>`, blocker `spread`). After
+     *    it, continue only when the idle gate says idle, or when
+     *    {@link mustForce} (24 h past eligibility, inside 03:00–05:00 UTC).
+     * 3. Build the exact `<name>@<version>` spec, check restart capability,
+     *    write the marker, and request the restart through `mutateConfigFile`.
+     * 4. A failure at any step clears the marker, reports
+     *    `auto_update.restart_failed` and fails the tick (backoff; `checkNow`
+     *    answers `failed`). Waiting for the spread or an idle box is not a failure.
+     */
+    async runRestartToApply(state, latest, spec) {
+        const { logger } = this.options;
+        const now = this.now();
+        if (!state.pendingUpdate) {
+            const firstSeenAt = new Date(now).toISOString();
+            const eligible = computeEligibleAt(now, this.options.mid ?? "", this.rand);
+            state.pendingUpdate = { version: latest, firstSeenAt, eligibleAt: new Date(eligible).toISOString() };
+            logger.info(`[senpi-auto-update] v${latest} seen; restart-to-apply eligible at ${state.pendingUpdate.eligibleAt}`);
+        }
+        else if (state.pendingUpdate.version !== latest) {
+            // A newer release during the wait keeps the box's place in the spread.
+            state.pendingUpdate = { ...state.pendingUpdate, version: latest };
+        }
+        const gate = this.applyGate(now, Date.parse(state.pendingUpdate.eligibleAt));
+        const forced = gate.forced;
+        if (!gate.open) {
+            this.pendingBlockers = gate.blockers;
+            this.snapshot = this.buildSnapshot("healthy", `pending-idle:${latest}`);
+            this.noteDeferred(state.pendingUpdate, latest, gate.blockers);
+            return;
+        }
+        let exactSpec;
+        try {
+            exactSpec = buildExactUpdateSpec(spec, latest);
+        }
+        catch (error) {
+            await this.reportModernRestartFailure(latest, formatErr(error), `restart-unavailable:${latest}`);
+            return;
+        }
+        const capability = evaluateModernRestartCapability({
+            readLiveConfig: this.options.readLiveConfig,
+            mutateConfigFile: this.options.mutateConfigFile,
+            pluginId: this.options.pluginId,
+        });
+        state.restartCapability = capability.capable ? "capable" : "not-capable";
+        state.restartCapabilityReasons = capability.reasons;
+        state.restartCapabilityMode = capability.mode;
+        const mutateConfigFile = this.options.mutateConfigFile;
+        const markerPath = this.options.updateMarkerPath;
+        if (!capability.capable || !mutateConfigFile) {
+            await this.reportModernRestartFailure(latest, capability.reasons.join(", ") || "unknown", `restart-unavailable:${latest}`);
+            return;
+        }
+        if (!markerPath) {
+            state.restartCapability = "not-capable";
+            state.restartCapabilityReasons = [...capability.reasons, "update-marker-path-unavailable"];
+            await this.reportModernRestartFailure(latest, "update-marker-path-unavailable", `restart-unavailable:${latest}`);
+            return;
+        }
+        const requestedAt = new Date(now).toISOString();
+        state.lastAttemptAt = requestedAt;
+        // The pending restart is saved before the restart is requested: the
+        // gateway may stop this process right after, and a lost pending restart
+        // would never count toward the retry cap. Rolled back if the request fails.
+        const before = {
+            pendingRestart: state.pendingRestart,
+            lastRestartTriggerAt: state.lastRestartTriggerAt,
+            lastRestartTriggerMode: state.lastRestartTriggerMode,
+        };
+        state.pendingRestart = {
+            fromVersion: this.loadedVersion,
+            toVersion: latest,
+            requestedAt,
+            mode: "restart-to-apply",
+        };
+        state.lastRestartTriggerAt = requestedAt;
+        state.lastRestartTriggerMode = "restart-to-apply";
+        const rollBack = () => {
+            state.pendingRestart = before.pendingRestart;
+            state.lastRestartTriggerAt = before.lastRestartTriggerAt;
+            state.lastRestartTriggerMode = before.lastRestartTriggerMode;
+        };
+        try {
+            await this.store.save(state);
+        }
+        catch (error) {
+            // Not a restart-capability problem: the state dir is not writable. No
+            // restart without a saved pending restart, or the retry cap is lost.
+            rollBack();
+            await this.reportModernRestartFailure(latest, `state-save-failed:${formatErr(error)}`, `state-save-failed:${latest}`, error);
+            return;
+        }
+        try {
+            await writeUpdateMarker(markerPath, { spec: exactSpec, targetVersion: latest, requestedAt });
+            // Read the gate again after the awaits above: a money call or a user
+            // turn that started meanwhile holds the restart, as at the first read.
+            const again = this.readIdle(this.now());
+            if (!again.idle && !(forced && onlyOverridable(again.blockers))) {
+                rollBack();
+                await this.clearMarkerQuietly();
+                this.pendingBlockers = [...again.blockers];
+                this.snapshot = this.buildSnapshot("healthy", `pending-idle:${latest}`);
+                if (state.pendingUpdate)
+                    this.noteDeferred(state.pendingUpdate, latest, again.blockers);
+                return;
+            }
+            await triggerGatewayRestartViaMutate({ mutateConfigFile, pluginId: this.options.pluginId, triggerValue: requestedAt });
+        }
+        catch (error) {
+            rollBack();
+            const reason = formatErr(error);
+            state.restartCapability = "not-capable";
+            state.restartCapabilityReasons = [...state.restartCapabilityReasons, `restart-trigger-failed:${reason}`];
+            await this.reportModernRestartFailure(latest, reason, `restart-request-failed:${latest}`, error);
+            return;
+        }
+        this.emitEvent(EVENT_NAME.autoUpdateRestartRequested, "info", {
+            body: `Restart requested to apply v${latest} (from v${this.loadedVersion}${forced ? ", forced after 24 h busy" : ""})`,
+            attributes: eventAttrs({
+                "senpi.auto_update.from_version": this.loadedVersion,
+                "senpi.auto_update.to_version": latest,
+                "senpi.auto_update.restart_mode": "restart-to-apply",
+                "senpi.auto_update.forced": forced,
+                "senpi.auto_update.blockers": forced ? blockerList(gate.blockers) : undefined,
+            }),
+        });
+        logger.info(`[senpi-auto-update] marker written (${exactSpec}); gateway restart requested to apply v${latest}` +
+            (forced ? ` (forced: busy for 24 h past eligibility; blockers ${gate.blockers.join(",")})` : ""));
+        this.snapshot = this.buildSnapshot("healthy", `updated-restart-requested:${latest}`);
+    }
+    /** The idle gate's answer; a throwing gate counts as busy. */
+    readIdle(now) {
+        const gate = this.options.idleGate;
+        if (!gate)
+            return { idle: true, blockers: [] };
+        try {
+            return gate.isIdle(now);
+        }
+        catch (error) {
+            this.options.logger.warn(`[senpi-auto-update] idle gate threw; treating the box as busy | err=${formatErr(error)}`);
+            return { idle: false, blockers: ["idle-gate-error"] };
+        }
+    }
+    /**
+     * Clear the marker, report `auto_update.restart_failed` for the 10.x path
+     * (snapshot degraded), and fail the tick: no restart was requested, so the
+     * tick must not count as a success. The throw makes `tick()` back off and
+     * `checkNow()` answer `failed`.
+     *
+     * @throws Always, with `reason`.
+     */
+    async reportModernRestartFailure(latest, reason, lastAction, error) {
+        await this.clearMarkerQuietly();
+        this.emitEvent(EVENT_NAME.autoUpdateRestartFailed, "error", {
+            body: `Restart to apply v${latest} was not requested: ${reason}`,
+            attributes: eventAttrs({
+                "senpi.auto_update.to_version": latest,
+                "senpi.auto_update.reason": reason,
+            }),
+            ...(error === undefined ? {} : { error }),
+        });
+        this.options.logger.warn(`[senpi-auto-update] v${latest} is available but the gateway restart that applies it was not requested. ` +
+            `Nothing was installed; the loaded version stays v${this.loadedVersion}. Reason(s): ${reason}`);
+        this.snapshot = this.buildSnapshot("degraded", lastAction, reason);
+        throw new Error(`restart to apply v${latest} was not requested: ${reason}`);
+    }
     /**
      * `currentVersion` is a compile-time constant frozen for the life of the
      * process, so it cannot reflect an install that already happened this
@@ -359,19 +946,30 @@ export class AutoUpdateCoordinator {
      * so this branch stops being reached on its own.
      */
     handleAlreadyAppliedAwaitingRestart(state, latest) {
+        const modern = this.options.modernHost === true;
         if (state.lastLoggedAwaitingRestartVersion !== latest) {
-            this.options.logger.info(`[senpi-auto-update] update to v${latest} already applied, awaiting gateway restart to take effect | skipping reinstall`);
+            this.options.logger.info(modern
+                ? `[senpi-auto-update] restart to apply v${latest} requested, awaiting gateway restart | not requesting again`
+                : `[senpi-auto-update] update to v${latest} already applied, awaiting gateway restart to take effect | skipping reinstall`);
             state.lastLoggedAwaitingRestartVersion = latest;
         }
         // Fired on every guard hit (not deduped like the log line above) — an operator watching for
         // "how long has this box been stuck awaiting restart" needs every occurrence, not just the first.
         this.emitEvent(EVENT_NAME.autoUpdateAwaitingRestart, "warn", {
-            body: `Update to v${latest} already applied, still awaiting gateway restart`,
+            body: modern
+                ? `Restart to apply v${latest} requested, still awaiting gateway restart`
+                : `Update to v${latest} already applied, still awaiting gateway restart`,
             attributes: eventAttrs({
                 "senpi.auto_update.to_version": latest,
             }),
         });
-        this.snapshot = this.buildSnapshot("degraded", `already-applied-awaiting-restart:${latest}`);
+        // On 10.x nothing is installed until the wrapper consumes the marker, so
+        // "already applied" would be false there.
+        this.snapshot = this.buildSnapshot("degraded", modern ? `awaiting-restart:${latest}` : `already-applied-awaiting-restart:${latest}`);
+    }
+    /** OpenClaw 10.x refuses `plugins update` while the gateway runs, so a manual instruction says so. */
+    manualUpdateSuffix() {
+        return this.options.modernHost ? " with the gateway stopped" : "";
     }
     isAllowedByPolicy(kind) {
         if (kind === "patch")
@@ -394,7 +992,7 @@ export class AutoUpdateCoordinator {
         });
         this.options.logger.info(`[senpi-auto-update] Senpi update available: v${latest} (major). ` +
             `Auto-update will not apply major versions. ` +
-            `Review changelog and run: openclaw plugins update ${spec}`);
+            `Review changelog and run: openclaw plugins update ${spec}${this.manualUpdateSuffix()}`);
         state.lastNotifiedMajorVersion = latest;
         this.snapshot = this.buildSnapshot("healthy", `major-notified:${latest}`);
     }
@@ -412,7 +1010,7 @@ export class AutoUpdateCoordinator {
         });
         this.options.logger.info(`[senpi-auto-update] Senpi update available: v${latest} (${kind}). ` +
             `Auto-update mode is notify-only. ` +
-            `Run: openclaw plugins update ${spec}`);
+            `Run: openclaw plugins update ${spec}${this.manualUpdateSuffix()}`);
         state.lastNotifiedUpdateVersion = latest;
         this.snapshot = this.buildSnapshot("healthy", `update-notified:${kind}:${latest}`);
     }
@@ -495,6 +1093,12 @@ export class AutoUpdateCoordinator {
             });
             state.lastRestartTriggerAt = nowIso;
             state.lastRestartTriggerMode = capability.mode ?? "unknown";
+            state.pendingRestart = {
+                fromVersion: this.loadedVersion,
+                toVersion: latest,
+                requestedAt: nowIso,
+                mode: state.lastRestartTriggerMode,
+            };
             this.emitEvent(EVENT_NAME.autoUpdateApplied, "info", {
                 body: `Update applied: ${this.options.currentVersion} -> v${latest}; restart requested (${state.lastRestartTriggerMode})`,
                 attributes: eventAttrs({
@@ -553,6 +1157,7 @@ export class AutoUpdateCoordinator {
             restartCapabilityReasons: state.restartCapabilityReasons,
             lastRestartTriggerAt: state.lastRestartTriggerAt ?? null,
             lastRestartTriggerMode: state.lastRestartTriggerMode ?? null,
+            pendingBlockers: this.pendingBlockers,
         };
     }
     async persistState(state) {
@@ -573,6 +1178,10 @@ async function defaultRunUpdateCommand(packageName) {
         throw new Error(result.stderr || `openclaw plugins update failed with code ${result.code}`);
     }
 }
+/** A blocker set as one stable value: de-duplicated, sorted, comma-joined. */
+function blockerList(blockers) {
+    return [...new Set(blockers)].sort().join(",");
+}
 function formatErr(error) {
     return error instanceof Error ? error.message : String(error);
 }