@senpi-ai/runtime 3.0.178 → 3.0.188
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actions/close-position/close-position.action.d.ts.map +1 -1
- package/dist/actions/close-position/close-position.action.js +6 -1
- package/dist/actions/close-position/close-position.action.js.map +1 -1
- package/dist/actions/llm-decision/openclaw-llm-decision.d.ts +8 -0
- package/dist/actions/llm-decision/openclaw-llm-decision.d.ts.map +1 -1
- package/dist/actions/llm-decision/openclaw-llm-decision.js +56 -1
- package/dist/actions/llm-decision/openclaw-llm-decision.js.map +1 -1
- package/dist/actions/position-tracker/position-tracker.action.d.ts.map +1 -1
- package/dist/actions/position-tracker/position-tracker.action.js +1 -0
- package/dist/actions/position-tracker/position-tracker.action.js.map +1 -1
- package/dist/approvals/approval-store.d.ts +37 -7
- package/dist/approvals/approval-store.d.ts.map +1 -1
- package/dist/approvals/approval-store.js +127 -23
- package/dist/approvals/approval-store.js.map +1 -1
- package/dist/approvals/telemetry.d.ts +28 -3
- package/dist/approvals/telemetry.d.ts.map +1 -1
- package/dist/approvals/telemetry.js +8 -4
- package/dist/approvals/telemetry.js.map +1 -1
- package/dist/approvals/tool-approval-gate.d.ts +85 -0
- package/dist/approvals/tool-approval-gate.d.ts.map +1 -1
- package/dist/approvals/tool-approval-gate.js +197 -15
- package/dist/approvals/tool-approval-gate.js.map +1 -1
- package/dist/approvals/turn-context.d.ts +296 -0
- package/dist/approvals/turn-context.d.ts.map +1 -0
- package/dist/approvals/turn-context.js +594 -0
- package/dist/approvals/turn-context.js.map +1 -0
- package/dist/cli/guide-commands.d.ts.map +1 -1
- package/dist/cli/guide-commands.js +18 -3
- package/dist/cli/guide-commands.js.map +1 -1
- package/dist/cli/senpi-commands.d.ts.map +1 -1
- package/dist/cli/senpi-commands.js +80 -5
- package/dist/cli/senpi-commands.js.map +1 -1
- package/dist/deploy/job.d.ts +32 -0
- package/dist/deploy/job.d.ts.map +1 -1
- package/dist/deploy/job.js +72 -7
- package/dist/deploy/job.js.map +1 -1
- package/dist/deploy/register.d.ts +12 -0
- package/dist/deploy/register.d.ts.map +1 -1
- package/dist/deploy/register.js +85 -2
- package/dist/deploy/register.js.map +1 -1
- package/dist/dsl/events/build-closed-event.d.ts +11 -0
- package/dist/dsl/events/build-closed-event.d.ts.map +1 -1
- package/dist/dsl/events/build-closed-event.js +13 -0
- package/dist/dsl/events/build-closed-event.js.map +1 -1
- package/dist/dsl/events/handlers.d.ts.map +1 -1
- package/dist/dsl/events/handlers.js +9 -5
- package/dist/dsl/events/handlers.js.map +1 -1
- package/dist/dsl/monitor/dsl-failure-events.d.ts +38 -2
- package/dist/dsl/monitor/dsl-failure-events.d.ts.map +1 -1
- package/dist/dsl/monitor/dsl-failure-events.js +67 -4
- package/dist/dsl/monitor/dsl-failure-events.js.map +1 -1
- package/dist/dsl/monitor/process-one-position.d.ts.map +1 -1
- package/dist/dsl/monitor/process-one-position.js +37 -9
- package/dist/dsl/monitor/process-one-position.js.map +1 -1
- package/dist/host/openclaw-host.d.ts +138 -0
- package/dist/host/openclaw-host.d.ts.map +1 -0
- package/dist/host/openclaw-host.js +177 -0
- package/dist/host/openclaw-host.js.map +1 -0
- package/dist/host/plugin-logger.d.ts +99 -0
- package/dist/host/plugin-logger.d.ts.map +1 -0
- package/dist/host/plugin-logger.js +142 -0
- package/dist/host/plugin-logger.js.map +1 -0
- package/dist/index.d.ts +57 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +719 -162
- package/dist/index.js.map +1 -1
- package/dist/prompt/session-context-seed-hook.d.ts +7 -0
- package/dist/prompt/session-context-seed-hook.d.ts.map +1 -1
- package/dist/prompt/session-context-seed-hook.js +1 -1
- package/dist/prompt/session-context-seed-hook.js.map +1 -1
- package/dist/prompt/skill-router-hook.d.ts +24 -0
- package/dist/prompt/skill-router-hook.d.ts.map +1 -0
- package/dist/prompt/skill-router-hook.js +381 -0
- package/dist/prompt/skill-router-hook.js.map +1 -0
- package/dist/runtime/auto-update/coordinator.d.ts +220 -0
- package/dist/runtime/auto-update/coordinator.d.ts.map +1 -1
- package/dist/runtime/auto-update/coordinator.js +715 -22
- package/dist/runtime/auto-update/coordinator.js.map +1 -1
- package/dist/runtime/auto-update/gateway-restart.d.ts +60 -1
- package/dist/runtime/auto-update/gateway-restart.d.ts.map +1 -1
- package/dist/runtime/auto-update/gateway-restart.js +95 -21
- package/dist/runtime/auto-update/gateway-restart.js.map +1 -1
- package/dist/runtime/auto-update/idle-gate.d.ts +175 -0
- package/dist/runtime/auto-update/idle-gate.d.ts.map +1 -0
- package/dist/runtime/auto-update/idle-gate.js +229 -0
- package/dist/runtime/auto-update/idle-gate.js.map +1 -0
- package/dist/runtime/auto-update/install-results.d.ts +130 -0
- package/dist/runtime/auto-update/install-results.d.ts.map +1 -0
- package/dist/runtime/auto-update/install-results.js +264 -0
- package/dist/runtime/auto-update/install-results.js.map +1 -0
- package/dist/runtime/auto-update/money-call-tracker.d.ts +36 -0
- package/dist/runtime/auto-update/money-call-tracker.d.ts.map +1 -0
- package/dist/runtime/auto-update/money-call-tracker.js +129 -0
- package/dist/runtime/auto-update/money-call-tracker.js.map +1 -0
- package/dist/runtime/auto-update/register-idle-gate.d.ts +70 -0
- package/dist/runtime/auto-update/register-idle-gate.d.ts.map +1 -0
- package/dist/runtime/auto-update/register-idle-gate.js +127 -0
- package/dist/runtime/auto-update/register-idle-gate.js.map +1 -0
- package/dist/runtime/auto-update/run-update-command.d.ts +13 -0
- package/dist/runtime/auto-update/run-update-command.d.ts.map +1 -1
- package/dist/runtime/auto-update/run-update-command.js +18 -0
- package/dist/runtime/auto-update/run-update-command.js.map +1 -1
- package/dist/runtime/auto-update/state-store.d.ts +14 -1
- package/dist/runtime/auto-update/state-store.d.ts.map +1 -1
- package/dist/runtime/auto-update/state-store.js +37 -3
- package/dist/runtime/auto-update/state-store.js.map +1 -1
- package/dist/runtime/auto-update/types.d.ts +52 -0
- package/dist/runtime/auto-update/types.d.ts.map +1 -1
- package/dist/runtime/auto-update/types.js.map +1 -1
- package/dist/runtime/auto-update/update-marker.d.ts +95 -0
- package/dist/runtime/auto-update/update-marker.d.ts.map +1 -0
- package/dist/runtime/auto-update/update-marker.js +185 -0
- package/dist/runtime/auto-update/update-marker.js.map +1 -0
- package/dist/runtime/auto-update/update-schedule.d.ts +67 -0
- package/dist/runtime/auto-update/update-schedule.d.ts.map +1 -0
- package/dist/runtime/auto-update/update-schedule.js +91 -0
- package/dist/runtime/auto-update/update-schedule.js.map +1 -0
- package/dist/runtime/decision-errors.d.ts +7 -0
- package/dist/runtime/decision-errors.d.ts.map +1 -1
- package/dist/runtime/decision-errors.js +15 -0
- package/dist/runtime/decision-errors.js.map +1 -1
- package/dist/runtime/process-state.d.ts +168 -0
- package/dist/runtime/process-state.d.ts.map +1 -0
- package/dist/runtime/process-state.js +81 -0
- package/dist/runtime/process-state.js.map +1 -0
- package/dist/runtime/reconcile-state.d.ts.map +1 -1
- package/dist/runtime/reconcile-state.js +21 -4
- package/dist/runtime/reconcile-state.js.map +1 -1
- package/dist/runtime/strategy-registry.d.ts +16 -0
- package/dist/runtime/strategy-registry.d.ts.map +1 -1
- package/dist/runtime/strategy-registry.js +7 -3
- package/dist/runtime/strategy-registry.js.map +1 -1
- package/dist/runtime/update-apply-deps.d.ts +5 -3
- package/dist/runtime/update-apply-deps.d.ts.map +1 -1
- package/dist/runtime/update-apply-deps.js +15 -3
- package/dist/runtime/update-apply-deps.js.map +1 -1
- package/dist/senpi/client.d.ts +29 -0
- package/dist/senpi/client.d.ts.map +1 -1
- package/dist/senpi/client.js +71 -2
- package/dist/senpi/client.js.map +1 -1
- package/dist/skill-router/catalog.d.ts +30 -0
- package/dist/skill-router/catalog.d.ts.map +1 -0
- package/dist/skill-router/catalog.js +131 -0
- package/dist/skill-router/catalog.js.map +1 -0
- package/dist/skill-router/classify-client.d.ts +80 -0
- package/dist/skill-router/classify-client.d.ts.map +1 -0
- package/dist/skill-router/classify-client.js +283 -0
- package/dist/skill-router/classify-client.js.map +1 -0
- package/dist/skill-router/prompt.d.ts +36 -0
- package/dist/skill-router/prompt.d.ts.map +1 -0
- package/dist/skill-router/prompt.js +318 -0
- package/dist/skill-router/prompt.js.map +1 -0
- package/dist/skill-router/resolve.d.ts +41 -0
- package/dist/skill-router/resolve.d.ts.map +1 -0
- package/dist/skill-router/resolve.js +43 -0
- package/dist/skill-router/resolve.js.map +1 -0
- package/dist/skills-manager/version-manager.d.ts +14 -0
- package/dist/skills-manager/version-manager.d.ts.map +1 -1
- package/dist/skills-manager/version-manager.js +31 -3
- package/dist/skills-manager/version-manager.js.map +1 -1
- package/dist/strategy/strategy-state.d.ts +11 -3
- package/dist/strategy/strategy-state.d.ts.map +1 -1
- package/dist/strategy/strategy-state.js +18 -8
- package/dist/strategy/strategy-state.js.map +1 -1
- package/dist/telemetry/event-builders.d.ts +4 -2
- package/dist/telemetry/event-builders.d.ts.map +1 -1
- package/dist/telemetry/event-builders.js +4 -2
- package/dist/telemetry/event-builders.js.map +1 -1
- package/dist/telemetry/metrics.d.ts +15 -1
- package/dist/telemetry/metrics.d.ts.map +1 -1
- package/dist/telemetry/metrics.js +25 -1
- package/dist/telemetry/metrics.js.map +1 -1
- package/dist/telemetry/otel-sdk.d.ts +12 -0
- package/dist/telemetry/otel-sdk.d.ts.map +1 -1
- package/dist/telemetry/otel-sdk.js +47 -1
- package/dist/telemetry/otel-sdk.js.map +1 -1
- package/dist/telemetry/spans/span-hooks.d.ts +18 -1
- package/dist/telemetry/spans/span-hooks.d.ts.map +1 -1
- package/dist/telemetry/spans/span-hooks.js +315 -96
- package/dist/telemetry/spans/span-hooks.js.map +1 -1
- package/dist/telemetry/tool-plane.d.ts +2 -0
- package/dist/telemetry/tool-plane.d.ts.map +1 -1
- package/dist/telemetry/tool-plane.js +1 -0
- package/dist/telemetry/tool-plane.js.map +1 -1
- package/dist/test-setup.js +7 -1
- package/dist/test-setup.js.map +1 -1
- package/dist/tool-surface/crib-cache.d.ts +21 -0
- package/dist/tool-surface/crib-cache.d.ts.map +1 -0
- package/dist/tool-surface/crib-cache.js +54 -0
- package/dist/tool-surface/crib-cache.js.map +1 -0
- package/dist/tool-surface/crib-hook.d.ts +78 -0
- package/dist/tool-surface/crib-hook.d.ts.map +1 -0
- package/dist/tool-surface/crib-hook.js +225 -0
- package/dist/tool-surface/crib-hook.js.map +1 -0
- package/dist/tool-surface/crib-sheet.d.ts +56 -0
- package/dist/tool-surface/crib-sheet.d.ts.map +1 -0
- package/dist/tool-surface/crib-sheet.js +129 -0
- package/dist/tool-surface/crib-sheet.js.map +1 -0
- package/dist/tool-surface/mcp-tool-source.d.ts +64 -0
- package/dist/tool-surface/mcp-tool-source.d.ts.map +1 -0
- package/dist/tool-surface/mcp-tool-source.js +189 -0
- package/dist/tool-surface/mcp-tool-source.js.map +1 -0
- package/dist/utils/event-catalog.d.ts +228 -6
- package/dist/utils/event-catalog.d.ts.map +1 -1
- package/dist/utils/event-catalog.js +32 -0
- package/dist/utils/event-catalog.js.map +1 -1
- package/dist/widgets/tool.d.ts +9 -0
- package/dist/widgets/tool.d.ts.map +1 -1
- package/dist/widgets/tool.js +1 -0
- package/dist/widgets/tool.js.map +1 -1
- package/openclaw.plugin.json +11 -1
- package/package.json +5 -2
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import { runCommand } from "./command-runner.js";
|
|
2
|
-
import { evaluateGatewayRestartCapability } from "./gateway-restart.js";
|
|
2
|
+
import { evaluateGatewayRestartCapability, evaluateModernRestartCapability, triggerGatewayRestartViaMutate, } from "./gateway-restart.js";
|
|
3
3
|
import { classifyUpdate, parseSemver } from "./policy.js";
|
|
4
4
|
import { evaluatePreflight } from "./preflight.js";
|
|
5
5
|
import { resolveConfiguredSpec } from "./resolve-package-name.js";
|
|
6
6
|
import { AutoUpdateStateStore } from "./state-store.js";
|
|
7
7
|
import { createDefaultAutoUpdateState, } from "./types.js";
|
|
8
|
+
import { consumeInstallResults, describeInstallResult, installResultLevel, } from "./install-results.js";
|
|
9
|
+
import { buildExactUpdateSpec, clearUpdateMarker, readUpdateMarker, writeUpdateMarker } from "./update-marker.js";
|
|
10
|
+
import { eligibleAt as computeEligibleAt, mustForce } from "./update-schedule.js";
|
|
8
11
|
import { NpmVersionSource } from "./version-source.js";
|
|
9
12
|
import { EVENT_NAME, eventAttrs } from "../../utils/event-catalog.js";
|
|
10
13
|
/**
|
|
@@ -22,6 +25,51 @@ const CHECK_NOW_REASON = {
|
|
|
22
25
|
disabledInConfig: "Auto-update is turned off in config; no check was run.",
|
|
23
26
|
coordinatorStopped: "The auto-update coordinator is not running; no check was run.",
|
|
24
27
|
};
|
|
28
|
+
/**
|
|
29
|
+
* How long a requested restart may take to load its target before the
|
|
30
|
+
* awaiting-restart state is treated as failed (the 3.0.171 latch fix). A 10.x
|
|
31
|
+
* restart-to-apply measured about 71 s on Box A, with starts up to 93 s; 30 min
|
|
32
|
+
* leaves room for the gateway's own deferral (forced after 300 s) and a slow
|
|
33
|
+
* offline install.
|
|
34
|
+
*/
|
|
35
|
+
export const RESTART_LOAD_TIMEOUT_MS = 30 * 60_000;
|
|
36
|
+
/**
|
|
37
|
+
* Restarts requested for one target that may fail to load before the box
|
|
38
|
+
* stops trying that version and waits for a newer one. Bounds the loop of a
|
|
39
|
+
* version that can never install (a blocked scan, a yanked version, a wrapper
|
|
40
|
+
* without the marker step): without it an idle box would restart about every
|
|
41
|
+
* 80 min forever.
|
|
42
|
+
*/
|
|
43
|
+
export const MAX_APPLY_ATTEMPTS = 3;
|
|
44
|
+
/**
|
|
45
|
+
* How often a 10.x box that is `pending-idle` re-reads its apply gate (the
|
|
46
|
+
* spread time and the idle gate) between registry polls. The idle gate has no
|
|
47
|
+
* change notification, so the change is caught by this re-read; when the gate
|
|
48
|
+
* opens, an ordinary check runs at once instead of up to one poll interval
|
|
49
|
+
* (60 min by default) later. The re-read itself is local: no registry call,
|
|
50
|
+
* no `onTickComplete`.
|
|
51
|
+
*/
|
|
52
|
+
export const PENDING_IDLE_RECHECK_MS = 3 * 60_000;
|
|
53
|
+
/**
|
|
54
|
+
* The only idle-gate blockers the 24 h forced restart may override: a busy
|
|
55
|
+
* user (`user-run`, `recent-message`) and a gate that just started
|
|
56
|
+
* (`recent-start`, no known activity at all). Every other blocker holds it, money
|
|
57
|
+
* first: an order call in flight (the restart would cut it), an open trade
|
|
58
|
+
* approval (it would be cancelled), a running `senpi deploy` (it would leave a
|
|
59
|
+
* funded wallet with no runtime), a queued user message, and a gate that
|
|
60
|
+
* threw (it cannot rule any of those out).
|
|
61
|
+
*/
|
|
62
|
+
const FORCE_MAY_OVERRIDE = new Set(["user-run", "recent-message", "recent-start"]);
|
|
63
|
+
/** Whether every blocker in `blockers` is one the forced restart may override. */
|
|
64
|
+
function onlyOverridable(blockers) {
|
|
65
|
+
return blockers.every((b) => FORCE_MAY_OVERRIDE.has(b));
|
|
66
|
+
}
|
|
67
|
+
/** Whether `loaded` is at or past `target` (major.minor.patch; both must parse). */
|
|
68
|
+
function isAtLeast(loaded, target) {
|
|
69
|
+
if (!parseSemver(loaded) || !parseSemver(target))
|
|
70
|
+
return false;
|
|
71
|
+
return classifyUpdate(loaded, target) === "none";
|
|
72
|
+
}
|
|
25
73
|
/** Cap a registry-supplied string before it reaches a log line or an event body. */
|
|
26
74
|
function summarizeForLog(value, maxLength = 120) {
|
|
27
75
|
const collapsed = value.replace(/\s+/g, " ").trim();
|
|
@@ -49,6 +97,8 @@ export class AutoUpdateCoordinator {
|
|
|
49
97
|
versionSource;
|
|
50
98
|
runUpdateCommand;
|
|
51
99
|
timer = null;
|
|
100
|
+
/** 10.x `pending-idle` gate re-read; see {@link PENDING_IDLE_RECHECK_MS}. */
|
|
101
|
+
idleRecheckTimer = null;
|
|
52
102
|
running = false;
|
|
53
103
|
stopped = true;
|
|
54
104
|
runToken = 0;
|
|
@@ -58,8 +108,26 @@ export class AutoUpdateCoordinator {
|
|
|
58
108
|
/** Spec the last resolve used. Seeded in the constructor, so the snapshot
|
|
59
109
|
* reports a line before the first tick. */
|
|
60
110
|
updateSpec;
|
|
111
|
+
/** Why the modern path is still waiting; reset every tick. See {@link AutoUpdateSnapshot.pendingBlockers}. */
|
|
112
|
+
pendingBlockers = [];
|
|
113
|
+
now;
|
|
114
|
+
rand;
|
|
115
|
+
loadedVersion;
|
|
116
|
+
/**
|
|
117
|
+
* 10.x: `<version>|<sorted blockers>` of the last `auto_update.deferred`, so the event fires on
|
|
118
|
+
* entering `pending-idle` and on a change only. Cleared when a tick ends outside `pending-idle` or fails.
|
|
119
|
+
* In memory: a new gateway process reports a wait that is still on once.
|
|
120
|
+
*/
|
|
121
|
+
lastDeferredKey = null;
|
|
122
|
+
/** The install-results report the last {@link start} began; see {@link whenInstallResultsReported}. */
|
|
123
|
+
installResultsReport = Promise.resolve();
|
|
124
|
+
/** The 10.x `auto_update.applied` report the last {@link start} began; see {@link whenAppliedReported}. */
|
|
125
|
+
appliedReport = Promise.resolve();
|
|
61
126
|
constructor(options) {
|
|
62
127
|
this.options = options;
|
|
128
|
+
this.now = options.now ?? Date.now;
|
|
129
|
+
this.rand = options.rand ?? Math.random;
|
|
130
|
+
this.loadedVersion = options.loadedVersion ?? options.currentVersion;
|
|
63
131
|
this.store = new AutoUpdateStateStore(options.stateDir, options.logger);
|
|
64
132
|
this.versionSource = options.versionSource ?? new NpmVersionSource();
|
|
65
133
|
this.runUpdateCommand = options.runUpdateCommand ?? defaultRunUpdateCommand;
|
|
@@ -79,6 +147,8 @@ export class AutoUpdateCoordinator {
|
|
|
79
147
|
this.stopped = true;
|
|
80
148
|
this.snapshot = this.buildSnapshot("disabled", "auto-update-disabled");
|
|
81
149
|
logger.info(`[senpi-auto-update] disabled by config (pluginEnabled=${String(this.options.enabled)}, configEnabled=${String(config.enabled)})`);
|
|
150
|
+
// Operator markers and the boot floor still install on a box with auto-update off.
|
|
151
|
+
this.startInstallResultsReport(null);
|
|
82
152
|
return;
|
|
83
153
|
}
|
|
84
154
|
logger.info(`[senpi-auto-update] starting | localVersion=${this.options.currentVersion} package=${this.options.packageName} spec=${this.resolveUpdateSpec()} mode=${config.mode} pollInterval=${config.pollIntervalMinutes}m startupDelay=${config.startupDelaySeconds}s failureBackoff=${config.failureBackoffMinutes}m maxBackoff=${config.maxBackoffMinutes}m`);
|
|
@@ -90,17 +160,219 @@ export class AutoUpdateCoordinator {
|
|
|
90
160
|
logger.info(`[senpi-auto-update] preflight result | ok=${String(preflight.ok)} effectiveMode=${preflight.effectiveMode} reasons=${preflight.reasons.join(",") || "none"}`);
|
|
91
161
|
this.state = await this.store.load(this.options.currentVersion, preflight.effectiveMode);
|
|
92
162
|
this.state.effectiveMode = preflight.effectiveMode;
|
|
163
|
+
const reconciled = await this.reconcileLoadedVersion(this.state);
|
|
93
164
|
try {
|
|
94
165
|
await this.store.save(this.state);
|
|
95
166
|
}
|
|
96
167
|
catch (error) {
|
|
97
168
|
this.options.logger.warn("[senpi-auto-update] failed to persist initial state", error);
|
|
98
169
|
}
|
|
99
|
-
this.
|
|
170
|
+
this.startInstallResultsReport(this.state);
|
|
171
|
+
this.snapshot = this.buildSnapshot("healthy", reconciled ?? "started");
|
|
100
172
|
const delayMs = config.startupDelaySeconds * 1000;
|
|
101
173
|
logger.info(`[senpi-auto-update] first check scheduled | delayMs=${delayMs} nextCheckAt=${new Date(Date.now() + delayMs).toISOString()}`);
|
|
102
174
|
this.scheduleNext(delayMs, this.runToken);
|
|
103
175
|
}
|
|
176
|
+
/**
|
|
177
|
+
* Startup half of the latch fix: compare what this process loaded with what
|
|
178
|
+
* a previous process asked for.
|
|
179
|
+
*
|
|
180
|
+
* When the loaded version reaches the requested restart's target, the update
|
|
181
|
+
* took effect: the pending restart and pending update are cleared, a stale
|
|
182
|
+
* install marker is removed, and on a 10.x host `auto_update.applied` fires
|
|
183
|
+
* (nothing was installed before this boot there). On 5.x `applied` already
|
|
184
|
+
* fired when the restart was requested, so it is not counted twice.
|
|
185
|
+
*
|
|
186
|
+
* A target the loaded version has not reached is left for the tick, which
|
|
187
|
+
* expires it once the request is older than {@link RESTART_LOAD_TIMEOUT_MS}.
|
|
188
|
+
*
|
|
189
|
+
* @returns `updated:<v>` when a requested update is now loaded, else `undefined`.
|
|
190
|
+
*/
|
|
191
|
+
async reconcileLoadedVersion(state) {
|
|
192
|
+
const loaded = this.loadedVersion;
|
|
193
|
+
let action;
|
|
194
|
+
const pending = state.pendingRestart;
|
|
195
|
+
if (pending && isAtLeast(loaded, pending.toVersion)) {
|
|
196
|
+
action = `updated:${pending.toVersion}`;
|
|
197
|
+
this.options.logger.info(`[senpi-auto-update] update to v${pending.toVersion} is loaded (requested ${pending.requestedAt}, mode ${pending.mode})`);
|
|
198
|
+
// A 5.x pending restart already reported `applied` when it was requested.
|
|
199
|
+
if (this.options.modernHost && pending.mode === "restart-to-apply") {
|
|
200
|
+
// How long the apply took: request to this boot (stop + wrapper install + start), and
|
|
201
|
+
// first seen to this boot (spread + idle wait + apply). Read before pendingUpdate is cleared.
|
|
202
|
+
const now = this.now();
|
|
203
|
+
const sinceMs = (iso) => {
|
|
204
|
+
const ms = iso ? Date.parse(iso) : Number.NaN;
|
|
205
|
+
return Number.isFinite(ms) ? Math.max(0, now - ms) : undefined;
|
|
206
|
+
};
|
|
207
|
+
const firstSeenAt = state.pendingUpdate?.firstSeenAt;
|
|
208
|
+
const pendingMs = sinceMs(firstSeenAt);
|
|
209
|
+
const applyMs = sinceMs(pending.requestedAt);
|
|
210
|
+
this.appliedReport = this.reportApplied(() => this.emitEvent(EVENT_NAME.autoUpdateApplied, "info", {
|
|
211
|
+
body: `Update applied: ${pending.fromVersion} -> v${pending.toVersion} (${pending.mode})`,
|
|
212
|
+
attributes: eventAttrs({
|
|
213
|
+
"senpi.auto_update.from_version": pending.fromVersion,
|
|
214
|
+
"senpi.auto_update.to_version": pending.toVersion,
|
|
215
|
+
"senpi.auto_update.restart_mode": pending.mode,
|
|
216
|
+
"senpi.auto_update.requested_at": pending.requestedAt,
|
|
217
|
+
"senpi.auto_update.first_seen_at": pendingMs === undefined ? undefined : firstSeenAt,
|
|
218
|
+
"senpi.auto_update.apply_ms": applyMs,
|
|
219
|
+
"senpi.auto_update.pending_ms": pendingMs,
|
|
220
|
+
}),
|
|
221
|
+
}));
|
|
222
|
+
}
|
|
223
|
+
state.pendingRestart = undefined;
|
|
224
|
+
}
|
|
225
|
+
if (state.pendingUpdate && isAtLeast(loaded, state.pendingUpdate.version)) {
|
|
226
|
+
state.pendingUpdate = undefined;
|
|
227
|
+
}
|
|
228
|
+
if (state.applyFailures && isAtLeast(loaded, state.applyFailures.version)) {
|
|
229
|
+
state.applyFailures = undefined;
|
|
230
|
+
}
|
|
231
|
+
if (state.lastAppliedVersion && isAtLeast(loaded, state.lastAppliedVersion)) {
|
|
232
|
+
state.lastLoggedAwaitingRestartVersion = undefined;
|
|
233
|
+
}
|
|
234
|
+
const markerPath = this.options.updateMarkerPath;
|
|
235
|
+
if (markerPath) {
|
|
236
|
+
const marker = await readUpdateMarker(markerPath);
|
|
237
|
+
if (marker && isAtLeast(loaded, marker.targetVersion)) {
|
|
238
|
+
await this.clearMarkerQuietly();
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
return action;
|
|
242
|
+
}
|
|
243
|
+
/**
|
|
244
|
+
* Emit the 10.x `auto_update.applied` once the OTel SDK that `register()` started is up, then
|
|
245
|
+
* flush. It fires right after boot, when the global LoggerProvider may still be the no-op one,
|
|
246
|
+
* and the pending restart is already cleared, so a dropped event would never be sent again. A
|
|
247
|
+
* late or failed init still emits: the gateway log keeps the line either way.
|
|
248
|
+
*/
|
|
249
|
+
async reportApplied(emit) {
|
|
250
|
+
const telemetry = this.options.installResultsTelemetry;
|
|
251
|
+
if (!telemetry) {
|
|
252
|
+
emit();
|
|
253
|
+
return;
|
|
254
|
+
}
|
|
255
|
+
try {
|
|
256
|
+
await telemetry.ready();
|
|
257
|
+
}
|
|
258
|
+
catch {
|
|
259
|
+
// Telemetry must not break the coordinator.
|
|
260
|
+
}
|
|
261
|
+
emit();
|
|
262
|
+
try {
|
|
263
|
+
await telemetry.flush();
|
|
264
|
+
}
|
|
265
|
+
catch {
|
|
266
|
+
// Telemetry must not break the coordinator.
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
/** Resolves when the `applied` report begun by the last {@link start}, if any, has finished. */
|
|
270
|
+
whenAppliedReported() {
|
|
271
|
+
return this.appliedReport;
|
|
272
|
+
}
|
|
273
|
+
/**
|
|
274
|
+
* Resolves when the install-results report begun by the last {@link start} has finished.
|
|
275
|
+
* `start` does not wait for it: the report waits for the OTel SDK first.
|
|
276
|
+
*/
|
|
277
|
+
whenInstallResultsReported() {
|
|
278
|
+
return this.installResultsReport;
|
|
279
|
+
}
|
|
280
|
+
/**
|
|
281
|
+
* 10.x: start reporting the wrapper's install outcomes in the background. `liveState` is the
|
|
282
|
+
* running coordinator's state (the ids are saved with it); `null` when auto-update is
|
|
283
|
+
* disabled, which loads only the ids and patches only them.
|
|
284
|
+
*/
|
|
285
|
+
startInstallResultsReport(liveState) {
|
|
286
|
+
if (!this.options.modernHost || !this.options.installResultsPath)
|
|
287
|
+
return;
|
|
288
|
+
this.installResultsReport = this.reportInstallResults(this.options.installResultsPath, liveState);
|
|
289
|
+
}
|
|
290
|
+
/**
|
|
291
|
+
* One `auto_update.install_result` per entry not reported before (`install-results.ts`), then
|
|
292
|
+
* the file is deleted. Runs from {@link start}, which only the gateway's plugin service calls,
|
|
293
|
+
* so a CLI process never consumes the file. Never throws: telemetry must not break the
|
|
294
|
+
* coordinator.
|
|
295
|
+
*/
|
|
296
|
+
async reportInstallResults(path, liveState) {
|
|
297
|
+
try {
|
|
298
|
+
const reportedIds = liveState
|
|
299
|
+
? liveState.reportedInstallResultIds
|
|
300
|
+
: (await this.store.load(this.options.currentVersion, this.options.config.mode)).reportedInstallResultIds;
|
|
301
|
+
const result = await consumeInstallResults({
|
|
302
|
+
path,
|
|
303
|
+
reportedIds: Array.isArray(reportedIds) ? reportedIds : [],
|
|
304
|
+
emit: (entry) => this.emitInstallResult(entry),
|
|
305
|
+
persistReportedIds: async (ids) => {
|
|
306
|
+
if (!liveState)
|
|
307
|
+
return this.store.patch({ reportedInstallResultIds: ids });
|
|
308
|
+
// The running coordinator saves this same object on every tick; saving it whole
|
|
309
|
+
// never writes back an older copy.
|
|
310
|
+
liveState.reportedInstallResultIds = ids;
|
|
311
|
+
await this.store.save(liveState);
|
|
312
|
+
},
|
|
313
|
+
telemetry: this.options.installResultsTelemetry,
|
|
314
|
+
logger: this.options.logger,
|
|
315
|
+
});
|
|
316
|
+
if (result.emitted > 0 || result.skipped > 0) {
|
|
317
|
+
this.options.logger.info(`[senpi-auto-update] reported ${result.emitted} wrapper install result(s)` +
|
|
318
|
+
(result.skipped > 0 ? ` (${result.skipped} already reported)` : ""));
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
catch (error) {
|
|
322
|
+
this.options.logger.warn(`[senpi-auto-update] reading the wrapper's install results failed | err=${formatErr(error)}`);
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
emitInstallResult(e) {
|
|
326
|
+
const opt = (v) => (v === null ? undefined : v);
|
|
327
|
+
this.emitEvent(EVENT_NAME.autoUpdateInstallResult, installResultLevel(e.outcome), {
|
|
328
|
+
body: describeInstallResult(e),
|
|
329
|
+
attributes: eventAttrs({
|
|
330
|
+
"senpi.auto_update.source": opt(e.source),
|
|
331
|
+
"senpi.auto_update.outcome": e.outcome,
|
|
332
|
+
"senpi.auto_update.spec": opt(e.spec),
|
|
333
|
+
"senpi.auto_update.from_version": opt(e.fromVersion),
|
|
334
|
+
"senpi.auto_update.to_version": opt(e.toVersion),
|
|
335
|
+
"senpi.auto_update.target_version": opt(e.targetVersion),
|
|
336
|
+
"senpi.auto_update.duration_ms": opt(e.durationMs),
|
|
337
|
+
"senpi.auto_update.reason": opt(e.reason),
|
|
338
|
+
"senpi.auto_update.ended_at": opt(e.at),
|
|
339
|
+
"senpi.auto_update.result_id": opt(e.id),
|
|
340
|
+
"senpi.auto_update.openclaw_version": opt(e.openclawVersion),
|
|
341
|
+
}),
|
|
342
|
+
});
|
|
343
|
+
}
|
|
344
|
+
/**
|
|
345
|
+
* 10.x: the box waits (`pending-idle:<v>`). `auto_update.deferred` on entering the wait and when
|
|
346
|
+
* the target or the sorted blocker set changes; an unchanged re-check or poll is silent.
|
|
347
|
+
*/
|
|
348
|
+
noteDeferred(pending, latest, blockers) {
|
|
349
|
+
const sorted = blockerList(blockers);
|
|
350
|
+
const key = `${latest}|${sorted}`;
|
|
351
|
+
if (key === this.lastDeferredKey)
|
|
352
|
+
return;
|
|
353
|
+
this.lastDeferredKey = key;
|
|
354
|
+
this.emitEvent(EVENT_NAME.autoUpdateDeferred, "info", {
|
|
355
|
+
body: `Restart to apply v${latest} deferred: waiting for ${sorted || "nothing"} (eligible at ${pending.eligibleAt})`,
|
|
356
|
+
attributes: eventAttrs({
|
|
357
|
+
"senpi.auto_update.to_version": latest,
|
|
358
|
+
"senpi.auto_update.blockers": sorted,
|
|
359
|
+
"senpi.auto_update.eligible_at": pending.eligibleAt,
|
|
360
|
+
"senpi.auto_update.first_seen_at": pending.firstSeenAt,
|
|
361
|
+
}),
|
|
362
|
+
});
|
|
363
|
+
}
|
|
364
|
+
/** Remove the install marker; a failure is logged, never thrown (the wrapper deletes it too). */
|
|
365
|
+
async clearMarkerQuietly() {
|
|
366
|
+
const markerPath = this.options.updateMarkerPath;
|
|
367
|
+
if (!markerPath)
|
|
368
|
+
return;
|
|
369
|
+
try {
|
|
370
|
+
await clearUpdateMarker(markerPath);
|
|
371
|
+
}
|
|
372
|
+
catch (error) {
|
|
373
|
+
this.options.logger.warn(`[senpi-auto-update] failed to clear the update marker | path=${markerPath} err=${formatErr(error)}`);
|
|
374
|
+
}
|
|
375
|
+
}
|
|
104
376
|
/** Cancel the next scheduled poll and prevent in-flight ticks from rescheduling. */
|
|
105
377
|
async stop() {
|
|
106
378
|
this.runToken += 1;
|
|
@@ -109,6 +381,7 @@ export class AutoUpdateCoordinator {
|
|
|
109
381
|
clearTimeout(this.timer);
|
|
110
382
|
this.timer = null;
|
|
111
383
|
}
|
|
384
|
+
this.clearIdleRecheck();
|
|
112
385
|
}
|
|
113
386
|
/** Return the latest point-in-time snapshot for health/state gateway RPCs. */
|
|
114
387
|
getSnapshot() {
|
|
@@ -154,9 +427,7 @@ export class AutoUpdateCoordinator {
|
|
|
154
427
|
// way to tell a completed check from a failed one is whether it counted a
|
|
155
428
|
// failure. Reporting "ran" either way tells an operator testing a config
|
|
156
429
|
// change that it worked when nothing reached the registry.
|
|
157
|
-
// Baseline on live state
|
|
158
|
-
// snapshot before `tick` zeroes the counter, so after a recovery the
|
|
159
|
-
// snapshot still carries the old total and 0 -> 1 reads as no change.
|
|
430
|
+
// Baseline on live state: it is what `tick` changes.
|
|
160
431
|
const failuresBefore = this.state.consecutiveFailures;
|
|
161
432
|
await this.tick(this.runToken);
|
|
162
433
|
if (this.state.consecutiveFailures > failuresBefore) {
|
|
@@ -194,15 +465,24 @@ export class AutoUpdateCoordinator {
|
|
|
194
465
|
return;
|
|
195
466
|
}
|
|
196
467
|
this.running = true;
|
|
468
|
+
this.clearIdleRecheck();
|
|
197
469
|
this.options.logger.info(`[senpi-auto-update] tick started | localVersion=${this.options.currentVersion}`);
|
|
198
470
|
const state = this.state;
|
|
199
471
|
let nextDelayMs = null;
|
|
472
|
+
let recheckIdle = false;
|
|
200
473
|
try {
|
|
201
474
|
await this.runTick(state);
|
|
202
475
|
state.consecutiveFailures = 0;
|
|
203
476
|
state.lastFailureAt = undefined;
|
|
204
477
|
state.lastFailureReason = undefined;
|
|
205
478
|
await this.persistState(state);
|
|
479
|
+
// `runTick` built the snapshot before the reset above; rebuild it so it
|
|
480
|
+
// carries the state that was just written.
|
|
481
|
+
this.snapshot = this.buildSnapshot(this.snapshot.health, this.snapshot.lastAction, this.snapshot.lastError);
|
|
482
|
+
recheckIdle = this.options.modernHost === true && this.snapshot.lastAction.startsWith("pending-idle:");
|
|
483
|
+
// The wait ended (restart requested, no update, a restart failure): the next wait is new.
|
|
484
|
+
if (!recheckIdle)
|
|
485
|
+
this.lastDeferredKey = null;
|
|
206
486
|
nextDelayMs = this.options.config.pollIntervalMinutes * 60_000;
|
|
207
487
|
this.options.logger.info(`[senpi-auto-update] tick succeeded | nextDelayMs=${nextDelayMs}`);
|
|
208
488
|
}
|
|
@@ -211,6 +491,8 @@ export class AutoUpdateCoordinator {
|
|
|
211
491
|
state.lastFailureAt = new Date().toISOString();
|
|
212
492
|
state.lastFailureReason = formatErr(error);
|
|
213
493
|
await this.persistState(state);
|
|
494
|
+
// A failed tick ends the wait it was in: the next wait is reported as new.
|
|
495
|
+
this.lastDeferredKey = null;
|
|
214
496
|
const base = this.options.config.failureBackoffMinutes * 60_000;
|
|
215
497
|
const max = this.options.config.maxBackoffMinutes * 60_000;
|
|
216
498
|
const backoff = Math.min(max, base * Math.max(1, state.consecutiveFailures));
|
|
@@ -247,8 +529,67 @@ export class AutoUpdateCoordinator {
|
|
|
247
529
|
this.options.logger.info(`[senpi-auto-update] scheduling next tick after onTickComplete | nextDelayMs=${nextDelayMs}`);
|
|
248
530
|
this.scheduleNext(nextDelayMs, token);
|
|
249
531
|
}
|
|
532
|
+
if (recheckIdle)
|
|
533
|
+
this.scheduleIdleRecheck(token);
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
/** Re-read the apply gate of a `pending-idle` box after {@link PENDING_IDLE_RECHECK_MS}. */
|
|
537
|
+
scheduleIdleRecheck(token) {
|
|
538
|
+
if (this.stopped || token !== this.runToken)
|
|
539
|
+
return;
|
|
540
|
+
this.clearIdleRecheck();
|
|
541
|
+
this.idleRecheckTimer = setTimeout(() => {
|
|
542
|
+
this.idleRecheckTimer = null;
|
|
543
|
+
this.recheckPendingIdle(token);
|
|
544
|
+
}, PENDING_IDLE_RECHECK_MS);
|
|
545
|
+
}
|
|
546
|
+
clearIdleRecheck() {
|
|
547
|
+
if (this.idleRecheckTimer) {
|
|
548
|
+
clearTimeout(this.idleRecheckTimer);
|
|
549
|
+
this.idleRecheckTimer = null;
|
|
250
550
|
}
|
|
251
551
|
}
|
|
552
|
+
/**
|
|
553
|
+
* Run a check now when the gate that held the pending update has opened,
|
|
554
|
+
* else re-read it again later. A check already in flight re-arms this
|
|
555
|
+
* itself when it ends `pending-idle`.
|
|
556
|
+
*/
|
|
557
|
+
recheckPendingIdle(token) {
|
|
558
|
+
if (this.running || this.stopped || token !== this.runToken)
|
|
559
|
+
return;
|
|
560
|
+
const pending = this.state.pendingUpdate;
|
|
561
|
+
if (!pending)
|
|
562
|
+
return;
|
|
563
|
+
const now = this.now();
|
|
564
|
+
const gate = this.applyGate(now, Date.parse(pending.eligibleAt));
|
|
565
|
+
if (!gate.open) {
|
|
566
|
+
// Still waiting; a blocker set that changed since the last poll is a new deferred,
|
|
567
|
+
// and the status snapshot reads the same set.
|
|
568
|
+
this.pendingBlockers = gate.blockers;
|
|
569
|
+
this.snapshot = this.buildSnapshot(this.snapshot.health, this.snapshot.lastAction, this.snapshot.lastError);
|
|
570
|
+
this.noteDeferred(pending, pending.version, gate.blockers);
|
|
571
|
+
this.scheduleIdleRecheck(token);
|
|
572
|
+
return;
|
|
573
|
+
}
|
|
574
|
+
this.options.logger.info(`[senpi-auto-update] pending v${pending.version}: apply gate open; checking now`);
|
|
575
|
+
void this.tick(token);
|
|
576
|
+
}
|
|
577
|
+
/**
|
|
578
|
+
* The 10.x apply gate: wait for `eligibleAt` (blocker `spread`), then for an
|
|
579
|
+
* idle box, unless {@link mustForce} applies and every blocker is one the
|
|
580
|
+
* forced restart may override.
|
|
581
|
+
*/
|
|
582
|
+
applyGate(now, eligibleMs) {
|
|
583
|
+
if (Number.isFinite(eligibleMs) && now < eligibleMs) {
|
|
584
|
+
return { open: false, forced: false, blockers: ["spread"] };
|
|
585
|
+
}
|
|
586
|
+
const decision = this.readIdle(now);
|
|
587
|
+
// Forcing overrides a busy user, never money in motion (FORCE_MAY_OVERRIDE).
|
|
588
|
+
const forced = !decision.idle &&
|
|
589
|
+
onlyOverridable(decision.blockers) &&
|
|
590
|
+
mustForce(now, Number.isFinite(eligibleMs) ? eligibleMs : now, this.options.mid);
|
|
591
|
+
return { open: decision.idle || forced, forced, blockers: [...decision.blockers] };
|
|
592
|
+
}
|
|
252
593
|
/**
|
|
253
594
|
* Emit a catalogued `auto_update.*` event, guarded so neither a throwing sink nor a bad
|
|
254
595
|
* attribute shape can disturb the tick — telemetry must never break auto-update itself.
|
|
@@ -274,6 +615,7 @@ export class AutoUpdateCoordinator {
|
|
|
274
615
|
/** Core update logic: fetch latest version, classify, and take action. */
|
|
275
616
|
async runTick(state) {
|
|
276
617
|
const { logger } = this.options;
|
|
618
|
+
this.pendingBlockers = [];
|
|
277
619
|
// Stamped before the tag is resolved so a rejected tag reads as "checked
|
|
278
620
|
// just now and failed", not as "stopped polling a while ago".
|
|
279
621
|
state.lastCheckedAt = new Date().toISOString();
|
|
@@ -324,29 +666,350 @@ export class AutoUpdateCoordinator {
|
|
|
324
666
|
"senpi.auto_update.will_update": willUpdate,
|
|
325
667
|
}),
|
|
326
668
|
});
|
|
669
|
+
// A pending 10.x update lives only while an applicable update is seen, so
|
|
670
|
+
// a dist-tag rollback or a mode change does not leave a stale eligibility
|
|
671
|
+
// time that would let the next release skip the spread.
|
|
327
672
|
if (updateType === "none") {
|
|
673
|
+
state.pendingUpdate = undefined;
|
|
674
|
+
// 5.x: `latest` was rolled back to the loaded version while a newer
|
|
675
|
+
// install sits on disk unloaded. Reinstall `latest` so the next restart
|
|
676
|
+
// cannot load the pulled release. It is already loaded: no restart.
|
|
677
|
+
const newerOnDisk = [state.lastAppliedVersion, state.pendingRestart?.toVersion].some((v) => v !== undefined && parseSemver(v) !== null && classifyUpdate(latest, v) !== "none");
|
|
678
|
+
if (!this.options.modernHost &&
|
|
679
|
+
state.effectiveMode === "auto-apply" &&
|
|
680
|
+
latest === this.options.currentVersion &&
|
|
681
|
+
newerOnDisk) {
|
|
682
|
+
this.options.logger.warn(`[senpi-auto-update] latest was rolled back to the loaded v${latest} while a newer install is on disk; reinstalling v${latest}`);
|
|
683
|
+
try {
|
|
684
|
+
await this.runUpdateCommand(spec);
|
|
685
|
+
}
|
|
686
|
+
catch (err) {
|
|
687
|
+
// Same report as a normal install failure; the tick's catch counts it and backs off.
|
|
688
|
+
this.reportInstallFailed(latest, err);
|
|
689
|
+
throw err;
|
|
690
|
+
}
|
|
691
|
+
state.lastAppliedVersion = undefined;
|
|
692
|
+
state.pendingRestart = undefined;
|
|
693
|
+
state.lastLoggedAwaitingRestartVersion = undefined;
|
|
694
|
+
this.snapshot = this.buildSnapshot("healthy", `reinstalled-loaded:${latest}`);
|
|
695
|
+
return;
|
|
696
|
+
}
|
|
697
|
+
// 10.x: a restart was requested for a release `latest` no longer names.
|
|
698
|
+
// The A12 wrapper installs whatever the marker says on the next gateway
|
|
699
|
+
// spawn, for any reason, so drop the request and the marker; otherwise
|
|
700
|
+
// the pulled release still loads. Nothing is installed on 10.x.
|
|
701
|
+
const pendingTarget = state.pendingRestart?.toVersion;
|
|
702
|
+
if (this.options.modernHost &&
|
|
703
|
+
pendingTarget !== undefined &&
|
|
704
|
+
parseSemver(pendingTarget) !== null &&
|
|
705
|
+
classifyUpdate(latest, pendingTarget) !== "none") {
|
|
706
|
+
this.options.logger.warn(`[senpi-auto-update] latest was rolled back to v${latest} while a restart for v${pendingTarget} is pending; cancelling it and clearing the update marker`);
|
|
707
|
+
state.pendingRestart = undefined;
|
|
708
|
+
state.lastLoggedAwaitingRestartVersion = undefined;
|
|
709
|
+
await this.clearMarkerQuietly();
|
|
710
|
+
}
|
|
328
711
|
this.snapshot = this.buildSnapshot("healthy", `no-update:${latest}`);
|
|
329
712
|
return;
|
|
330
713
|
}
|
|
331
714
|
if (updateType === "major") {
|
|
715
|
+
state.pendingUpdate = undefined;
|
|
332
716
|
await this.handleMajorAvailable(state, latest, spec);
|
|
333
717
|
return;
|
|
334
718
|
}
|
|
335
719
|
if (!this.isAllowedByPolicy(updateType)) {
|
|
720
|
+
state.pendingUpdate = undefined;
|
|
336
721
|
logger.info(`[senpi-auto-update] update blocked by policy | updateType=${updateType} remoteVersion=${latest}`);
|
|
337
722
|
this.snapshot = this.buildSnapshot("healthy", `policy-blocked:${updateType}:${latest}`);
|
|
338
723
|
return;
|
|
339
724
|
}
|
|
340
725
|
if (state.effectiveMode !== "auto-apply") {
|
|
726
|
+
state.pendingUpdate = undefined;
|
|
341
727
|
await this.logUpdateAvailable(state, latest, updateType, spec);
|
|
342
728
|
return;
|
|
343
729
|
}
|
|
344
|
-
if (state.
|
|
345
|
-
|
|
730
|
+
if (state.applyFailures && state.applyFailures.version !== latest) {
|
|
731
|
+
// A newer release after failed applies of an older one starts a new
|
|
732
|
+
// episode with a fresh spread wait. Keeping the old `eligibleAt` (long
|
|
733
|
+
// past) would restart every such box on its first idle tick, and a
|
|
734
|
+
// fleet-wide apply failure makes that the whole fleet at once.
|
|
735
|
+
state.applyFailures = undefined;
|
|
736
|
+
state.pendingUpdate = undefined;
|
|
737
|
+
}
|
|
738
|
+
// The latch fix: a persisted target equal to `latest` means a restart was
|
|
739
|
+
// requested for it and the loaded version is still behind (classifyUpdate
|
|
740
|
+
// above said so). That is trusted only while the request is recent. On 5.x
|
|
741
|
+
// the runtime installs, so `lastAppliedVersion` names the target; a stale
|
|
742
|
+
// `pendingRestart` from an earlier install must not latch a rolled-back
|
|
743
|
+
// `latest`. On 10.x only `pendingRestart` names it.
|
|
744
|
+
if (state.lastAppliedVersion === latest || (this.options.modernHost && state.pendingRestart?.toVersion === latest)) {
|
|
745
|
+
if (this.restartStillPending(state) || this.awaitingManualRestart(state)) {
|
|
746
|
+
this.handleAlreadyAppliedAwaitingRestart(state, latest);
|
|
747
|
+
return;
|
|
748
|
+
}
|
|
749
|
+
await this.expireUnloadedUpdate(state, latest);
|
|
750
|
+
}
|
|
751
|
+
if ((state.applyFailures?.count ?? 0) >= MAX_APPLY_ATTEMPTS) {
|
|
752
|
+
this.handleParkedUpdate(state, latest);
|
|
753
|
+
return;
|
|
754
|
+
}
|
|
755
|
+
if (this.options.modernHost) {
|
|
756
|
+
await this.runRestartToApply(state, latest, spec);
|
|
346
757
|
return;
|
|
347
758
|
}
|
|
348
759
|
await this.runUpdateTransaction(state, latest, updateType, spec);
|
|
349
760
|
}
|
|
761
|
+
/**
|
|
762
|
+
* Whether the restart requested for the persisted target may still land:
|
|
763
|
+
* its request time (the pending restart's, else the last restart trigger,
|
|
764
|
+
* else the last install) is under {@link RESTART_LOAD_TIMEOUT_MS} old. With
|
|
765
|
+
* no time on record the persisted version is not trusted at all.
|
|
766
|
+
*/
|
|
767
|
+
restartStillPending(state) {
|
|
768
|
+
// The latest of the recorded times: an older trigger for a previous
|
|
769
|
+
// version must not expire a fresh install.
|
|
770
|
+
const times = [state.pendingRestart?.requestedAt, state.lastRestartTriggerAt, state.lastSuccessAt]
|
|
771
|
+
.map((iso) => (iso ? Date.parse(iso) : Number.NaN))
|
|
772
|
+
.filter((ms) => Number.isFinite(ms));
|
|
773
|
+
if (times.length === 0)
|
|
774
|
+
return false;
|
|
775
|
+
return this.now() - Math.max(...times) < RESTART_LOAD_TIMEOUT_MS;
|
|
776
|
+
}
|
|
777
|
+
/**
|
|
778
|
+
* The 5.x latch, as before R2: once this flow installed `latest` (an
|
|
779
|
+
* install or restart time is on record), the box waits for a restart and
|
|
780
|
+
* never runs the in-process install and config-bump restart for that
|
|
781
|
+
* version again, however long it waits. 5.x has no idle gate, so a retry
|
|
782
|
+
* would restart the gateway over an order call or an open approval. A
|
|
783
|
+
* restart trigger that threw is latched too (the warning asks for a manual
|
|
784
|
+
* restart). A persisted applied version with no time at all is still not
|
|
785
|
+
* trusted. 10.x expires its wait after 30 min instead: its retries are
|
|
786
|
+
* idle-gated.
|
|
787
|
+
*/
|
|
788
|
+
awaitingManualRestart(state) {
|
|
789
|
+
if (this.options.modernHost)
|
|
790
|
+
return false;
|
|
791
|
+
return [state.pendingRestart?.requestedAt, state.lastRestartTriggerAt, state.lastSuccessAt].some((iso) => typeof iso === "string" && Number.isFinite(Date.parse(iso)));
|
|
792
|
+
}
|
|
793
|
+
/**
|
|
794
|
+
* {@link MAX_APPLY_ATTEMPTS} restarts for `latest` did not load it. Stop
|
|
795
|
+
* trying this version: no install and no restart until a newer one ships
|
|
796
|
+
* (or a restart loads it some other way). Reported, not thrown, so the poll
|
|
797
|
+
* keeps its normal cadence.
|
|
798
|
+
*/
|
|
799
|
+
handleParkedUpdate(state, latest) {
|
|
800
|
+
const count = state.applyFailures?.count ?? 0;
|
|
801
|
+
const message = `update to v${latest} did not load after ${count} restarts; not retrying this version. ` +
|
|
802
|
+
`The gateway stays on v${this.loadedVersion} until a newer release ships or an operator installs it`;
|
|
803
|
+
if (state.lastLoggedAwaitingRestartVersion !== `parked:${latest}`) {
|
|
804
|
+
this.options.logger.warn(`[senpi-auto-update] ${message}`);
|
|
805
|
+
state.lastLoggedAwaitingRestartVersion = `parked:${latest}`;
|
|
806
|
+
}
|
|
807
|
+
this.snapshot = this.buildSnapshot("degraded", `update-not-loaded:${latest}`, message);
|
|
808
|
+
}
|
|
809
|
+
/**
|
|
810
|
+
* The requested update did not load. Drop the awaiting-restart state and the
|
|
811
|
+
* marker, report `auto_update.apply_failed`, and fail this tick (so the
|
|
812
|
+
* backoff applies). The next tick starts over: a new install and restart on
|
|
813
|
+
* 5.x, a new idle-gated restart request on 10.x. The pending update keeps
|
|
814
|
+
* its eligibility time, so the box does not wait out its spread again.
|
|
815
|
+
*
|
|
816
|
+
* @throws Always.
|
|
817
|
+
*/
|
|
818
|
+
async expireUnloadedUpdate(state, target) {
|
|
819
|
+
const loaded = this.loadedVersion;
|
|
820
|
+
const requestedAt = state.pendingRestart?.requestedAt ?? state.lastRestartTriggerAt ?? "never";
|
|
821
|
+
const message = `update to v${target} did not load: the gateway still runs v${loaded} ` +
|
|
822
|
+
`(restart requested ${requestedAt}); retrying from the start`;
|
|
823
|
+
this.emitEvent(EVENT_NAME.autoUpdateApplyFailed, "warn", {
|
|
824
|
+
body: `Update to v${target} did not load; loaded v${loaded}`,
|
|
825
|
+
attributes: eventAttrs({
|
|
826
|
+
"senpi.auto_update.to_version": target,
|
|
827
|
+
"senpi.auto_update.loaded_version": loaded,
|
|
828
|
+
}),
|
|
829
|
+
});
|
|
830
|
+
this.options.logger.warn(`[senpi-auto-update] ${message}`);
|
|
831
|
+
state.lastAppliedVersion = undefined;
|
|
832
|
+
state.pendingRestart = undefined;
|
|
833
|
+
state.lastLoggedAwaitingRestartVersion = undefined;
|
|
834
|
+
state.applyFailures = {
|
|
835
|
+
version: target,
|
|
836
|
+
count: (state.applyFailures?.version === target ? state.applyFailures.count : 0) + 1,
|
|
837
|
+
};
|
|
838
|
+
await this.clearMarkerQuietly();
|
|
839
|
+
this.snapshot = this.buildSnapshot("degraded", `update-not-loaded:${target}`, message);
|
|
840
|
+
throw new Error(message);
|
|
841
|
+
}
|
|
842
|
+
/**
|
|
843
|
+
* The OpenClaw 10.x apply path (restart-to-apply). Nothing is installed in
|
|
844
|
+
* this process: 10.10 refuses `plugins update` while the gateway runs.
|
|
845
|
+
*
|
|
846
|
+
* 1. Record the pending update once per episode (`firstSeenAt`,
|
|
847
|
+
* `eligibleAt` = first seen + spread offset + jitter).
|
|
848
|
+
* 2. Before `eligibleAt`, wait (`pending-idle:<v>`, blocker `spread`). After
|
|
849
|
+
* it, continue only when the idle gate says idle, or when
|
|
850
|
+
* {@link mustForce} (24 h past eligibility, inside 03:00–05:00 UTC).
|
|
851
|
+
* 3. Build the exact `<name>@<version>` spec, check restart capability,
|
|
852
|
+
* write the marker, and request the restart through `mutateConfigFile`.
|
|
853
|
+
* 4. A failure at any step clears the marker, reports
|
|
854
|
+
* `auto_update.restart_failed` and fails the tick (backoff; `checkNow`
|
|
855
|
+
* answers `failed`). Waiting for the spread or an idle box is not a failure.
|
|
856
|
+
*/
|
|
857
|
+
async runRestartToApply(state, latest, spec) {
|
|
858
|
+
const { logger } = this.options;
|
|
859
|
+
const now = this.now();
|
|
860
|
+
if (!state.pendingUpdate) {
|
|
861
|
+
const firstSeenAt = new Date(now).toISOString();
|
|
862
|
+
const eligible = computeEligibleAt(now, this.options.mid ?? "", this.rand);
|
|
863
|
+
state.pendingUpdate = { version: latest, firstSeenAt, eligibleAt: new Date(eligible).toISOString() };
|
|
864
|
+
logger.info(`[senpi-auto-update] v${latest} seen; restart-to-apply eligible at ${state.pendingUpdate.eligibleAt}`);
|
|
865
|
+
}
|
|
866
|
+
else if (state.pendingUpdate.version !== latest) {
|
|
867
|
+
// A newer release during the wait keeps the box's place in the spread.
|
|
868
|
+
state.pendingUpdate = { ...state.pendingUpdate, version: latest };
|
|
869
|
+
}
|
|
870
|
+
const gate = this.applyGate(now, Date.parse(state.pendingUpdate.eligibleAt));
|
|
871
|
+
const forced = gate.forced;
|
|
872
|
+
if (!gate.open) {
|
|
873
|
+
this.pendingBlockers = gate.blockers;
|
|
874
|
+
this.snapshot = this.buildSnapshot("healthy", `pending-idle:${latest}`);
|
|
875
|
+
this.noteDeferred(state.pendingUpdate, latest, gate.blockers);
|
|
876
|
+
return;
|
|
877
|
+
}
|
|
878
|
+
let exactSpec;
|
|
879
|
+
try {
|
|
880
|
+
exactSpec = buildExactUpdateSpec(spec, latest);
|
|
881
|
+
}
|
|
882
|
+
catch (error) {
|
|
883
|
+
await this.reportModernRestartFailure(latest, formatErr(error), `restart-unavailable:${latest}`);
|
|
884
|
+
return;
|
|
885
|
+
}
|
|
886
|
+
const capability = evaluateModernRestartCapability({
|
|
887
|
+
readLiveConfig: this.options.readLiveConfig,
|
|
888
|
+
mutateConfigFile: this.options.mutateConfigFile,
|
|
889
|
+
pluginId: this.options.pluginId,
|
|
890
|
+
});
|
|
891
|
+
state.restartCapability = capability.capable ? "capable" : "not-capable";
|
|
892
|
+
state.restartCapabilityReasons = capability.reasons;
|
|
893
|
+
state.restartCapabilityMode = capability.mode;
|
|
894
|
+
const mutateConfigFile = this.options.mutateConfigFile;
|
|
895
|
+
const markerPath = this.options.updateMarkerPath;
|
|
896
|
+
if (!capability.capable || !mutateConfigFile) {
|
|
897
|
+
await this.reportModernRestartFailure(latest, capability.reasons.join(", ") || "unknown", `restart-unavailable:${latest}`);
|
|
898
|
+
return;
|
|
899
|
+
}
|
|
900
|
+
if (!markerPath) {
|
|
901
|
+
state.restartCapability = "not-capable";
|
|
902
|
+
state.restartCapabilityReasons = [...capability.reasons, "update-marker-path-unavailable"];
|
|
903
|
+
await this.reportModernRestartFailure(latest, "update-marker-path-unavailable", `restart-unavailable:${latest}`);
|
|
904
|
+
return;
|
|
905
|
+
}
|
|
906
|
+
const requestedAt = new Date(now).toISOString();
|
|
907
|
+
state.lastAttemptAt = requestedAt;
|
|
908
|
+
// The pending restart is saved before the restart is requested: the
|
|
909
|
+
// gateway may stop this process right after, and a lost pending restart
|
|
910
|
+
// would never count toward the retry cap. Rolled back if the request fails.
|
|
911
|
+
const before = {
|
|
912
|
+
pendingRestart: state.pendingRestart,
|
|
913
|
+
lastRestartTriggerAt: state.lastRestartTriggerAt,
|
|
914
|
+
lastRestartTriggerMode: state.lastRestartTriggerMode,
|
|
915
|
+
};
|
|
916
|
+
state.pendingRestart = {
|
|
917
|
+
fromVersion: this.loadedVersion,
|
|
918
|
+
toVersion: latest,
|
|
919
|
+
requestedAt,
|
|
920
|
+
mode: "restart-to-apply",
|
|
921
|
+
};
|
|
922
|
+
state.lastRestartTriggerAt = requestedAt;
|
|
923
|
+
state.lastRestartTriggerMode = "restart-to-apply";
|
|
924
|
+
const rollBack = () => {
|
|
925
|
+
state.pendingRestart = before.pendingRestart;
|
|
926
|
+
state.lastRestartTriggerAt = before.lastRestartTriggerAt;
|
|
927
|
+
state.lastRestartTriggerMode = before.lastRestartTriggerMode;
|
|
928
|
+
};
|
|
929
|
+
try {
|
|
930
|
+
await this.store.save(state);
|
|
931
|
+
}
|
|
932
|
+
catch (error) {
|
|
933
|
+
// Not a restart-capability problem: the state dir is not writable. No
|
|
934
|
+
// restart without a saved pending restart, or the retry cap is lost.
|
|
935
|
+
rollBack();
|
|
936
|
+
await this.reportModernRestartFailure(latest, `state-save-failed:${formatErr(error)}`, `state-save-failed:${latest}`, error);
|
|
937
|
+
return;
|
|
938
|
+
}
|
|
939
|
+
try {
|
|
940
|
+
await writeUpdateMarker(markerPath, { spec: exactSpec, targetVersion: latest, requestedAt });
|
|
941
|
+
// Read the gate again after the awaits above: a money call or a user
|
|
942
|
+
// turn that started meanwhile holds the restart, as at the first read.
|
|
943
|
+
const again = this.readIdle(this.now());
|
|
944
|
+
if (!again.idle && !(forced && onlyOverridable(again.blockers))) {
|
|
945
|
+
rollBack();
|
|
946
|
+
await this.clearMarkerQuietly();
|
|
947
|
+
this.pendingBlockers = [...again.blockers];
|
|
948
|
+
this.snapshot = this.buildSnapshot("healthy", `pending-idle:${latest}`);
|
|
949
|
+
if (state.pendingUpdate)
|
|
950
|
+
this.noteDeferred(state.pendingUpdate, latest, again.blockers);
|
|
951
|
+
return;
|
|
952
|
+
}
|
|
953
|
+
await triggerGatewayRestartViaMutate({ mutateConfigFile, pluginId: this.options.pluginId, triggerValue: requestedAt });
|
|
954
|
+
}
|
|
955
|
+
catch (error) {
|
|
956
|
+
rollBack();
|
|
957
|
+
const reason = formatErr(error);
|
|
958
|
+
state.restartCapability = "not-capable";
|
|
959
|
+
state.restartCapabilityReasons = [...state.restartCapabilityReasons, `restart-trigger-failed:${reason}`];
|
|
960
|
+
await this.reportModernRestartFailure(latest, reason, `restart-request-failed:${latest}`, error);
|
|
961
|
+
return;
|
|
962
|
+
}
|
|
963
|
+
this.emitEvent(EVENT_NAME.autoUpdateRestartRequested, "info", {
|
|
964
|
+
body: `Restart requested to apply v${latest} (from v${this.loadedVersion}${forced ? ", forced after 24 h busy" : ""})`,
|
|
965
|
+
attributes: eventAttrs({
|
|
966
|
+
"senpi.auto_update.from_version": this.loadedVersion,
|
|
967
|
+
"senpi.auto_update.to_version": latest,
|
|
968
|
+
"senpi.auto_update.restart_mode": "restart-to-apply",
|
|
969
|
+
"senpi.auto_update.forced": forced,
|
|
970
|
+
"senpi.auto_update.blockers": forced ? blockerList(gate.blockers) : undefined,
|
|
971
|
+
}),
|
|
972
|
+
});
|
|
973
|
+
logger.info(`[senpi-auto-update] marker written (${exactSpec}); gateway restart requested to apply v${latest}` +
|
|
974
|
+
(forced ? ` (forced: busy for 24 h past eligibility; blockers ${gate.blockers.join(",")})` : ""));
|
|
975
|
+
this.snapshot = this.buildSnapshot("healthy", `updated-restart-requested:${latest}`);
|
|
976
|
+
}
|
|
977
|
+
/** The idle gate's answer; a throwing gate counts as busy. */
|
|
978
|
+
readIdle(now) {
|
|
979
|
+
const gate = this.options.idleGate;
|
|
980
|
+
if (!gate)
|
|
981
|
+
return { idle: true, blockers: [] };
|
|
982
|
+
try {
|
|
983
|
+
return gate.isIdle(now);
|
|
984
|
+
}
|
|
985
|
+
catch (error) {
|
|
986
|
+
this.options.logger.warn(`[senpi-auto-update] idle gate threw; treating the box as busy | err=${formatErr(error)}`);
|
|
987
|
+
return { idle: false, blockers: ["idle-gate-error"] };
|
|
988
|
+
}
|
|
989
|
+
}
|
|
990
|
+
/**
|
|
991
|
+
* Clear the marker, report `auto_update.restart_failed` for the 10.x path
|
|
992
|
+
* (snapshot degraded), and fail the tick: no restart was requested, so the
|
|
993
|
+
* tick must not count as a success. The throw makes `tick()` back off and
|
|
994
|
+
* `checkNow()` answer `failed`.
|
|
995
|
+
*
|
|
996
|
+
* @throws Always, with `reason`.
|
|
997
|
+
*/
|
|
998
|
+
async reportModernRestartFailure(latest, reason, lastAction, error) {
|
|
999
|
+
await this.clearMarkerQuietly();
|
|
1000
|
+
this.emitEvent(EVENT_NAME.autoUpdateRestartFailed, "error", {
|
|
1001
|
+
body: `Restart to apply v${latest} was not requested: ${reason}`,
|
|
1002
|
+
attributes: eventAttrs({
|
|
1003
|
+
"senpi.auto_update.to_version": latest,
|
|
1004
|
+
"senpi.auto_update.reason": reason,
|
|
1005
|
+
}),
|
|
1006
|
+
...(error === undefined ? {} : { error }),
|
|
1007
|
+
});
|
|
1008
|
+
this.options.logger.warn(`[senpi-auto-update] v${latest} is available but the gateway restart that applies it was not requested. ` +
|
|
1009
|
+
`Nothing was installed; the loaded version stays v${this.loadedVersion}. Reason(s): ${reason}`);
|
|
1010
|
+
this.snapshot = this.buildSnapshot("degraded", lastAction, reason);
|
|
1011
|
+
throw new Error(`restart to apply v${latest} was not requested: ${reason}`);
|
|
1012
|
+
}
|
|
350
1013
|
/**
|
|
351
1014
|
* `currentVersion` is a compile-time constant frozen for the life of the
|
|
352
1015
|
* process, so it cannot reflect an install that already happened this
|
|
@@ -359,19 +1022,30 @@ export class AutoUpdateCoordinator {
|
|
|
359
1022
|
* so this branch stops being reached on its own.
|
|
360
1023
|
*/
|
|
361
1024
|
handleAlreadyAppliedAwaitingRestart(state, latest) {
|
|
1025
|
+
const modern = this.options.modernHost === true;
|
|
362
1026
|
if (state.lastLoggedAwaitingRestartVersion !== latest) {
|
|
363
|
-
this.options.logger.info(
|
|
1027
|
+
this.options.logger.info(modern
|
|
1028
|
+
? `[senpi-auto-update] restart to apply v${latest} requested, awaiting gateway restart | not requesting again`
|
|
1029
|
+
: `[senpi-auto-update] update to v${latest} already applied, awaiting gateway restart to take effect | skipping reinstall`);
|
|
364
1030
|
state.lastLoggedAwaitingRestartVersion = latest;
|
|
365
1031
|
}
|
|
366
1032
|
// Fired on every guard hit (not deduped like the log line above) — an operator watching for
|
|
367
1033
|
// "how long has this box been stuck awaiting restart" needs every occurrence, not just the first.
|
|
368
1034
|
this.emitEvent(EVENT_NAME.autoUpdateAwaitingRestart, "warn", {
|
|
369
|
-
body:
|
|
1035
|
+
body: modern
|
|
1036
|
+
? `Restart to apply v${latest} requested, still awaiting gateway restart`
|
|
1037
|
+
: `Update to v${latest} already applied, still awaiting gateway restart`,
|
|
370
1038
|
attributes: eventAttrs({
|
|
371
1039
|
"senpi.auto_update.to_version": latest,
|
|
372
1040
|
}),
|
|
373
1041
|
});
|
|
374
|
-
|
|
1042
|
+
// On 10.x nothing is installed until the wrapper consumes the marker, so
|
|
1043
|
+
// "already applied" would be false there.
|
|
1044
|
+
this.snapshot = this.buildSnapshot("degraded", modern ? `awaiting-restart:${latest}` : `already-applied-awaiting-restart:${latest}`);
|
|
1045
|
+
}
|
|
1046
|
+
/** OpenClaw 10.x refuses `plugins update` while the gateway runs, so a manual instruction says so. */
|
|
1047
|
+
manualUpdateSuffix() {
|
|
1048
|
+
return this.options.modernHost ? " with the gateway stopped" : "";
|
|
375
1049
|
}
|
|
376
1050
|
isAllowedByPolicy(kind) {
|
|
377
1051
|
if (kind === "patch")
|
|
@@ -394,7 +1068,7 @@ export class AutoUpdateCoordinator {
|
|
|
394
1068
|
});
|
|
395
1069
|
this.options.logger.info(`[senpi-auto-update] Senpi update available: v${latest} (major). ` +
|
|
396
1070
|
`Auto-update will not apply major versions. ` +
|
|
397
|
-
`Review changelog and run: openclaw plugins update ${spec}`);
|
|
1071
|
+
`Review changelog and run: openclaw plugins update ${spec}${this.manualUpdateSuffix()}`);
|
|
398
1072
|
state.lastNotifiedMajorVersion = latest;
|
|
399
1073
|
this.snapshot = this.buildSnapshot("healthy", `major-notified:${latest}`);
|
|
400
1074
|
}
|
|
@@ -412,10 +1086,27 @@ export class AutoUpdateCoordinator {
|
|
|
412
1086
|
});
|
|
413
1087
|
this.options.logger.info(`[senpi-auto-update] Senpi update available: v${latest} (${kind}). ` +
|
|
414
1088
|
`Auto-update mode is notify-only. ` +
|
|
415
|
-
`Run: openclaw plugins update ${spec}`);
|
|
1089
|
+
`Run: openclaw plugins update ${spec}${this.manualUpdateSuffix()}`);
|
|
416
1090
|
state.lastNotifiedUpdateVersion = latest;
|
|
417
1091
|
this.snapshot = this.buildSnapshot("healthy", `update-notified:${kind}:${latest}`);
|
|
418
1092
|
}
|
|
1093
|
+
/**
|
|
1094
|
+
* The `auto_update.install_failed` event and warn line for an install of
|
|
1095
|
+
* `latest` that threw. Shared by the normal install and the 5.x reinstall of
|
|
1096
|
+
* a rolled-back `latest`; the caller rethrows to the tick's catch.
|
|
1097
|
+
*/
|
|
1098
|
+
reportInstallFailed(latest, err) {
|
|
1099
|
+
const reason = formatErr(err);
|
|
1100
|
+
this.emitEvent(EVENT_NAME.autoUpdateInstallFailed, "error", {
|
|
1101
|
+
body: `Install failed: ${this.options.currentVersion} -> v${latest}: ${reason}`,
|
|
1102
|
+
attributes: eventAttrs({
|
|
1103
|
+
"senpi.auto_update.to_version": latest,
|
|
1104
|
+
}),
|
|
1105
|
+
error: err,
|
|
1106
|
+
});
|
|
1107
|
+
this.options.logger.warn(`[senpi-auto-update] Senpi auto-update failed: ${this.options.currentVersion} -> ${latest}. ` +
|
|
1108
|
+
`Reason: ${reason}. Will retry after backoff.`);
|
|
1109
|
+
}
|
|
419
1110
|
/**
|
|
420
1111
|
* Install the update, evaluate restart capability, then either trigger a
|
|
421
1112
|
* gateway restart or log that a manual restart is needed.
|
|
@@ -443,16 +1134,7 @@ export class AutoUpdateCoordinator {
|
|
|
443
1134
|
await this.runUpdateCommand(spec);
|
|
444
1135
|
}
|
|
445
1136
|
catch (err) {
|
|
446
|
-
|
|
447
|
-
this.emitEvent(EVENT_NAME.autoUpdateInstallFailed, "error", {
|
|
448
|
-
body: `Install failed: ${this.options.currentVersion} -> v${latest}: ${reason}`,
|
|
449
|
-
attributes: eventAttrs({
|
|
450
|
-
"senpi.auto_update.to_version": latest,
|
|
451
|
-
}),
|
|
452
|
-
error: err,
|
|
453
|
-
});
|
|
454
|
-
this.options.logger.warn(`[senpi-auto-update] Senpi auto-update failed: ${this.options.currentVersion} -> ${latest}. ` +
|
|
455
|
-
`Reason: ${reason}. Will retry after backoff.`);
|
|
1137
|
+
this.reportInstallFailed(latest, err);
|
|
456
1138
|
throw err;
|
|
457
1139
|
}
|
|
458
1140
|
// Step 2: record success before evaluating restart (no runtimes stopped yet).
|
|
@@ -495,6 +1177,12 @@ export class AutoUpdateCoordinator {
|
|
|
495
1177
|
});
|
|
496
1178
|
state.lastRestartTriggerAt = nowIso;
|
|
497
1179
|
state.lastRestartTriggerMode = capability.mode ?? "unknown";
|
|
1180
|
+
state.pendingRestart = {
|
|
1181
|
+
fromVersion: this.loadedVersion,
|
|
1182
|
+
toVersion: latest,
|
|
1183
|
+
requestedAt: nowIso,
|
|
1184
|
+
mode: state.lastRestartTriggerMode,
|
|
1185
|
+
};
|
|
498
1186
|
this.emitEvent(EVENT_NAME.autoUpdateApplied, "info", {
|
|
499
1187
|
body: `Update applied: ${this.options.currentVersion} -> v${latest}; restart requested (${state.lastRestartTriggerMode})`,
|
|
500
1188
|
attributes: eventAttrs({
|
|
@@ -553,6 +1241,7 @@ export class AutoUpdateCoordinator {
|
|
|
553
1241
|
restartCapabilityReasons: state.restartCapabilityReasons,
|
|
554
1242
|
lastRestartTriggerAt: state.lastRestartTriggerAt ?? null,
|
|
555
1243
|
lastRestartTriggerMode: state.lastRestartTriggerMode ?? null,
|
|
1244
|
+
pendingBlockers: this.pendingBlockers,
|
|
556
1245
|
};
|
|
557
1246
|
}
|
|
558
1247
|
async persistState(state) {
|
|
@@ -573,6 +1262,10 @@ async function defaultRunUpdateCommand(packageName) {
|
|
|
573
1262
|
throw new Error(result.stderr || `openclaw plugins update failed with code ${result.code}`);
|
|
574
1263
|
}
|
|
575
1264
|
}
|
|
1265
|
+
/** A blocker set as one stable value: de-duplicated, sorted, comma-joined. */
|
|
1266
|
+
function blockerList(blockers) {
|
|
1267
|
+
return [...new Set(blockers)].sort().join(",");
|
|
1268
|
+
}
|
|
576
1269
|
function formatErr(error) {
|
|
577
1270
|
return error instanceof Error ? error.message : String(error);
|
|
578
1271
|
}
|