@senpi-ai/runtime 3.0.179 → 3.0.188

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. package/dist/actions/llm-decision/openclaw-llm-decision.d.ts +8 -0
  2. package/dist/actions/llm-decision/openclaw-llm-decision.d.ts.map +1 -1
  3. package/dist/actions/llm-decision/openclaw-llm-decision.js +56 -1
  4. package/dist/actions/llm-decision/openclaw-llm-decision.js.map +1 -1
  5. package/dist/approvals/approval-store.d.ts +37 -7
  6. package/dist/approvals/approval-store.d.ts.map +1 -1
  7. package/dist/approvals/approval-store.js +127 -23
  8. package/dist/approvals/approval-store.js.map +1 -1
  9. package/dist/approvals/telemetry.d.ts +28 -3
  10. package/dist/approvals/telemetry.d.ts.map +1 -1
  11. package/dist/approvals/telemetry.js +8 -4
  12. package/dist/approvals/telemetry.js.map +1 -1
  13. package/dist/approvals/tool-approval-gate.d.ts +85 -0
  14. package/dist/approvals/tool-approval-gate.d.ts.map +1 -1
  15. package/dist/approvals/tool-approval-gate.js +197 -15
  16. package/dist/approvals/tool-approval-gate.js.map +1 -1
  17. package/dist/approvals/turn-context.d.ts +296 -0
  18. package/dist/approvals/turn-context.d.ts.map +1 -0
  19. package/dist/approvals/turn-context.js +594 -0
  20. package/dist/approvals/turn-context.js.map +1 -0
  21. package/dist/cli/guide-commands.d.ts.map +1 -1
  22. package/dist/cli/guide-commands.js +18 -3
  23. package/dist/cli/guide-commands.js.map +1 -1
  24. package/dist/cli/senpi-commands.d.ts.map +1 -1
  25. package/dist/cli/senpi-commands.js +80 -5
  26. package/dist/cli/senpi-commands.js.map +1 -1
  27. package/dist/deploy/job.d.ts +32 -0
  28. package/dist/deploy/job.d.ts.map +1 -1
  29. package/dist/deploy/job.js +72 -7
  30. package/dist/deploy/job.js.map +1 -1
  31. package/dist/deploy/register.d.ts +12 -0
  32. package/dist/deploy/register.d.ts.map +1 -1
  33. package/dist/deploy/register.js +85 -2
  34. package/dist/deploy/register.js.map +1 -1
  35. package/dist/dsl/monitor/dsl-failure-events.d.ts +38 -2
  36. package/dist/dsl/monitor/dsl-failure-events.d.ts.map +1 -1
  37. package/dist/dsl/monitor/dsl-failure-events.js +67 -4
  38. package/dist/dsl/monitor/dsl-failure-events.js.map +1 -1
  39. package/dist/dsl/monitor/process-one-position.d.ts.map +1 -1
  40. package/dist/dsl/monitor/process-one-position.js +25 -1
  41. package/dist/dsl/monitor/process-one-position.js.map +1 -1
  42. package/dist/host/openclaw-host.d.ts +138 -0
  43. package/dist/host/openclaw-host.d.ts.map +1 -0
  44. package/dist/host/openclaw-host.js +177 -0
  45. package/dist/host/openclaw-host.js.map +1 -0
  46. package/dist/host/plugin-logger.d.ts +99 -0
  47. package/dist/host/plugin-logger.d.ts.map +1 -0
  48. package/dist/host/plugin-logger.js +142 -0
  49. package/dist/host/plugin-logger.js.map +1 -0
  50. package/dist/index.d.ts +56 -7
  51. package/dist/index.d.ts.map +1 -1
  52. package/dist/index.js +695 -161
  53. package/dist/index.js.map +1 -1
  54. package/dist/runtime/auto-update/coordinator.d.ts +220 -0
  55. package/dist/runtime/auto-update/coordinator.d.ts.map +1 -1
  56. package/dist/runtime/auto-update/coordinator.js +715 -22
  57. package/dist/runtime/auto-update/coordinator.js.map +1 -1
  58. package/dist/runtime/auto-update/gateway-restart.d.ts +60 -1
  59. package/dist/runtime/auto-update/gateway-restart.d.ts.map +1 -1
  60. package/dist/runtime/auto-update/gateway-restart.js +95 -21
  61. package/dist/runtime/auto-update/gateway-restart.js.map +1 -1
  62. package/dist/runtime/auto-update/idle-gate.d.ts +175 -0
  63. package/dist/runtime/auto-update/idle-gate.d.ts.map +1 -0
  64. package/dist/runtime/auto-update/idle-gate.js +229 -0
  65. package/dist/runtime/auto-update/idle-gate.js.map +1 -0
  66. package/dist/runtime/auto-update/install-results.d.ts +130 -0
  67. package/dist/runtime/auto-update/install-results.d.ts.map +1 -0
  68. package/dist/runtime/auto-update/install-results.js +264 -0
  69. package/dist/runtime/auto-update/install-results.js.map +1 -0
  70. package/dist/runtime/auto-update/money-call-tracker.d.ts +36 -0
  71. package/dist/runtime/auto-update/money-call-tracker.d.ts.map +1 -0
  72. package/dist/runtime/auto-update/money-call-tracker.js +129 -0
  73. package/dist/runtime/auto-update/money-call-tracker.js.map +1 -0
  74. package/dist/runtime/auto-update/register-idle-gate.d.ts +70 -0
  75. package/dist/runtime/auto-update/register-idle-gate.d.ts.map +1 -0
  76. package/dist/runtime/auto-update/register-idle-gate.js +127 -0
  77. package/dist/runtime/auto-update/register-idle-gate.js.map +1 -0
  78. package/dist/runtime/auto-update/run-update-command.d.ts +13 -0
  79. package/dist/runtime/auto-update/run-update-command.d.ts.map +1 -1
  80. package/dist/runtime/auto-update/run-update-command.js +18 -0
  81. package/dist/runtime/auto-update/run-update-command.js.map +1 -1
  82. package/dist/runtime/auto-update/state-store.d.ts +14 -1
  83. package/dist/runtime/auto-update/state-store.d.ts.map +1 -1
  84. package/dist/runtime/auto-update/state-store.js +37 -3
  85. package/dist/runtime/auto-update/state-store.js.map +1 -1
  86. package/dist/runtime/auto-update/types.d.ts +52 -0
  87. package/dist/runtime/auto-update/types.d.ts.map +1 -1
  88. package/dist/runtime/auto-update/types.js.map +1 -1
  89. package/dist/runtime/auto-update/update-marker.d.ts +95 -0
  90. package/dist/runtime/auto-update/update-marker.d.ts.map +1 -0
  91. package/dist/runtime/auto-update/update-marker.js +185 -0
  92. package/dist/runtime/auto-update/update-marker.js.map +1 -0
  93. package/dist/runtime/auto-update/update-schedule.d.ts +67 -0
  94. package/dist/runtime/auto-update/update-schedule.d.ts.map +1 -0
  95. package/dist/runtime/auto-update/update-schedule.js +91 -0
  96. package/dist/runtime/auto-update/update-schedule.js.map +1 -0
  97. package/dist/runtime/decision-errors.d.ts +7 -0
  98. package/dist/runtime/decision-errors.d.ts.map +1 -1
  99. package/dist/runtime/decision-errors.js +15 -0
  100. package/dist/runtime/decision-errors.js.map +1 -1
  101. package/dist/runtime/process-state.d.ts +168 -0
  102. package/dist/runtime/process-state.d.ts.map +1 -0
  103. package/dist/runtime/process-state.js +81 -0
  104. package/dist/runtime/process-state.js.map +1 -0
  105. package/dist/runtime/reconcile-state.d.ts.map +1 -1
  106. package/dist/runtime/reconcile-state.js +18 -4
  107. package/dist/runtime/reconcile-state.js.map +1 -1
  108. package/dist/runtime/strategy-registry.d.ts +16 -0
  109. package/dist/runtime/strategy-registry.d.ts.map +1 -1
  110. package/dist/runtime/strategy-registry.js +7 -3
  111. package/dist/runtime/strategy-registry.js.map +1 -1
  112. package/dist/runtime/update-apply-deps.d.ts +5 -3
  113. package/dist/runtime/update-apply-deps.d.ts.map +1 -1
  114. package/dist/runtime/update-apply-deps.js +15 -3
  115. package/dist/runtime/update-apply-deps.js.map +1 -1
  116. package/dist/senpi/client.d.ts +29 -0
  117. package/dist/senpi/client.d.ts.map +1 -1
  118. package/dist/senpi/client.js +71 -2
  119. package/dist/senpi/client.js.map +1 -1
  120. package/dist/telemetry/metrics.d.ts +15 -1
  121. package/dist/telemetry/metrics.d.ts.map +1 -1
  122. package/dist/telemetry/metrics.js +25 -1
  123. package/dist/telemetry/metrics.js.map +1 -1
  124. package/dist/telemetry/otel-sdk.d.ts +12 -0
  125. package/dist/telemetry/otel-sdk.d.ts.map +1 -1
  126. package/dist/telemetry/otel-sdk.js +47 -1
  127. package/dist/telemetry/otel-sdk.js.map +1 -1
  128. package/dist/telemetry/spans/span-hooks.d.ts +18 -1
  129. package/dist/telemetry/spans/span-hooks.d.ts.map +1 -1
  130. package/dist/telemetry/spans/span-hooks.js +306 -96
  131. package/dist/telemetry/spans/span-hooks.js.map +1 -1
  132. package/dist/telemetry/tool-plane.d.ts +2 -0
  133. package/dist/telemetry/tool-plane.d.ts.map +1 -1
  134. package/dist/telemetry/tool-plane.js +1 -0
  135. package/dist/telemetry/tool-plane.js.map +1 -1
  136. package/dist/test-setup.js +7 -1
  137. package/dist/test-setup.js.map +1 -1
  138. package/dist/tool-surface/crib-cache.d.ts +21 -0
  139. package/dist/tool-surface/crib-cache.d.ts.map +1 -0
  140. package/dist/tool-surface/crib-cache.js +54 -0
  141. package/dist/tool-surface/crib-cache.js.map +1 -0
  142. package/dist/tool-surface/crib-hook.d.ts +78 -0
  143. package/dist/tool-surface/crib-hook.d.ts.map +1 -0
  144. package/dist/tool-surface/crib-hook.js +225 -0
  145. package/dist/tool-surface/crib-hook.js.map +1 -0
  146. package/dist/tool-surface/crib-sheet.d.ts +56 -0
  147. package/dist/tool-surface/crib-sheet.d.ts.map +1 -0
  148. package/dist/tool-surface/crib-sheet.js +129 -0
  149. package/dist/tool-surface/crib-sheet.js.map +1 -0
  150. package/dist/tool-surface/mcp-tool-source.d.ts +64 -0
  151. package/dist/tool-surface/mcp-tool-source.d.ts.map +1 -0
  152. package/dist/tool-surface/mcp-tool-source.js +189 -0
  153. package/dist/tool-surface/mcp-tool-source.js.map +1 -0
  154. package/dist/utils/event-catalog.d.ts +144 -6
  155. package/dist/utils/event-catalog.d.ts.map +1 -1
  156. package/dist/utils/event-catalog.js +25 -0
  157. package/dist/utils/event-catalog.js.map +1 -1
  158. package/dist/widgets/tool.d.ts +9 -0
  159. package/dist/widgets/tool.d.ts.map +1 -1
  160. package/dist/widgets/tool.js +1 -0
  161. package/dist/widgets/tool.js.map +1 -1
  162. package/openclaw.plugin.json +11 -1
  163. package/package.json +5 -2
@@ -1,10 +1,13 @@
1
1
  import { runCommand } from "./command-runner.js";
2
- import { evaluateGatewayRestartCapability } from "./gateway-restart.js";
2
+ import { evaluateGatewayRestartCapability, evaluateModernRestartCapability, triggerGatewayRestartViaMutate, } from "./gateway-restart.js";
3
3
  import { classifyUpdate, parseSemver } from "./policy.js";
4
4
  import { evaluatePreflight } from "./preflight.js";
5
5
  import { resolveConfiguredSpec } from "./resolve-package-name.js";
6
6
  import { AutoUpdateStateStore } from "./state-store.js";
7
7
  import { createDefaultAutoUpdateState, } from "./types.js";
8
+ import { consumeInstallResults, describeInstallResult, installResultLevel, } from "./install-results.js";
9
+ import { buildExactUpdateSpec, clearUpdateMarker, readUpdateMarker, writeUpdateMarker } from "./update-marker.js";
10
+ import { eligibleAt as computeEligibleAt, mustForce } from "./update-schedule.js";
8
11
  import { NpmVersionSource } from "./version-source.js";
9
12
  import { EVENT_NAME, eventAttrs } from "../../utils/event-catalog.js";
10
13
  /**
@@ -22,6 +25,51 @@ const CHECK_NOW_REASON = {
22
25
  disabledInConfig: "Auto-update is turned off in config; no check was run.",
23
26
  coordinatorStopped: "The auto-update coordinator is not running; no check was run.",
24
27
  };
28
+ /**
29
+ * How long a requested restart may take to load its target before the
30
+ * awaiting-restart state is treated as failed (the 3.0.171 latch fix). A 10.x
31
+ * restart-to-apply measured about 71 s on Box A, with starts up to 93 s; 30 min
32
+ * leaves room for the gateway's own deferral (forced after 300 s) and a slow
33
+ * offline install.
34
+ */
35
+ export const RESTART_LOAD_TIMEOUT_MS = 30 * 60_000;
36
+ /**
37
+ * Restarts requested for one target that may fail to load before the box
38
+ * stops trying that version and waits for a newer one. Bounds the loop of a
39
+ * version that can never install (a blocked scan, a yanked version, a wrapper
40
+ * without the marker step): without it an idle box would restart about every
41
+ * 80 min forever.
42
+ */
43
+ export const MAX_APPLY_ATTEMPTS = 3;
44
+ /**
45
+ * How often a 10.x box that is `pending-idle` re-reads its apply gate (the
46
+ * spread time and the idle gate) between registry polls. The idle gate has no
47
+ * change notification, so the change is caught by this re-read; when the gate
48
+ * opens, an ordinary check runs at once instead of up to one poll interval
49
+ * (60 min by default) later. The re-read itself is local: no registry call,
50
+ * no `onTickComplete`.
51
+ */
52
+ export const PENDING_IDLE_RECHECK_MS = 3 * 60_000;
53
+ /**
54
+ * The only idle-gate blockers the 24 h forced restart may override: a busy
55
+ * user (`user-run`, `recent-message`) and a gate that just started
56
+ * (`recent-start`, no known activity at all). Every other blocker holds it, money
57
+ * first: an order call in flight (the restart would cut it), an open trade
58
+ * approval (it would be cancelled), a running `senpi deploy` (it would leave a
59
+ * funded wallet with no runtime), a queued user message, and a gate that
60
+ * threw (it cannot rule any of those out).
61
+ */
62
+ const FORCE_MAY_OVERRIDE = new Set(["user-run", "recent-message", "recent-start"]);
63
+ /** Whether every blocker in `blockers` is one the forced restart may override. */
64
+ function onlyOverridable(blockers) {
65
+ return blockers.every((b) => FORCE_MAY_OVERRIDE.has(b));
66
+ }
67
+ /** Whether `loaded` is at or past `target` (major.minor.patch; both must parse). */
68
+ function isAtLeast(loaded, target) {
69
+ if (!parseSemver(loaded) || !parseSemver(target))
70
+ return false;
71
+ return classifyUpdate(loaded, target) === "none";
72
+ }
25
73
  /** Cap a registry-supplied string before it reaches a log line or an event body. */
26
74
  function summarizeForLog(value, maxLength = 120) {
27
75
  const collapsed = value.replace(/\s+/g, " ").trim();
@@ -49,6 +97,8 @@ export class AutoUpdateCoordinator {
49
97
  versionSource;
50
98
  runUpdateCommand;
51
99
  timer = null;
100
+ /** 10.x `pending-idle` gate re-read; see {@link PENDING_IDLE_RECHECK_MS}. */
101
+ idleRecheckTimer = null;
52
102
  running = false;
53
103
  stopped = true;
54
104
  runToken = 0;
@@ -58,8 +108,26 @@ export class AutoUpdateCoordinator {
58
108
  /** Spec the last resolve used. Seeded in the constructor, so the snapshot
59
109
  * reports a line before the first tick. */
60
110
  updateSpec;
111
+ /** Why the modern path is still waiting; reset every tick. See {@link AutoUpdateSnapshot.pendingBlockers}. */
112
+ pendingBlockers = [];
113
+ now;
114
+ rand;
115
+ loadedVersion;
116
+ /**
117
+ * 10.x: `<version>|<sorted blockers>` of the last `auto_update.deferred`, so the event fires on
118
+ * entering `pending-idle` and on a change only. Cleared when a tick ends outside `pending-idle` or fails.
119
+ * In memory: a new gateway process reports a wait that is still on once.
120
+ */
121
+ lastDeferredKey = null;
122
+ /** The install-results report the last {@link start} began; see {@link whenInstallResultsReported}. */
123
+ installResultsReport = Promise.resolve();
124
+ /** The 10.x `auto_update.applied` report the last {@link start} began; see {@link whenAppliedReported}. */
125
+ appliedReport = Promise.resolve();
61
126
  constructor(options) {
62
127
  this.options = options;
128
+ this.now = options.now ?? Date.now;
129
+ this.rand = options.rand ?? Math.random;
130
+ this.loadedVersion = options.loadedVersion ?? options.currentVersion;
63
131
  this.store = new AutoUpdateStateStore(options.stateDir, options.logger);
64
132
  this.versionSource = options.versionSource ?? new NpmVersionSource();
65
133
  this.runUpdateCommand = options.runUpdateCommand ?? defaultRunUpdateCommand;
@@ -79,6 +147,8 @@ export class AutoUpdateCoordinator {
79
147
  this.stopped = true;
80
148
  this.snapshot = this.buildSnapshot("disabled", "auto-update-disabled");
81
149
  logger.info(`[senpi-auto-update] disabled by config (pluginEnabled=${String(this.options.enabled)}, configEnabled=${String(config.enabled)})`);
150
+ // Operator markers and the boot floor still install on a box with auto-update off.
151
+ this.startInstallResultsReport(null);
82
152
  return;
83
153
  }
84
154
  logger.info(`[senpi-auto-update] starting | localVersion=${this.options.currentVersion} package=${this.options.packageName} spec=${this.resolveUpdateSpec()} mode=${config.mode} pollInterval=${config.pollIntervalMinutes}m startupDelay=${config.startupDelaySeconds}s failureBackoff=${config.failureBackoffMinutes}m maxBackoff=${config.maxBackoffMinutes}m`);
@@ -90,17 +160,219 @@ export class AutoUpdateCoordinator {
90
160
  logger.info(`[senpi-auto-update] preflight result | ok=${String(preflight.ok)} effectiveMode=${preflight.effectiveMode} reasons=${preflight.reasons.join(",") || "none"}`);
91
161
  this.state = await this.store.load(this.options.currentVersion, preflight.effectiveMode);
92
162
  this.state.effectiveMode = preflight.effectiveMode;
163
+ const reconciled = await this.reconcileLoadedVersion(this.state);
93
164
  try {
94
165
  await this.store.save(this.state);
95
166
  }
96
167
  catch (error) {
97
168
  this.options.logger.warn("[senpi-auto-update] failed to persist initial state", error);
98
169
  }
99
- this.snapshot = this.buildSnapshot("healthy", "started");
170
+ this.startInstallResultsReport(this.state);
171
+ this.snapshot = this.buildSnapshot("healthy", reconciled ?? "started");
100
172
  const delayMs = config.startupDelaySeconds * 1000;
101
173
  logger.info(`[senpi-auto-update] first check scheduled | delayMs=${delayMs} nextCheckAt=${new Date(Date.now() + delayMs).toISOString()}`);
102
174
  this.scheduleNext(delayMs, this.runToken);
103
175
  }
176
+ /**
177
+ * Startup half of the latch fix: compare what this process loaded with what
178
+ * a previous process asked for.
179
+ *
180
+ * When the loaded version reaches the requested restart's target, the update
181
+ * took effect: the pending restart and pending update are cleared, a stale
182
+ * install marker is removed, and on a 10.x host `auto_update.applied` fires
183
+ * (nothing was installed before this boot there). On 5.x `applied` already
184
+ * fired when the restart was requested, so it is not counted twice.
185
+ *
186
+ * A target the loaded version has not reached is left for the tick, which
187
+ * expires it once the request is older than {@link RESTART_LOAD_TIMEOUT_MS}.
188
+ *
189
+ * @returns `updated:<v>` when a requested update is now loaded, else `undefined`.
190
+ */
191
+ async reconcileLoadedVersion(state) {
192
+ const loaded = this.loadedVersion;
193
+ let action;
194
+ const pending = state.pendingRestart;
195
+ if (pending && isAtLeast(loaded, pending.toVersion)) {
196
+ action = `updated:${pending.toVersion}`;
197
+ this.options.logger.info(`[senpi-auto-update] update to v${pending.toVersion} is loaded (requested ${pending.requestedAt}, mode ${pending.mode})`);
198
+ // A 5.x pending restart already reported `applied` when it was requested.
199
+ if (this.options.modernHost && pending.mode === "restart-to-apply") {
200
+ // How long the apply took: request to this boot (stop + wrapper install + start), and
201
+ // first seen to this boot (spread + idle wait + apply). Read before pendingUpdate is cleared.
202
+ const now = this.now();
203
+ const sinceMs = (iso) => {
204
+ const ms = iso ? Date.parse(iso) : Number.NaN;
205
+ return Number.isFinite(ms) ? Math.max(0, now - ms) : undefined;
206
+ };
207
+ const firstSeenAt = state.pendingUpdate?.firstSeenAt;
208
+ const pendingMs = sinceMs(firstSeenAt);
209
+ const applyMs = sinceMs(pending.requestedAt);
210
+ this.appliedReport = this.reportApplied(() => this.emitEvent(EVENT_NAME.autoUpdateApplied, "info", {
211
+ body: `Update applied: ${pending.fromVersion} -> v${pending.toVersion} (${pending.mode})`,
212
+ attributes: eventAttrs({
213
+ "senpi.auto_update.from_version": pending.fromVersion,
214
+ "senpi.auto_update.to_version": pending.toVersion,
215
+ "senpi.auto_update.restart_mode": pending.mode,
216
+ "senpi.auto_update.requested_at": pending.requestedAt,
217
+ "senpi.auto_update.first_seen_at": pendingMs === undefined ? undefined : firstSeenAt,
218
+ "senpi.auto_update.apply_ms": applyMs,
219
+ "senpi.auto_update.pending_ms": pendingMs,
220
+ }),
221
+ }));
222
+ }
223
+ state.pendingRestart = undefined;
224
+ }
225
+ if (state.pendingUpdate && isAtLeast(loaded, state.pendingUpdate.version)) {
226
+ state.pendingUpdate = undefined;
227
+ }
228
+ if (state.applyFailures && isAtLeast(loaded, state.applyFailures.version)) {
229
+ state.applyFailures = undefined;
230
+ }
231
+ if (state.lastAppliedVersion && isAtLeast(loaded, state.lastAppliedVersion)) {
232
+ state.lastLoggedAwaitingRestartVersion = undefined;
233
+ }
234
+ const markerPath = this.options.updateMarkerPath;
235
+ if (markerPath) {
236
+ const marker = await readUpdateMarker(markerPath);
237
+ if (marker && isAtLeast(loaded, marker.targetVersion)) {
238
+ await this.clearMarkerQuietly();
239
+ }
240
+ }
241
+ return action;
242
+ }
243
+ /**
244
+ * Emit the 10.x `auto_update.applied` once the OTel SDK that `register()` started is up, then
245
+ * flush. It fires right after boot, when the global LoggerProvider may still be the no-op one,
246
+ * and the pending restart is already cleared, so a dropped event would never be sent again. A
247
+ * late or failed init still emits: the gateway log keeps the line either way.
248
+ */
249
+ async reportApplied(emit) {
250
+ const telemetry = this.options.installResultsTelemetry;
251
+ if (!telemetry) {
252
+ emit();
253
+ return;
254
+ }
255
+ try {
256
+ await telemetry.ready();
257
+ }
258
+ catch {
259
+ // Telemetry must not break the coordinator.
260
+ }
261
+ emit();
262
+ try {
263
+ await telemetry.flush();
264
+ }
265
+ catch {
266
+ // Telemetry must not break the coordinator.
267
+ }
268
+ }
269
+ /** Resolves when the `applied` report begun by the last {@link start}, if any, has finished. */
270
+ whenAppliedReported() {
271
+ return this.appliedReport;
272
+ }
273
+ /**
274
+ * Resolves when the install-results report begun by the last {@link start} has finished.
275
+ * `start` does not wait for it: the report waits for the OTel SDK first.
276
+ */
277
+ whenInstallResultsReported() {
278
+ return this.installResultsReport;
279
+ }
280
+ /**
281
+ * 10.x: start reporting the wrapper's install outcomes in the background. `liveState` is the
282
+ * running coordinator's state (the ids are saved with it); `null` when auto-update is
283
+ * disabled, which loads only the ids and patches only them.
284
+ */
285
+ startInstallResultsReport(liveState) {
286
+ if (!this.options.modernHost || !this.options.installResultsPath)
287
+ return;
288
+ this.installResultsReport = this.reportInstallResults(this.options.installResultsPath, liveState);
289
+ }
290
+ /**
291
+ * One `auto_update.install_result` per entry not reported before (`install-results.ts`), then
292
+ * the file is deleted. Runs from {@link start}, which only the gateway's plugin service calls,
293
+ * so a CLI process never consumes the file. Never throws: telemetry must not break the
294
+ * coordinator.
295
+ */
296
+ async reportInstallResults(path, liveState) {
297
+ try {
298
+ const reportedIds = liveState
299
+ ? liveState.reportedInstallResultIds
300
+ : (await this.store.load(this.options.currentVersion, this.options.config.mode)).reportedInstallResultIds;
301
+ const result = await consumeInstallResults({
302
+ path,
303
+ reportedIds: Array.isArray(reportedIds) ? reportedIds : [],
304
+ emit: (entry) => this.emitInstallResult(entry),
305
+ persistReportedIds: async (ids) => {
306
+ if (!liveState)
307
+ return this.store.patch({ reportedInstallResultIds: ids });
308
+ // The running coordinator saves this same object on every tick; saving it whole
309
+ // never writes back an older copy.
310
+ liveState.reportedInstallResultIds = ids;
311
+ await this.store.save(liveState);
312
+ },
313
+ telemetry: this.options.installResultsTelemetry,
314
+ logger: this.options.logger,
315
+ });
316
+ if (result.emitted > 0 || result.skipped > 0) {
317
+ this.options.logger.info(`[senpi-auto-update] reported ${result.emitted} wrapper install result(s)` +
318
+ (result.skipped > 0 ? ` (${result.skipped} already reported)` : ""));
319
+ }
320
+ }
321
+ catch (error) {
322
+ this.options.logger.warn(`[senpi-auto-update] reading the wrapper's install results failed | err=${formatErr(error)}`);
323
+ }
324
+ }
325
+ emitInstallResult(e) {
326
+ const opt = (v) => (v === null ? undefined : v);
327
+ this.emitEvent(EVENT_NAME.autoUpdateInstallResult, installResultLevel(e.outcome), {
328
+ body: describeInstallResult(e),
329
+ attributes: eventAttrs({
330
+ "senpi.auto_update.source": opt(e.source),
331
+ "senpi.auto_update.outcome": e.outcome,
332
+ "senpi.auto_update.spec": opt(e.spec),
333
+ "senpi.auto_update.from_version": opt(e.fromVersion),
334
+ "senpi.auto_update.to_version": opt(e.toVersion),
335
+ "senpi.auto_update.target_version": opt(e.targetVersion),
336
+ "senpi.auto_update.duration_ms": opt(e.durationMs),
337
+ "senpi.auto_update.reason": opt(e.reason),
338
+ "senpi.auto_update.ended_at": opt(e.at),
339
+ "senpi.auto_update.result_id": opt(e.id),
340
+ "senpi.auto_update.openclaw_version": opt(e.openclawVersion),
341
+ }),
342
+ });
343
+ }
344
+ /**
345
+ * 10.x: the box waits (`pending-idle:<v>`). `auto_update.deferred` on entering the wait and when
346
+ * the target or the sorted blocker set changes; an unchanged re-check or poll is silent.
347
+ */
348
+ noteDeferred(pending, latest, blockers) {
349
+ const sorted = blockerList(blockers);
350
+ const key = `${latest}|${sorted}`;
351
+ if (key === this.lastDeferredKey)
352
+ return;
353
+ this.lastDeferredKey = key;
354
+ this.emitEvent(EVENT_NAME.autoUpdateDeferred, "info", {
355
+ body: `Restart to apply v${latest} deferred: waiting for ${sorted || "nothing"} (eligible at ${pending.eligibleAt})`,
356
+ attributes: eventAttrs({
357
+ "senpi.auto_update.to_version": latest,
358
+ "senpi.auto_update.blockers": sorted,
359
+ "senpi.auto_update.eligible_at": pending.eligibleAt,
360
+ "senpi.auto_update.first_seen_at": pending.firstSeenAt,
361
+ }),
362
+ });
363
+ }
364
+ /** Remove the install marker; a failure is logged, never thrown (the wrapper deletes it too). */
365
+ async clearMarkerQuietly() {
366
+ const markerPath = this.options.updateMarkerPath;
367
+ if (!markerPath)
368
+ return;
369
+ try {
370
+ await clearUpdateMarker(markerPath);
371
+ }
372
+ catch (error) {
373
+ this.options.logger.warn(`[senpi-auto-update] failed to clear the update marker | path=${markerPath} err=${formatErr(error)}`);
374
+ }
375
+ }
104
376
  /** Cancel the next scheduled poll and prevent in-flight ticks from rescheduling. */
105
377
  async stop() {
106
378
  this.runToken += 1;
@@ -109,6 +381,7 @@ export class AutoUpdateCoordinator {
109
381
  clearTimeout(this.timer);
110
382
  this.timer = null;
111
383
  }
384
+ this.clearIdleRecheck();
112
385
  }
113
386
  /** Return the latest point-in-time snapshot for health/state gateway RPCs. */
114
387
  getSnapshot() {
@@ -154,9 +427,7 @@ export class AutoUpdateCoordinator {
154
427
  // way to tell a completed check from a failed one is whether it counted a
155
428
  // failure. Reporting "ran" either way tells an operator testing a config
156
429
  // change that it worked when nothing reached the registry.
157
- // Baseline on live state, not the snapshot: `runTick` builds the healthy
158
- // snapshot before `tick` zeroes the counter, so after a recovery the
159
- // snapshot still carries the old total and 0 -> 1 reads as no change.
430
+ // Baseline on live state: it is what `tick` changes.
160
431
  const failuresBefore = this.state.consecutiveFailures;
161
432
  await this.tick(this.runToken);
162
433
  if (this.state.consecutiveFailures > failuresBefore) {
@@ -194,15 +465,24 @@ export class AutoUpdateCoordinator {
194
465
  return;
195
466
  }
196
467
  this.running = true;
468
+ this.clearIdleRecheck();
197
469
  this.options.logger.info(`[senpi-auto-update] tick started | localVersion=${this.options.currentVersion}`);
198
470
  const state = this.state;
199
471
  let nextDelayMs = null;
472
+ let recheckIdle = false;
200
473
  try {
201
474
  await this.runTick(state);
202
475
  state.consecutiveFailures = 0;
203
476
  state.lastFailureAt = undefined;
204
477
  state.lastFailureReason = undefined;
205
478
  await this.persistState(state);
479
+ // `runTick` built the snapshot before the reset above; rebuild it so it
480
+ // carries the state that was just written.
481
+ this.snapshot = this.buildSnapshot(this.snapshot.health, this.snapshot.lastAction, this.snapshot.lastError);
482
+ recheckIdle = this.options.modernHost === true && this.snapshot.lastAction.startsWith("pending-idle:");
483
+ // The wait ended (restart requested, no update, a restart failure): the next wait is new.
484
+ if (!recheckIdle)
485
+ this.lastDeferredKey = null;
206
486
  nextDelayMs = this.options.config.pollIntervalMinutes * 60_000;
207
487
  this.options.logger.info(`[senpi-auto-update] tick succeeded | nextDelayMs=${nextDelayMs}`);
208
488
  }
@@ -211,6 +491,8 @@ export class AutoUpdateCoordinator {
211
491
  state.lastFailureAt = new Date().toISOString();
212
492
  state.lastFailureReason = formatErr(error);
213
493
  await this.persistState(state);
494
+ // A failed tick ends the wait it was in: the next wait is reported as new.
495
+ this.lastDeferredKey = null;
214
496
  const base = this.options.config.failureBackoffMinutes * 60_000;
215
497
  const max = this.options.config.maxBackoffMinutes * 60_000;
216
498
  const backoff = Math.min(max, base * Math.max(1, state.consecutiveFailures));
@@ -247,8 +529,67 @@ export class AutoUpdateCoordinator {
247
529
  this.options.logger.info(`[senpi-auto-update] scheduling next tick after onTickComplete | nextDelayMs=${nextDelayMs}`);
248
530
  this.scheduleNext(nextDelayMs, token);
249
531
  }
532
+ if (recheckIdle)
533
+ this.scheduleIdleRecheck(token);
534
+ }
535
+ }
536
+ /** Re-read the apply gate of a `pending-idle` box after {@link PENDING_IDLE_RECHECK_MS}. */
537
+ scheduleIdleRecheck(token) {
538
+ if (this.stopped || token !== this.runToken)
539
+ return;
540
+ this.clearIdleRecheck();
541
+ this.idleRecheckTimer = setTimeout(() => {
542
+ this.idleRecheckTimer = null;
543
+ this.recheckPendingIdle(token);
544
+ }, PENDING_IDLE_RECHECK_MS);
545
+ }
546
+ clearIdleRecheck() {
547
+ if (this.idleRecheckTimer) {
548
+ clearTimeout(this.idleRecheckTimer);
549
+ this.idleRecheckTimer = null;
250
550
  }
251
551
  }
552
+ /**
553
+ * Run a check now when the gate that held the pending update has opened,
554
+ * else re-read it again later. A check already in flight re-arms this
555
+ * itself when it ends `pending-idle`.
556
+ */
557
+ recheckPendingIdle(token) {
558
+ if (this.running || this.stopped || token !== this.runToken)
559
+ return;
560
+ const pending = this.state.pendingUpdate;
561
+ if (!pending)
562
+ return;
563
+ const now = this.now();
564
+ const gate = this.applyGate(now, Date.parse(pending.eligibleAt));
565
+ if (!gate.open) {
566
+ // Still waiting; a blocker set that changed since the last poll is a new deferred,
567
+ // and the status snapshot reads the same set.
568
+ this.pendingBlockers = gate.blockers;
569
+ this.snapshot = this.buildSnapshot(this.snapshot.health, this.snapshot.lastAction, this.snapshot.lastError);
570
+ this.noteDeferred(pending, pending.version, gate.blockers);
571
+ this.scheduleIdleRecheck(token);
572
+ return;
573
+ }
574
+ this.options.logger.info(`[senpi-auto-update] pending v${pending.version}: apply gate open; checking now`);
575
+ void this.tick(token);
576
+ }
577
+ /**
578
+ * The 10.x apply gate: wait for `eligibleAt` (blocker `spread`), then for an
579
+ * idle box, unless {@link mustForce} applies and every blocker is one the
580
+ * forced restart may override.
581
+ */
582
+ applyGate(now, eligibleMs) {
583
+ if (Number.isFinite(eligibleMs) && now < eligibleMs) {
584
+ return { open: false, forced: false, blockers: ["spread"] };
585
+ }
586
+ const decision = this.readIdle(now);
587
+ // Forcing overrides a busy user, never money in motion (FORCE_MAY_OVERRIDE).
588
+ const forced = !decision.idle &&
589
+ onlyOverridable(decision.blockers) &&
590
+ mustForce(now, Number.isFinite(eligibleMs) ? eligibleMs : now, this.options.mid);
591
+ return { open: decision.idle || forced, forced, blockers: [...decision.blockers] };
592
+ }
252
593
  /**
253
594
  * Emit a catalogued `auto_update.*` event, guarded so neither a throwing sink nor a bad
254
595
  * attribute shape can disturb the tick — telemetry must never break auto-update itself.
@@ -274,6 +615,7 @@ export class AutoUpdateCoordinator {
274
615
  /** Core update logic: fetch latest version, classify, and take action. */
275
616
  async runTick(state) {
276
617
  const { logger } = this.options;
618
+ this.pendingBlockers = [];
277
619
  // Stamped before the tag is resolved so a rejected tag reads as "checked
278
620
  // just now and failed", not as "stopped polling a while ago".
279
621
  state.lastCheckedAt = new Date().toISOString();
@@ -324,29 +666,350 @@ export class AutoUpdateCoordinator {
324
666
  "senpi.auto_update.will_update": willUpdate,
325
667
  }),
326
668
  });
669
+ // A pending 10.x update lives only while an applicable update is seen, so
670
+ // a dist-tag rollback or a mode change does not leave a stale eligibility
671
+ // time that would let the next release skip the spread.
327
672
  if (updateType === "none") {
673
+ state.pendingUpdate = undefined;
674
+ // 5.x: `latest` was rolled back to the loaded version while a newer
675
+ // install sits on disk unloaded. Reinstall `latest` so the next restart
676
+ // cannot load the pulled release. It is already loaded: no restart.
677
+ const newerOnDisk = [state.lastAppliedVersion, state.pendingRestart?.toVersion].some((v) => v !== undefined && parseSemver(v) !== null && classifyUpdate(latest, v) !== "none");
678
+ if (!this.options.modernHost &&
679
+ state.effectiveMode === "auto-apply" &&
680
+ latest === this.options.currentVersion &&
681
+ newerOnDisk) {
682
+ this.options.logger.warn(`[senpi-auto-update] latest was rolled back to the loaded v${latest} while a newer install is on disk; reinstalling v${latest}`);
683
+ try {
684
+ await this.runUpdateCommand(spec);
685
+ }
686
+ catch (err) {
687
+ // Same report as a normal install failure; the tick's catch counts it and backs off.
688
+ this.reportInstallFailed(latest, err);
689
+ throw err;
690
+ }
691
+ state.lastAppliedVersion = undefined;
692
+ state.pendingRestart = undefined;
693
+ state.lastLoggedAwaitingRestartVersion = undefined;
694
+ this.snapshot = this.buildSnapshot("healthy", `reinstalled-loaded:${latest}`);
695
+ return;
696
+ }
697
+ // 10.x: a restart was requested for a release `latest` no longer names.
698
+ // The A12 wrapper installs whatever the marker says on the next gateway
699
+ // spawn, for any reason, so drop the request and the marker; otherwise
700
+ // the pulled release still loads. Nothing is installed on 10.x.
701
+ const pendingTarget = state.pendingRestart?.toVersion;
702
+ if (this.options.modernHost &&
703
+ pendingTarget !== undefined &&
704
+ parseSemver(pendingTarget) !== null &&
705
+ classifyUpdate(latest, pendingTarget) !== "none") {
706
+ this.options.logger.warn(`[senpi-auto-update] latest was rolled back to v${latest} while a restart for v${pendingTarget} is pending; cancelling it and clearing the update marker`);
707
+ state.pendingRestart = undefined;
708
+ state.lastLoggedAwaitingRestartVersion = undefined;
709
+ await this.clearMarkerQuietly();
710
+ }
328
711
  this.snapshot = this.buildSnapshot("healthy", `no-update:${latest}`);
329
712
  return;
330
713
  }
331
714
  if (updateType === "major") {
715
+ state.pendingUpdate = undefined;
332
716
  await this.handleMajorAvailable(state, latest, spec);
333
717
  return;
334
718
  }
335
719
  if (!this.isAllowedByPolicy(updateType)) {
720
+ state.pendingUpdate = undefined;
336
721
  logger.info(`[senpi-auto-update] update blocked by policy | updateType=${updateType} remoteVersion=${latest}`);
337
722
  this.snapshot = this.buildSnapshot("healthy", `policy-blocked:${updateType}:${latest}`);
338
723
  return;
339
724
  }
340
725
  if (state.effectiveMode !== "auto-apply") {
726
+ state.pendingUpdate = undefined;
341
727
  await this.logUpdateAvailable(state, latest, updateType, spec);
342
728
  return;
343
729
  }
344
- if (state.lastAppliedVersion === latest) {
345
- this.handleAlreadyAppliedAwaitingRestart(state, latest);
730
+ if (state.applyFailures && state.applyFailures.version !== latest) {
731
+ // A newer release after failed applies of an older one starts a new
732
+ // episode with a fresh spread wait. Keeping the old `eligibleAt` (long
733
+ // past) would restart every such box on its first idle tick, and a
734
+ // fleet-wide apply failure makes that the whole fleet at once.
735
+ state.applyFailures = undefined;
736
+ state.pendingUpdate = undefined;
737
+ }
738
+ // The latch fix: a persisted target equal to `latest` means a restart was
739
+ // requested for it and the loaded version is still behind (classifyUpdate
740
+ // above said so). That is trusted only while the request is recent. On 5.x
741
+ // the runtime installs, so `lastAppliedVersion` names the target; a stale
742
+ // `pendingRestart` from an earlier install must not latch a rolled-back
743
+ // `latest`. On 10.x only `pendingRestart` names it.
744
+ if (state.lastAppliedVersion === latest || (this.options.modernHost && state.pendingRestart?.toVersion === latest)) {
745
+ if (this.restartStillPending(state) || this.awaitingManualRestart(state)) {
746
+ this.handleAlreadyAppliedAwaitingRestart(state, latest);
747
+ return;
748
+ }
749
+ await this.expireUnloadedUpdate(state, latest);
750
+ }
751
+ if ((state.applyFailures?.count ?? 0) >= MAX_APPLY_ATTEMPTS) {
752
+ this.handleParkedUpdate(state, latest);
753
+ return;
754
+ }
755
+ if (this.options.modernHost) {
756
+ await this.runRestartToApply(state, latest, spec);
346
757
  return;
347
758
  }
348
759
  await this.runUpdateTransaction(state, latest, updateType, spec);
349
760
  }
761
+ /**
762
+ * Whether the restart requested for the persisted target may still land:
763
+ * its request time (the pending restart's, else the last restart trigger,
764
+ * else the last install) is under {@link RESTART_LOAD_TIMEOUT_MS} old. With
765
+ * no time on record the persisted version is not trusted at all.
766
+ */
767
+ restartStillPending(state) {
768
+ // The latest of the recorded times: an older trigger for a previous
769
+ // version must not expire a fresh install.
770
+ const times = [state.pendingRestart?.requestedAt, state.lastRestartTriggerAt, state.lastSuccessAt]
771
+ .map((iso) => (iso ? Date.parse(iso) : Number.NaN))
772
+ .filter((ms) => Number.isFinite(ms));
773
+ if (times.length === 0)
774
+ return false;
775
+ return this.now() - Math.max(...times) < RESTART_LOAD_TIMEOUT_MS;
776
+ }
777
+ /**
778
+ * The 5.x latch, as before R2: once this flow installed `latest` (an
779
+ * install or restart time is on record), the box waits for a restart and
780
+ * never runs the in-process install and config-bump restart for that
781
+ * version again, however long it waits. 5.x has no idle gate, so a retry
782
+ * would restart the gateway over an order call or an open approval. A
783
+ * restart trigger that threw is latched too (the warning asks for a manual
784
+ * restart). A persisted applied version with no time at all is still not
785
+ * trusted. 10.x expires its wait after 30 min instead: its retries are
786
+ * idle-gated.
787
+ */
788
+ awaitingManualRestart(state) {
789
+ if (this.options.modernHost)
790
+ return false;
791
+ return [state.pendingRestart?.requestedAt, state.lastRestartTriggerAt, state.lastSuccessAt].some((iso) => typeof iso === "string" && Number.isFinite(Date.parse(iso)));
792
+ }
793
+ /**
794
+ * {@link MAX_APPLY_ATTEMPTS} restarts for `latest` did not load it. Stop
795
+ * trying this version: no install and no restart until a newer one ships
796
+ * (or a restart loads it some other way). Reported, not thrown, so the poll
797
+ * keeps its normal cadence.
798
+ */
799
+ handleParkedUpdate(state, latest) {
800
+ const count = state.applyFailures?.count ?? 0;
801
+ const message = `update to v${latest} did not load after ${count} restarts; not retrying this version. ` +
802
+ `The gateway stays on v${this.loadedVersion} until a newer release ships or an operator installs it`;
803
+ if (state.lastLoggedAwaitingRestartVersion !== `parked:${latest}`) {
804
+ this.options.logger.warn(`[senpi-auto-update] ${message}`);
805
+ state.lastLoggedAwaitingRestartVersion = `parked:${latest}`;
806
+ }
807
+ this.snapshot = this.buildSnapshot("degraded", `update-not-loaded:${latest}`, message);
808
+ }
809
+ /**
810
+ * The requested update did not load. Drop the awaiting-restart state and the
811
+ * marker, report `auto_update.apply_failed`, and fail this tick (so the
812
+ * backoff applies). The next tick starts over: a new install and restart on
813
+ * 5.x, a new idle-gated restart request on 10.x. The pending update keeps
814
+ * its eligibility time, so the box does not wait out its spread again.
815
+ *
816
+ * @throws Always.
817
+ */
818
+ async expireUnloadedUpdate(state, target) {
819
+ const loaded = this.loadedVersion;
820
+ const requestedAt = state.pendingRestart?.requestedAt ?? state.lastRestartTriggerAt ?? "never";
821
+ const message = `update to v${target} did not load: the gateway still runs v${loaded} ` +
822
+ `(restart requested ${requestedAt}); retrying from the start`;
823
+ this.emitEvent(EVENT_NAME.autoUpdateApplyFailed, "warn", {
824
+ body: `Update to v${target} did not load; loaded v${loaded}`,
825
+ attributes: eventAttrs({
826
+ "senpi.auto_update.to_version": target,
827
+ "senpi.auto_update.loaded_version": loaded,
828
+ }),
829
+ });
830
+ this.options.logger.warn(`[senpi-auto-update] ${message}`);
831
+ state.lastAppliedVersion = undefined;
832
+ state.pendingRestart = undefined;
833
+ state.lastLoggedAwaitingRestartVersion = undefined;
834
+ state.applyFailures = {
835
+ version: target,
836
+ count: (state.applyFailures?.version === target ? state.applyFailures.count : 0) + 1,
837
+ };
838
+ await this.clearMarkerQuietly();
839
+ this.snapshot = this.buildSnapshot("degraded", `update-not-loaded:${target}`, message);
840
+ throw new Error(message);
841
+ }
842
+ /**
843
+ * The OpenClaw 10.x apply path (restart-to-apply). Nothing is installed in
844
+ * this process: 10.10 refuses `plugins update` while the gateway runs.
845
+ *
846
+ * 1. Record the pending update once per episode (`firstSeenAt`,
847
+ * `eligibleAt` = first seen + spread offset + jitter).
848
+ * 2. Before `eligibleAt`, wait (`pending-idle:<v>`, blocker `spread`). After
849
+ * it, continue only when the idle gate says idle, or when
850
+ * {@link mustForce} (24 h past eligibility, inside 03:00–05:00 UTC).
851
+ * 3. Build the exact `<name>@<version>` spec, check restart capability,
852
+ * write the marker, and request the restart through `mutateConfigFile`.
853
+ * 4. A failure at any step clears the marker, reports
854
+ * `auto_update.restart_failed` and fails the tick (backoff; `checkNow`
855
+ * answers `failed`). Waiting for the spread or an idle box is not a failure.
856
+ */
857
+ async runRestartToApply(state, latest, spec) {
858
+ const { logger } = this.options;
859
+ const now = this.now();
860
+ if (!state.pendingUpdate) {
861
+ const firstSeenAt = new Date(now).toISOString();
862
+ const eligible = computeEligibleAt(now, this.options.mid ?? "", this.rand);
863
+ state.pendingUpdate = { version: latest, firstSeenAt, eligibleAt: new Date(eligible).toISOString() };
864
+ logger.info(`[senpi-auto-update] v${latest} seen; restart-to-apply eligible at ${state.pendingUpdate.eligibleAt}`);
865
+ }
866
+ else if (state.pendingUpdate.version !== latest) {
867
+ // A newer release during the wait keeps the box's place in the spread.
868
+ state.pendingUpdate = { ...state.pendingUpdate, version: latest };
869
+ }
870
+ const gate = this.applyGate(now, Date.parse(state.pendingUpdate.eligibleAt));
871
+ const forced = gate.forced;
872
+ if (!gate.open) {
873
+ this.pendingBlockers = gate.blockers;
874
+ this.snapshot = this.buildSnapshot("healthy", `pending-idle:${latest}`);
875
+ this.noteDeferred(state.pendingUpdate, latest, gate.blockers);
876
+ return;
877
+ }
878
+ let exactSpec;
879
+ try {
880
+ exactSpec = buildExactUpdateSpec(spec, latest);
881
+ }
882
+ catch (error) {
883
+ await this.reportModernRestartFailure(latest, formatErr(error), `restart-unavailable:${latest}`);
884
+ return;
885
+ }
886
+ const capability = evaluateModernRestartCapability({
887
+ readLiveConfig: this.options.readLiveConfig,
888
+ mutateConfigFile: this.options.mutateConfigFile,
889
+ pluginId: this.options.pluginId,
890
+ });
891
+ state.restartCapability = capability.capable ? "capable" : "not-capable";
892
+ state.restartCapabilityReasons = capability.reasons;
893
+ state.restartCapabilityMode = capability.mode;
894
+ const mutateConfigFile = this.options.mutateConfigFile;
895
+ const markerPath = this.options.updateMarkerPath;
896
+ if (!capability.capable || !mutateConfigFile) {
897
+ await this.reportModernRestartFailure(latest, capability.reasons.join(", ") || "unknown", `restart-unavailable:${latest}`);
898
+ return;
899
+ }
900
+ if (!markerPath) {
901
+ state.restartCapability = "not-capable";
902
+ state.restartCapabilityReasons = [...capability.reasons, "update-marker-path-unavailable"];
903
+ await this.reportModernRestartFailure(latest, "update-marker-path-unavailable", `restart-unavailable:${latest}`);
904
+ return;
905
+ }
906
+ const requestedAt = new Date(now).toISOString();
907
+ state.lastAttemptAt = requestedAt;
908
+ // The pending restart is saved before the restart is requested: the
909
+ // gateway may stop this process right after, and a lost pending restart
910
+ // would never count toward the retry cap. Rolled back if the request fails.
911
+ const before = {
912
+ pendingRestart: state.pendingRestart,
913
+ lastRestartTriggerAt: state.lastRestartTriggerAt,
914
+ lastRestartTriggerMode: state.lastRestartTriggerMode,
915
+ };
916
+ state.pendingRestart = {
917
+ fromVersion: this.loadedVersion,
918
+ toVersion: latest,
919
+ requestedAt,
920
+ mode: "restart-to-apply",
921
+ };
922
+ state.lastRestartTriggerAt = requestedAt;
923
+ state.lastRestartTriggerMode = "restart-to-apply";
924
+ const rollBack = () => {
925
+ state.pendingRestart = before.pendingRestart;
926
+ state.lastRestartTriggerAt = before.lastRestartTriggerAt;
927
+ state.lastRestartTriggerMode = before.lastRestartTriggerMode;
928
+ };
929
+ try {
930
+ await this.store.save(state);
931
+ }
932
+ catch (error) {
933
+ // Not a restart-capability problem: the state dir is not writable. No
934
+ // restart without a saved pending restart, or the retry cap is lost.
935
+ rollBack();
936
+ await this.reportModernRestartFailure(latest, `state-save-failed:${formatErr(error)}`, `state-save-failed:${latest}`, error);
937
+ return;
938
+ }
939
+ try {
940
+ await writeUpdateMarker(markerPath, { spec: exactSpec, targetVersion: latest, requestedAt });
941
+ // Read the gate again after the awaits above: a money call or a user
942
+ // turn that started meanwhile holds the restart, as at the first read.
943
+ const again = this.readIdle(this.now());
944
+ if (!again.idle && !(forced && onlyOverridable(again.blockers))) {
945
+ rollBack();
946
+ await this.clearMarkerQuietly();
947
+ this.pendingBlockers = [...again.blockers];
948
+ this.snapshot = this.buildSnapshot("healthy", `pending-idle:${latest}`);
949
+ if (state.pendingUpdate)
950
+ this.noteDeferred(state.pendingUpdate, latest, again.blockers);
951
+ return;
952
+ }
953
+ await triggerGatewayRestartViaMutate({ mutateConfigFile, pluginId: this.options.pluginId, triggerValue: requestedAt });
954
+ }
955
+ catch (error) {
956
+ rollBack();
957
+ const reason = formatErr(error);
958
+ state.restartCapability = "not-capable";
959
+ state.restartCapabilityReasons = [...state.restartCapabilityReasons, `restart-trigger-failed:${reason}`];
960
+ await this.reportModernRestartFailure(latest, reason, `restart-request-failed:${latest}`, error);
961
+ return;
962
+ }
963
+ this.emitEvent(EVENT_NAME.autoUpdateRestartRequested, "info", {
964
+ body: `Restart requested to apply v${latest} (from v${this.loadedVersion}${forced ? ", forced after 24 h busy" : ""})`,
965
+ attributes: eventAttrs({
966
+ "senpi.auto_update.from_version": this.loadedVersion,
967
+ "senpi.auto_update.to_version": latest,
968
+ "senpi.auto_update.restart_mode": "restart-to-apply",
969
+ "senpi.auto_update.forced": forced,
970
+ "senpi.auto_update.blockers": forced ? blockerList(gate.blockers) : undefined,
971
+ }),
972
+ });
973
+ logger.info(`[senpi-auto-update] marker written (${exactSpec}); gateway restart requested to apply v${latest}` +
974
+ (forced ? ` (forced: busy for 24 h past eligibility; blockers ${gate.blockers.join(",")})` : ""));
975
+ this.snapshot = this.buildSnapshot("healthy", `updated-restart-requested:${latest}`);
976
+ }
977
+ /** The idle gate's answer; a throwing gate counts as busy. */
978
+ readIdle(now) {
979
+ const gate = this.options.idleGate;
980
+ if (!gate)
981
+ return { idle: true, blockers: [] };
982
+ try {
983
+ return gate.isIdle(now);
984
+ }
985
+ catch (error) {
986
+ this.options.logger.warn(`[senpi-auto-update] idle gate threw; treating the box as busy | err=${formatErr(error)}`);
987
+ return { idle: false, blockers: ["idle-gate-error"] };
988
+ }
989
+ }
990
+ /**
991
+ * Clear the marker, report `auto_update.restart_failed` for the 10.x path
992
+ * (snapshot degraded), and fail the tick: no restart was requested, so the
993
+ * tick must not count as a success. The throw makes `tick()` back off and
994
+ * `checkNow()` answer `failed`.
995
+ *
996
+ * @throws Always, with `reason`.
997
+ */
998
+ async reportModernRestartFailure(latest, reason, lastAction, error) {
999
+ await this.clearMarkerQuietly();
1000
+ this.emitEvent(EVENT_NAME.autoUpdateRestartFailed, "error", {
1001
+ body: `Restart to apply v${latest} was not requested: ${reason}`,
1002
+ attributes: eventAttrs({
1003
+ "senpi.auto_update.to_version": latest,
1004
+ "senpi.auto_update.reason": reason,
1005
+ }),
1006
+ ...(error === undefined ? {} : { error }),
1007
+ });
1008
+ this.options.logger.warn(`[senpi-auto-update] v${latest} is available but the gateway restart that applies it was not requested. ` +
1009
+ `Nothing was installed; the loaded version stays v${this.loadedVersion}. Reason(s): ${reason}`);
1010
+ this.snapshot = this.buildSnapshot("degraded", lastAction, reason);
1011
+ throw new Error(`restart to apply v${latest} was not requested: ${reason}`);
1012
+ }
350
1013
  /**
351
1014
  * `currentVersion` is a compile-time constant frozen for the life of the
352
1015
  * process, so it cannot reflect an install that already happened this
@@ -359,19 +1022,30 @@ export class AutoUpdateCoordinator {
359
1022
  * so this branch stops being reached on its own.
360
1023
  */
361
1024
  handleAlreadyAppliedAwaitingRestart(state, latest) {
1025
+ const modern = this.options.modernHost === true;
362
1026
  if (state.lastLoggedAwaitingRestartVersion !== latest) {
363
- this.options.logger.info(`[senpi-auto-update] update to v${latest} already applied, awaiting gateway restart to take effect | skipping reinstall`);
1027
+ this.options.logger.info(modern
1028
+ ? `[senpi-auto-update] restart to apply v${latest} requested, awaiting gateway restart | not requesting again`
1029
+ : `[senpi-auto-update] update to v${latest} already applied, awaiting gateway restart to take effect | skipping reinstall`);
364
1030
  state.lastLoggedAwaitingRestartVersion = latest;
365
1031
  }
366
1032
  // Fired on every guard hit (not deduped like the log line above) — an operator watching for
367
1033
  // "how long has this box been stuck awaiting restart" needs every occurrence, not just the first.
368
1034
  this.emitEvent(EVENT_NAME.autoUpdateAwaitingRestart, "warn", {
369
- body: `Update to v${latest} already applied, still awaiting gateway restart`,
1035
+ body: modern
1036
+ ? `Restart to apply v${latest} requested, still awaiting gateway restart`
1037
+ : `Update to v${latest} already applied, still awaiting gateway restart`,
370
1038
  attributes: eventAttrs({
371
1039
  "senpi.auto_update.to_version": latest,
372
1040
  }),
373
1041
  });
374
- this.snapshot = this.buildSnapshot("degraded", `already-applied-awaiting-restart:${latest}`);
1042
+ // On 10.x nothing is installed until the wrapper consumes the marker, so
1043
+ // "already applied" would be false there.
1044
+ this.snapshot = this.buildSnapshot("degraded", modern ? `awaiting-restart:${latest}` : `already-applied-awaiting-restart:${latest}`);
1045
+ }
1046
+ /** OpenClaw 10.x refuses `plugins update` while the gateway runs, so a manual instruction says so. */
1047
+ manualUpdateSuffix() {
1048
+ return this.options.modernHost ? " with the gateway stopped" : "";
375
1049
  }
376
1050
  isAllowedByPolicy(kind) {
377
1051
  if (kind === "patch")
@@ -394,7 +1068,7 @@ export class AutoUpdateCoordinator {
394
1068
  });
395
1069
  this.options.logger.info(`[senpi-auto-update] Senpi update available: v${latest} (major). ` +
396
1070
  `Auto-update will not apply major versions. ` +
397
- `Review changelog and run: openclaw plugins update ${spec}`);
1071
+ `Review changelog and run: openclaw plugins update ${spec}${this.manualUpdateSuffix()}`);
398
1072
  state.lastNotifiedMajorVersion = latest;
399
1073
  this.snapshot = this.buildSnapshot("healthy", `major-notified:${latest}`);
400
1074
  }
@@ -412,10 +1086,27 @@ export class AutoUpdateCoordinator {
412
1086
  });
413
1087
  this.options.logger.info(`[senpi-auto-update] Senpi update available: v${latest} (${kind}). ` +
414
1088
  `Auto-update mode is notify-only. ` +
415
- `Run: openclaw plugins update ${spec}`);
1089
+ `Run: openclaw plugins update ${spec}${this.manualUpdateSuffix()}`);
416
1090
  state.lastNotifiedUpdateVersion = latest;
417
1091
  this.snapshot = this.buildSnapshot("healthy", `update-notified:${kind}:${latest}`);
418
1092
  }
1093
+ /**
1094
+ * The `auto_update.install_failed` event and warn line for an install of
1095
+ * `latest` that threw. Shared by the normal install and the 5.x reinstall of
1096
+ * a rolled-back `latest`; the caller rethrows to the tick's catch.
1097
+ */
1098
+ reportInstallFailed(latest, err) {
1099
+ const reason = formatErr(err);
1100
+ this.emitEvent(EVENT_NAME.autoUpdateInstallFailed, "error", {
1101
+ body: `Install failed: ${this.options.currentVersion} -> v${latest}: ${reason}`,
1102
+ attributes: eventAttrs({
1103
+ "senpi.auto_update.to_version": latest,
1104
+ }),
1105
+ error: err,
1106
+ });
1107
+ this.options.logger.warn(`[senpi-auto-update] Senpi auto-update failed: ${this.options.currentVersion} -> ${latest}. ` +
1108
+ `Reason: ${reason}. Will retry after backoff.`);
1109
+ }
419
1110
  /**
420
1111
  * Install the update, evaluate restart capability, then either trigger a
421
1112
  * gateway restart or log that a manual restart is needed.
@@ -443,16 +1134,7 @@ export class AutoUpdateCoordinator {
443
1134
  await this.runUpdateCommand(spec);
444
1135
  }
445
1136
  catch (err) {
446
- const reason = formatErr(err);
447
- this.emitEvent(EVENT_NAME.autoUpdateInstallFailed, "error", {
448
- body: `Install failed: ${this.options.currentVersion} -> v${latest}: ${reason}`,
449
- attributes: eventAttrs({
450
- "senpi.auto_update.to_version": latest,
451
- }),
452
- error: err,
453
- });
454
- this.options.logger.warn(`[senpi-auto-update] Senpi auto-update failed: ${this.options.currentVersion} -> ${latest}. ` +
455
- `Reason: ${reason}. Will retry after backoff.`);
1137
+ this.reportInstallFailed(latest, err);
456
1138
  throw err;
457
1139
  }
458
1140
  // Step 2: record success before evaluating restart (no runtimes stopped yet).
@@ -495,6 +1177,12 @@ export class AutoUpdateCoordinator {
495
1177
  });
496
1178
  state.lastRestartTriggerAt = nowIso;
497
1179
  state.lastRestartTriggerMode = capability.mode ?? "unknown";
1180
+ state.pendingRestart = {
1181
+ fromVersion: this.loadedVersion,
1182
+ toVersion: latest,
1183
+ requestedAt: nowIso,
1184
+ mode: state.lastRestartTriggerMode,
1185
+ };
498
1186
  this.emitEvent(EVENT_NAME.autoUpdateApplied, "info", {
499
1187
  body: `Update applied: ${this.options.currentVersion} -> v${latest}; restart requested (${state.lastRestartTriggerMode})`,
500
1188
  attributes: eventAttrs({
@@ -553,6 +1241,7 @@ export class AutoUpdateCoordinator {
553
1241
  restartCapabilityReasons: state.restartCapabilityReasons,
554
1242
  lastRestartTriggerAt: state.lastRestartTriggerAt ?? null,
555
1243
  lastRestartTriggerMode: state.lastRestartTriggerMode ?? null,
1244
+ pendingBlockers: this.pendingBlockers,
556
1245
  };
557
1246
  }
558
1247
  async persistState(state) {
@@ -573,6 +1262,10 @@ async function defaultRunUpdateCommand(packageName) {
573
1262
  throw new Error(result.stderr || `openclaw plugins update failed with code ${result.code}`);
574
1263
  }
575
1264
  }
1265
+ /** A blocker set as one stable value: de-duplicated, sorted, comma-joined. */
1266
+ function blockerList(blockers) {
1267
+ return [...new Set(blockers)].sort().join(",");
1268
+ }
576
1269
  function formatErr(error) {
577
1270
  return error instanceof Error ? error.message : String(error);
578
1271
  }