dsh-autorestart 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +457 -41
- package/assets/screenshot-1.png +0 -0
- package/assets/screenshot-2.png +0 -0
- package/assets/screenshot-3.png +0 -0
- package/assets/screenshot-4.png +0 -0
- package/assets/screenshot-5.png +0 -0
- package/client.js +5973 -3222
- package/cordis.patch.yml +55 -9
- package/docs/README.zh.md +305 -27
- package/index.js +2522 -127
- package/lib/access.js +212 -0
- package/lib/activity.js +124 -0
- package/lib/autostart-advice.js +198 -0
- package/lib/backup.js +131 -4
- package/lib/blockers.js +228 -0
- package/lib/checkpoint.js +147 -0
- package/lib/diagnostics.js +180 -0
- package/lib/health.js +264 -0
- package/lib/host-kind.js +145 -0
- package/lib/lifecycle.js +17 -4
- package/lib/node-runtime.js +144 -0
- package/lib/queued.js +146 -0
- package/lib/restart-verdict.js +327 -0
- package/lib/restart.js +823 -56
- package/lib/sleep.js +46 -49
- package/lib/startup-mode.js +491 -0
- package/lib/supervisor-config.js +735 -0
- package/lib/supervisor-install.js +557 -0
- package/lib/supervisor.js +1276 -0
- package/locale/en.json +247 -60
- package/locale/zh.json +247 -60
- package/package.json +7 -2
- package/screenshots.json +6 -4
- package/scripts/emergency-disable.cmd +27 -0
- package/scripts/emergency-toggle.ps1 +302 -0
- package/scripts/run-tests.ps1 +46 -0
- package/supervisor/supervisor.js +884 -0
package/index.js
CHANGED
|
@@ -26,6 +26,7 @@ import path from 'node:path'
|
|
|
26
26
|
import {
|
|
27
27
|
armSleep,
|
|
28
28
|
beginBoot,
|
|
29
|
+
cancelOwnRestart,
|
|
29
30
|
cancelRestart,
|
|
30
31
|
clearSleep,
|
|
31
32
|
disarmedRecord,
|
|
@@ -37,6 +38,7 @@ import {
|
|
|
37
38
|
parkedRecord,
|
|
38
39
|
pruneSessions,
|
|
39
40
|
readState,
|
|
41
|
+
recordOnboarding,
|
|
40
42
|
recoverableSessions,
|
|
41
43
|
requestRestart,
|
|
42
44
|
setContextOverflow,
|
|
@@ -62,20 +64,78 @@ import {
|
|
|
62
64
|
} from './lib/lifecycle.js'
|
|
63
65
|
import {
|
|
64
66
|
backupsRoot,
|
|
67
|
+
claimRollbackLock,
|
|
65
68
|
clearRollbackReport,
|
|
66
69
|
createBackup,
|
|
70
|
+
listBackups,
|
|
67
71
|
readRollbackReport,
|
|
72
|
+
releaseRollbackLock,
|
|
73
|
+
restoreBackup,
|
|
74
|
+
ROLLBACK_LOCK_NAME,
|
|
68
75
|
ROLLBACK_REPORT_NAME,
|
|
69
76
|
} from './lib/backup.js'
|
|
77
|
+
import {
|
|
78
|
+
accessFilePath,
|
|
79
|
+
readAccessRecord,
|
|
80
|
+
resolveAccessUrl,
|
|
81
|
+
writeAccessRecord,
|
|
82
|
+
} from './lib/access.js'
|
|
83
|
+
import {
|
|
84
|
+
detectSupervisor,
|
|
85
|
+
installSupervisor,
|
|
86
|
+
planSupervisorInstall,
|
|
87
|
+
readSupervisorConfig,
|
|
88
|
+
requestSupervisorAction,
|
|
89
|
+
supervisorPaths,
|
|
90
|
+
supervisorStatus,
|
|
91
|
+
uninstallSupervisor,
|
|
92
|
+
writeRollbackRequest,
|
|
93
|
+
} from './lib/supervisor.js'
|
|
94
|
+
import { composeFragment, detectInitSystems } from './lib/supervisor-install.js'
|
|
95
|
+
import { classifyStartup } from './lib/startup-mode.js'
|
|
96
|
+
import { detectHostKind } from './lib/host-kind.js'
|
|
97
|
+
import { planAutostartAdvice } from './lib/autostart-advice.js'
|
|
98
|
+
import { buildDiagnostics, ISSUE_URL_BASE, SUPPORT_EMAIL } from './lib/diagnostics.js'
|
|
99
|
+
import { countBlockerTasks } from './lib/blockers.js'
|
|
100
|
+
import {
|
|
101
|
+
buildDryRunPlan,
|
|
102
|
+
composeRestartFragment,
|
|
103
|
+
deriveStartCommand,
|
|
104
|
+
detectContainer,
|
|
105
|
+
detectExternalSupervisor,
|
|
106
|
+
inspectExternalSupervisor,
|
|
107
|
+
pickFreePort,
|
|
108
|
+
} from './lib/supervisor-config.js'
|
|
109
|
+
import {
|
|
110
|
+
buildRestartVerdict,
|
|
111
|
+
buildSnapshotRecord,
|
|
112
|
+
evaluateRestartChecks,
|
|
113
|
+
readRestartVerdict,
|
|
114
|
+
RESTART_RESULTS,
|
|
115
|
+
summarizeRestartVerdict,
|
|
116
|
+
writeRestartVerdict,
|
|
117
|
+
} from './lib/restart-verdict.js'
|
|
118
|
+
import { waitForHealthyInstance } from './lib/health.js'
|
|
70
119
|
import {
|
|
71
120
|
buildCustomCommand,
|
|
72
121
|
buildWatchScript,
|
|
122
|
+
DEFAULT_COMMAND_TIMEOUT_MS,
|
|
123
|
+
DEFAULT_ROLLBACK_COPY_TIMEOUT_MS,
|
|
73
124
|
DEFAULT_ROLLBACK_TIMEOUT_MS,
|
|
74
125
|
detectEnvironment,
|
|
126
|
+
launchDetachedWatcher,
|
|
127
|
+
readProcessChain,
|
|
75
128
|
runCommandNow,
|
|
76
129
|
scheduleDetachedRestart,
|
|
130
|
+
writeTextFileAtomic,
|
|
77
131
|
} from './lib/restart.js'
|
|
78
132
|
import { compareSemver } from './lib/semver.js'
|
|
133
|
+
import { isWorkSessionEvent } from './lib/activity.js'
|
|
134
|
+
import {
|
|
135
|
+
createQueuedTracker,
|
|
136
|
+
DEFAULT_QUEUED_GRACE_MS,
|
|
137
|
+
normalizeQueuedGraceMs,
|
|
138
|
+
} from './lib/queued.js'
|
|
79
139
|
import {
|
|
80
140
|
createSleepController,
|
|
81
141
|
COUNTDOWN_CHOICES,
|
|
@@ -87,34 +147,160 @@ import {
|
|
|
87
147
|
} from './lib/sleep.js'
|
|
88
148
|
|
|
89
149
|
/**
|
|
90
|
-
* The
|
|
91
|
-
* finished before it is really restarted
|
|
150
|
+
* The RESTART reserve: how long the host waits after the LAST conversation
|
|
151
|
+
* finished before it is really restarted.
|
|
92
152
|
*
|
|
93
153
|
* This plugin can only see the DSH agents. Other plugins (a memory plugin
|
|
94
154
|
* that writes its notes a few seconds after the conversation ended, a summary,
|
|
95
155
|
* an MCP callback) queue their work with a timer of their own — work that is
|
|
96
|
-
* invisible here. A short, configurable
|
|
156
|
+
* invisible here. A short, configurable reserve window is left for it.
|
|
157
|
+
*
|
|
158
|
+
* The reserve belongs to the RESTART only: a restart is a daily operation, so
|
|
159
|
+
* it should happen as soon as the conversations are done. Default: none.
|
|
160
|
+
*
|
|
161
|
+
* The graceful shutdown has NO reserve of its own: it has a VISIBLE countdown
|
|
162
|
+
* (`sleepCountdownSeconds`), and that countdown IS the wait before the
|
|
163
|
+
* machine goes off. A second, invisible window next to it would only say the
|
|
164
|
+
* same thing twice — so there is none.
|
|
97
165
|
*
|
|
98
166
|
* The values are the ones the settings page offers (in milliseconds).
|
|
99
167
|
*/
|
|
100
|
-
const GRACE_CHOICES_MS = Object.freeze([0,
|
|
168
|
+
const GRACE_CHOICES_MS = Object.freeze([0, 5000, 10000, 20000, 30000, 60000])
|
|
169
|
+
|
|
170
|
+
/** The RESTART reserve that is used when nothing was configured. */
|
|
171
|
+
const DEFAULT_RESTART_GRACE_MS = 0
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* How often the "the restart is waiting for a long time" line may be logged.
|
|
175
|
+
*
|
|
176
|
+
* The restart gate polls every few seconds, so an unthrottled line would
|
|
177
|
+
* flood the log with the same sentence. Once a minute is enough to say that
|
|
178
|
+
* the wait is still going on.
|
|
179
|
+
*/
|
|
180
|
+
const LONG_WAIT_LOG_INTERVAL_MS = 60000
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* The marker the plugin writes when it has decided the restart ITSELF.
|
|
184
|
+
*
|
|
185
|
+
* The rollback watcher is the fallback for the case that the plugin dies
|
|
186
|
+
* with the host (a hosted restart kills the very process the plugin runs in).
|
|
187
|
+
* When the plugin survives the restart command, it decides alone — and writes
|
|
188
|
+
* this marker, so the watcher stands down instead of rolling back a restart
|
|
189
|
+
* that was fine.
|
|
190
|
+
*/
|
|
191
|
+
const RESTART_OUTCOME_FILE = 'restart-outcome.json'
|
|
101
192
|
|
|
102
|
-
/**
|
|
103
|
-
|
|
193
|
+
/**
|
|
194
|
+
* The name of the rollback report of a restart that the plugin decided itself.
|
|
195
|
+
*
|
|
196
|
+
* It is the same file the detached watcher writes (`last-rollback.json`), so
|
|
197
|
+
* the interface shows the rollback either way.
|
|
198
|
+
*/
|
|
199
|
+
const RESTART_REPORT_NAME = ROLLBACK_REPORT_NAME
|
|
104
200
|
|
|
105
201
|
/**
|
|
106
|
-
*
|
|
202
|
+
* Resolve the connection service of the host, when it provides one.
|
|
203
|
+
*
|
|
204
|
+
* The connection service signs the access URL of the running host (the boot
|
|
205
|
+
* token). It is optional: a host without it still gets the plain origin.
|
|
206
|
+
*
|
|
207
|
+
* @param {any} ctx
|
|
208
|
+
* @returns {any|null}
|
|
209
|
+
*/
|
|
210
|
+
function resolveConnectionService(ctx) {
|
|
211
|
+
try {
|
|
212
|
+
if (ctx && ctx.connection && typeof ctx.connection.authenticatedUrl === 'function') return ctx.connection
|
|
213
|
+
} catch {}
|
|
214
|
+
try {
|
|
215
|
+
if (ctx && typeof ctx.get === 'function') {
|
|
216
|
+
const service = ctx.get('connection')
|
|
217
|
+
if (service && typeof service.authenticatedUrl === 'function') return service
|
|
218
|
+
}
|
|
219
|
+
} catch {}
|
|
220
|
+
return null
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* The access link of the running instance, or null.
|
|
225
|
+
*
|
|
226
|
+
* @param {any} runtime
|
|
227
|
+
* @returns {string|null}
|
|
228
|
+
*/
|
|
229
|
+
function accessUrlOf(runtime) {
|
|
230
|
+
try {
|
|
231
|
+
const record = runtime.accessRecord
|
|
232
|
+
if (record && typeof record.url === 'string' && record.url !== '') return record.url
|
|
233
|
+
} catch {}
|
|
234
|
+
return null
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* Publish the instance record of the running host.
|
|
239
|
+
*
|
|
240
|
+
* The record carries three things that nobody else knows:
|
|
241
|
+
*
|
|
242
|
+
* - the PORT this instance listens on (the desktop app uses a random one,
|
|
243
|
+
* so the port of the previous instance is worthless);
|
|
244
|
+
* - the BOOT ID of this instance, so a watcher can tell "the new instance"
|
|
245
|
+
* from "the old one";
|
|
246
|
+
* - the ACCESS LINK with the boot token, so the user finds the working link
|
|
247
|
+
* after a restart instead of being locked out by a 401.
|
|
248
|
+
*
|
|
249
|
+
* The file is written with the mode 0600: it holds a secret.
|
|
250
|
+
*
|
|
251
|
+
* @param {any} ctx the plugin context
|
|
252
|
+
* @param {any} runtime the runtime of this plugin
|
|
253
|
+
* @param {object} config the normalized config
|
|
254
|
+
* @param {string} origin the origin of the web server (`protocol//host:port`)
|
|
255
|
+
* @returns {{ ok: boolean, path: string|null, url: string|null, error: string|null }}
|
|
256
|
+
*/
|
|
257
|
+
function publishInstanceRecord(ctx, runtime, config, origin) {
|
|
258
|
+
const stateDir = path.dirname(runtime.stateFile)
|
|
259
|
+
try {
|
|
260
|
+
const { url, token } = resolveAccessUrl({
|
|
261
|
+
origin,
|
|
262
|
+
connection: resolveConnectionService(ctx),
|
|
263
|
+
})
|
|
264
|
+
const written = writeAccessRecord({
|
|
265
|
+
stateDir,
|
|
266
|
+
bootId: runtime.bootId,
|
|
267
|
+
pid: process.pid,
|
|
268
|
+
port: runtime.webPort,
|
|
269
|
+
url,
|
|
270
|
+
token,
|
|
271
|
+
})
|
|
272
|
+
if (!written.ok) {
|
|
273
|
+
runtime.log.warn('could not publish the instance record:', written.error)
|
|
274
|
+
return { ok: false, path: written.path, url, error: written.error }
|
|
275
|
+
}
|
|
276
|
+
runtime.log.info('the instance record was published', {
|
|
277
|
+
path: written.path,
|
|
278
|
+
port: runtime.webPort,
|
|
279
|
+
// The token itself is a secret: the record carries it, the log only
|
|
280
|
+
// says that there is one.
|
|
281
|
+
hasToken: Boolean(token),
|
|
282
|
+
})
|
|
283
|
+
return { ok: true, path: written.path, url, error: null }
|
|
284
|
+
} catch (error) {
|
|
285
|
+
runtime.log.warn('could not publish the instance record:', String(error))
|
|
286
|
+
return { ok: false, path: null, url: null, error: String(error) }
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/**
|
|
291
|
+
* Snap a reserve value onto the nearest offered choice.
|
|
107
292
|
*
|
|
108
293
|
* A profile that carries a value that is not offered (a hand-edited row, an
|
|
109
294
|
* older build) must not break the boot: the value is snapped onto the
|
|
110
295
|
* nearest choice, exactly like the countdown length is.
|
|
111
296
|
*
|
|
112
297
|
* @param {unknown} value
|
|
298
|
+
* @param {number} [fallback] the value used when the input is not a number
|
|
113
299
|
* @returns {number} one of {@link GRACE_CHOICES_MS}
|
|
114
300
|
*/
|
|
115
|
-
function snapGraceChoice(value) {
|
|
301
|
+
function snapGraceChoice(value, fallback = DEFAULT_RESTART_GRACE_MS) {
|
|
116
302
|
const raw = Number(value)
|
|
117
|
-
if (!Number.isFinite(raw) || raw < 0) return
|
|
303
|
+
if (!Number.isFinite(raw) || raw < 0) return fallback
|
|
118
304
|
let best = GRACE_CHOICES_MS[0]
|
|
119
305
|
let bestDistance = Math.abs(raw - best)
|
|
120
306
|
for (const choice of GRACE_CHOICES_MS) {
|
|
@@ -134,7 +320,7 @@ function snapGraceChoice(value) {
|
|
|
134
320
|
* `pluginVersion()`), but a bundled installation may not ship that file — so
|
|
135
321
|
* the constant below is the last resort. A test keeps the two in sync.
|
|
136
322
|
*/
|
|
137
|
-
const PLUGIN_VERSION_FALLBACK = '0.
|
|
323
|
+
const PLUGIN_VERSION_FALLBACK = '0.7.0'
|
|
138
324
|
|
|
139
325
|
/**
|
|
140
326
|
* The version of the graceful-shutdown FEATURE, not the version of the package.
|
|
@@ -155,6 +341,14 @@ const PLUGIN_VERSION_FALLBACK = '0.6.1'
|
|
|
155
341
|
*/
|
|
156
342
|
const SLEEP_FEATURE_VERSION = '0.5.0'
|
|
157
343
|
|
|
344
|
+
/**
|
|
345
|
+
* The version of the self-starting supervisor ("开机自启守护").
|
|
346
|
+
*
|
|
347
|
+
* The one-time question about it is bound to this version: a release never
|
|
348
|
+
* brings an answered question back.
|
|
349
|
+
*/
|
|
350
|
+
const SUPERVISOR_FEATURE_VERSION = '0.7.0'
|
|
351
|
+
|
|
158
352
|
/** The cached version of this plugin (read once per process). */
|
|
159
353
|
let pluginVersionCache = null
|
|
160
354
|
|
|
@@ -202,6 +396,8 @@ function featureAnnouncements() {
|
|
|
202
396
|
return [
|
|
203
397
|
// The graceful shutdown ("收工关机") and its moon button.
|
|
204
398
|
{ feature: 'sleep', version: SLEEP_FEATURE_VERSION },
|
|
399
|
+
// The self-starting supervisor ("开机自启守护").
|
|
400
|
+
{ feature: 'supervisor', version: SUPERVISOR_FEATURE_VERSION },
|
|
205
401
|
]
|
|
206
402
|
}
|
|
207
403
|
|
|
@@ -380,8 +576,19 @@ function buildConfigSchema() {
|
|
|
380
576
|
// Other plugins queue work with timers of their own (a memory plugin
|
|
381
577
|
// writing its notes, a summary, an MCP callback) and that work is
|
|
382
578
|
// invisible here — so a short, configurable grace window is left for it.
|
|
383
|
-
|
|
384
|
-
|
|
579
|
+
// The RESTART reserve: the extra wait after the last conversation
|
|
580
|
+
// finished, before the host is really restarted. A restart is a daily
|
|
581
|
+
// operation, so the default is none (0 / 5 / 10 / 20 / 30 / 60 seconds).
|
|
582
|
+
restartGraceMs: schemastery.number().min(0).default(DEFAULT_RESTART_GRACE_MS)
|
|
583
|
+
.description('Milliseconds to wait after the last conversation finished before the host is really RESTARTED: the reserve for work other plugins queue with a timer (0 / 5 / 10 / 20 / 30 / 60 seconds, default 0 = no extra wait).'),
|
|
584
|
+
// The graceful shutdown has no reserve of its own: the visible countdown
|
|
585
|
+
// (sleepCountdownSeconds) IS the wait before the machine goes off.
|
|
586
|
+
// The upper bound of a QUEUED inbox message: a message that nobody
|
|
587
|
+
// claims within this window is stuck, and it stops blocking the restart.
|
|
588
|
+
// Without it a single unclaimed message kept the restart pending for
|
|
589
|
+
// ever (the "the restart never happens" defect).
|
|
590
|
+
queuedGraceMs: schemastery.number().min(0).default(DEFAULT_QUEUED_GRACE_MS)
|
|
591
|
+
.description('Milliseconds an unclaimed (queued) inbox message may block the restart: a message that is still unclaimed after this window is treated as stuck and no longer blocks the restart (0 = no upper bound, the old behaviour; default 60000).'),
|
|
385
592
|
autoReloadPage: schemastery.boolean().default(true)
|
|
386
593
|
.description('Reload the web page automatically when the host boot token changes.'),
|
|
387
594
|
blockManualRestart: schemastery.boolean().default(true)
|
|
@@ -407,6 +614,30 @@ function buildConfigSchema() {
|
|
|
407
614
|
.description('How many snapshots are kept; older ones are deleted automatically.'),
|
|
408
615
|
rollbackTimeoutMs: schemastery.number().min(1000).default(90000)
|
|
409
616
|
.description('Milliseconds the detached watcher waits for the relaunched host before it restores the snapshot.'),
|
|
617
|
+
// The self-starting supervisor: a separate process that brings the host
|
|
618
|
+
// back when it dies. See the "开机自启" section of the settings.
|
|
619
|
+
supervisorMode: schemastery.union(['auto', 'installed', 'external', 'off']).default('auto')
|
|
620
|
+
.description('Who starts this host: auto = nobody decided yet, installed = the supervisor of this plugin, external = your own way, off = no automatic start.'),
|
|
621
|
+
supervisorKind: schemastery.union([
|
|
622
|
+
schemastery.const(''),
|
|
623
|
+
schemastery.const('windows-schtasks'),
|
|
624
|
+
schemastery.const('windows-startup'),
|
|
625
|
+
schemastery.const('systemd-user'),
|
|
626
|
+
schemastery.const('systemd-system'),
|
|
627
|
+
]).default('')
|
|
628
|
+
.description('Which form of the self-start record was installed (empty = the default of this platform).'),
|
|
629
|
+
supervisorRestartDelayMs: schemastery.number().min(0).default(3000)
|
|
630
|
+
.description('Milliseconds the supervisor waits before it starts the host again after it died.'),
|
|
631
|
+
supervisorRestartDelayMaxMs: schemastery.number().min(0).default(30000)
|
|
632
|
+
.description('The upper bound of that delay: the supervisor backs off further with every failed start, but never longer than this.'),
|
|
633
|
+
supervisorStartTimeoutMs: schemastery.number().min(1000).default(90000)
|
|
634
|
+
.description('Milliseconds the supervisor waits for the host to answer its health route before it restores the snapshot.'),
|
|
635
|
+
supervisorKillGraceMs: schemastery.number().min(0).default(15000)
|
|
636
|
+
.description('Milliseconds the supervisor waits between asking the host to stop and killing it.'),
|
|
637
|
+
supervisorPort: schemastery.number().min(0).default(0)
|
|
638
|
+
.description('The port the supervisor waits for to become free before it starts the host (0 = the port of the current instance).'),
|
|
639
|
+
supervisorAutoInstall: schemastery.boolean().default(false)
|
|
640
|
+
.description('Whether the one-time question may install the supervisor without asking again.'),
|
|
410
641
|
// graceful shutdown ("收工关机"): wait for every conversation, count down, then
|
|
411
642
|
// switch the REAL host machine off (a WSL host shuts Windows down, not the distro)
|
|
412
643
|
showSleepButton: schemastery.boolean().default(true)
|
|
@@ -448,7 +679,11 @@ const DEFAULTS = {
|
|
|
448
679
|
restartCommand: '',
|
|
449
680
|
relaunchDelaySeconds: 2,
|
|
450
681
|
settleMs: 2000,
|
|
451
|
-
restartGraceMs:
|
|
682
|
+
restartGraceMs: DEFAULT_RESTART_GRACE_MS,
|
|
683
|
+
// The upper bound of a queued (unclaimed) inbox message: after this time it
|
|
684
|
+
// is treated as stuck and stops blocking the restart. 0 switches the
|
|
685
|
+
// protection off (the old behaviour: a queued message blocks for ever).
|
|
686
|
+
queuedGraceMs: DEFAULT_QUEUED_GRACE_MS,
|
|
452
687
|
autoReloadPage: true,
|
|
453
688
|
blockManualRestart: true,
|
|
454
689
|
healthPath: '/__dsh_autorestart/health',
|
|
@@ -461,6 +696,19 @@ const DEFAULTS = {
|
|
|
461
696
|
backupIncludePlugins: true,
|
|
462
697
|
maxBackups: 2,
|
|
463
698
|
rollbackTimeoutMs: 90000,
|
|
699
|
+
// How long the restart command may run before the plugin stops waiting for
|
|
700
|
+
// it, and how long ONE restore step may take before the watcher gives up.
|
|
701
|
+
restartCommandTimeoutMs: DEFAULT_COMMAND_TIMEOUT_MS,
|
|
702
|
+
rollbackCopyTimeoutMs: DEFAULT_ROLLBACK_COPY_TIMEOUT_MS,
|
|
703
|
+
// The self-starting supervisor.
|
|
704
|
+
supervisorMode: 'auto',
|
|
705
|
+
supervisorKind: '',
|
|
706
|
+
supervisorRestartDelayMs: 3000,
|
|
707
|
+
supervisorRestartDelayMaxMs: 30000,
|
|
708
|
+
supervisorStartTimeoutMs: 90000,
|
|
709
|
+
supervisorKillGraceMs: 15000,
|
|
710
|
+
supervisorPort: 0,
|
|
711
|
+
supervisorAutoInstall: false,
|
|
464
712
|
// Graceful shutdown ("收工关机"): the button at the foot of the left sidebar,
|
|
465
713
|
// the countdown before the machine is switched off and the optional custom
|
|
466
714
|
// shutdown command (empty = the host is detected automatically).
|
|
@@ -491,13 +739,45 @@ function normalizeConfig(raw) {
|
|
|
491
739
|
const rollbackTimeout = Number(config.rollbackTimeoutMs)
|
|
492
740
|
config.rollbackTimeoutMs =
|
|
493
741
|
Number.isFinite(rollbackTimeout) && rollbackTimeout >= 1000 ? Math.trunc(rollbackTimeout) : 90000
|
|
742
|
+
// How long the restart command may run, and how long ONE restore step may
|
|
743
|
+
// take. Both are bounded, so neither a hung command nor a hung `robocopy`
|
|
744
|
+
// can hold the restart for ever.
|
|
745
|
+
const commandTimeout = Number(config.restartCommandTimeoutMs)
|
|
746
|
+
config.restartCommandTimeoutMs =
|
|
747
|
+
Number.isFinite(commandTimeout) && commandTimeout >= 1000 ? Math.trunc(commandTimeout) : DEFAULT_COMMAND_TIMEOUT_MS
|
|
748
|
+
const copyTimeout = Number(config.rollbackCopyTimeoutMs)
|
|
749
|
+
config.rollbackCopyTimeoutMs =
|
|
750
|
+
Number.isFinite(copyTimeout) && copyTimeout >= 1000 ? Math.trunc(copyTimeout) : DEFAULT_ROLLBACK_COPY_TIMEOUT_MS
|
|
751
|
+
// The supervisor knobs: a wrong value must not break the boot, it falls back
|
|
752
|
+
// to the documented default.
|
|
753
|
+
if (!['auto', 'installed', 'external', 'off'].includes(config.supervisorMode)) config.supervisorMode = 'auto'
|
|
754
|
+
if (!['', 'windows-schtasks', 'windows-startup', 'systemd-user', 'systemd-system'].includes(config.supervisorKind)) {
|
|
755
|
+
config.supervisorKind = ''
|
|
756
|
+
}
|
|
757
|
+
const supervisorDelay = Number(config.supervisorRestartDelayMs)
|
|
758
|
+
config.supervisorRestartDelayMs = Number.isFinite(supervisorDelay) && supervisorDelay >= 0 ? Math.trunc(supervisorDelay) : 3000
|
|
759
|
+
const supervisorDelayMax = Number(config.supervisorRestartDelayMaxMs)
|
|
760
|
+
config.supervisorRestartDelayMaxMs =
|
|
761
|
+
Number.isFinite(supervisorDelayMax) && supervisorDelayMax >= 0 ? Math.trunc(supervisorDelayMax) : 30000
|
|
762
|
+
const supervisorStartTimeout = Number(config.supervisorStartTimeoutMs)
|
|
763
|
+
config.supervisorStartTimeoutMs =
|
|
764
|
+
Number.isFinite(supervisorStartTimeout) && supervisorStartTimeout >= 1000 ? Math.trunc(supervisorStartTimeout) : 90000
|
|
765
|
+
const supervisorKillGrace = Number(config.supervisorKillGraceMs)
|
|
766
|
+
config.supervisorKillGraceMs =
|
|
767
|
+
Number.isFinite(supervisorKillGrace) && supervisorKillGrace >= 0 ? Math.trunc(supervisorKillGrace) : 15000
|
|
768
|
+
const supervisorPort = Number(config.supervisorPort)
|
|
769
|
+
config.supervisorPort = Number.isFinite(supervisorPort) && supervisorPort >= 0 ? Math.trunc(supervisorPort) : 0
|
|
770
|
+
if (typeof config.supervisorAutoInstall !== 'boolean') config.supervisorAutoInstall = false
|
|
494
771
|
// The countdown of the graceful shutdown is one of the offered choices: a
|
|
495
772
|
// profile that still carries an old value (15, 45, …) is snapped onto the
|
|
496
773
|
// nearest one instead of breaking, and 0 means "no countdown at all".
|
|
497
774
|
config.sleepCountdownSeconds = snapCountdownChoice(config.sleepCountdownSeconds)
|
|
498
|
-
// The
|
|
775
|
+
// The restart reserve is one of the offered choices as well: a hand-edited
|
|
499
776
|
// profile value is snapped onto the nearest one instead of breaking.
|
|
500
|
-
config.restartGraceMs = snapGraceChoice(config.restartGraceMs)
|
|
777
|
+
config.restartGraceMs = snapGraceChoice(config.restartGraceMs, DEFAULT_RESTART_GRACE_MS)
|
|
778
|
+
// The upper bound of a queued message is a plain number: a wrong or negative
|
|
779
|
+
// value falls back to the default, and 0 switches the protection off.
|
|
780
|
+
config.queuedGraceMs = normalizeQueuedGraceMs(config.queuedGraceMs)
|
|
501
781
|
if (typeof config.sleepCommand !== 'string') config.sleepCommand = ''
|
|
502
782
|
if (typeof config.showSleepButton !== 'boolean') config.showSleepButton = true
|
|
503
783
|
return config
|
|
@@ -566,6 +846,21 @@ function resolveChoicePath(config) {
|
|
|
566
846
|
return `${prefix}/choice`
|
|
567
847
|
}
|
|
568
848
|
|
|
849
|
+
/**
|
|
850
|
+
* Resolve the supervisor-route path: a sibling of the health route, under the
|
|
851
|
+
* same prefix (`POST /supervisor` installs, uninstalls and reports the
|
|
852
|
+
* self-starting supervisor).
|
|
853
|
+
*
|
|
854
|
+
* @param {object} config
|
|
855
|
+
* @returns {string}
|
|
856
|
+
*/
|
|
857
|
+
function resolveSupervisorPath(config) {
|
|
858
|
+
const healthPath = resolveHealthPath(config)
|
|
859
|
+
const index = healthPath.lastIndexOf('/')
|
|
860
|
+
const prefix = index > 0 ? healthPath.slice(0, index) : '/__dsh_autorestart'
|
|
861
|
+
return `${prefix}/supervisor`
|
|
862
|
+
}
|
|
863
|
+
|
|
569
864
|
/**
|
|
570
865
|
* Resolve the sleep-route path: a sibling of the health route, under the same
|
|
571
866
|
* prefix (`GET /sleep` reports the state of the graceful sleep, `POST /sleep`
|
|
@@ -581,6 +876,20 @@ function resolveSleepPath(config) {
|
|
|
581
876
|
return `${prefix}/sleep`
|
|
582
877
|
}
|
|
583
878
|
|
|
879
|
+
/**
|
|
880
|
+
* Resolve the cancel-route path: a sibling of the health route, under the same
|
|
881
|
+
* prefix (`POST /cancel` withdraws ONE session's own restart request).
|
|
882
|
+
*
|
|
883
|
+
* @param {object} config
|
|
884
|
+
* @returns {string}
|
|
885
|
+
*/
|
|
886
|
+
function resolveCancelPath(config) {
|
|
887
|
+
const healthPath = resolveHealthPath(config)
|
|
888
|
+
const index = healthPath.lastIndexOf('/')
|
|
889
|
+
const prefix = index > 0 ? healthPath.slice(0, index) : '/__dsh_autorestart'
|
|
890
|
+
return `${prefix}/cancel`
|
|
891
|
+
}
|
|
892
|
+
|
|
584
893
|
/**
|
|
585
894
|
* Resolve the announcements-route path: a sibling of the health route, under
|
|
586
895
|
* the same prefix (`GET /announcements` reports the feature announcements the
|
|
@@ -977,6 +1286,20 @@ function resolveProfileDir() {
|
|
|
977
1286
|
return resolveProfileDirDetailed().dir
|
|
978
1287
|
}
|
|
979
1288
|
|
|
1289
|
+
/**
|
|
1290
|
+
* THE BACKUPS DIRECTORY.
|
|
1291
|
+
*
|
|
1292
|
+
* It is built in exactly ONE place: the snapshot writes here, and the
|
|
1293
|
+
* settings page reads here. Two places would mean two answers to "where do
|
|
1294
|
+
* the backups live".
|
|
1295
|
+
*
|
|
1296
|
+
* @param {string} dir the state directory
|
|
1297
|
+
* @returns {string}
|
|
1298
|
+
*/
|
|
1299
|
+
function backupsDirFor(dir) {
|
|
1300
|
+
return backupsRoot(typeof dir === 'string' ? dir : '')
|
|
1301
|
+
}
|
|
1302
|
+
|
|
980
1303
|
/**
|
|
981
1304
|
* The pre-restart snapshot: copy the profile into `<stateDir>/backups/<stamp>/`
|
|
982
1305
|
* and clear the rollback report an older boot may have left behind.
|
|
@@ -989,7 +1312,10 @@ function resolveProfileDir() {
|
|
|
989
1312
|
* rollback possible at all.
|
|
990
1313
|
*
|
|
991
1314
|
* A failing backup never blocks the restart: the reason is logged and the
|
|
992
|
-
* caller continues without a rollback point.
|
|
1315
|
+
* caller continues without a rollback point. It is NOT silent though: the
|
|
1316
|
+
* facts of the snapshot (was one asked for, was one taken, which directory,
|
|
1317
|
+
* how long it took, which older snapshots were pruned) travel with the answer,
|
|
1318
|
+
* so the restart record can tell the operator what the rollback net is.
|
|
993
1319
|
*
|
|
994
1320
|
* @param {{
|
|
995
1321
|
* config?: object,
|
|
@@ -998,7 +1324,14 @@ function resolveProfileDir() {
|
|
|
998
1324
|
* profile?: { dir?: string|null, source?: string },
|
|
999
1325
|
* createBackup?: Function,
|
|
1000
1326
|
* }} [options]
|
|
1001
|
-
* @returns {{
|
|
1327
|
+
* @returns {{
|
|
1328
|
+
* backup: object|null,
|
|
1329
|
+
* profileDir: string|null,
|
|
1330
|
+
* profile: { dir: string|null, source: string },
|
|
1331
|
+
* requested: boolean,
|
|
1332
|
+
* durationMs: number,
|
|
1333
|
+
* error: string|null,
|
|
1334
|
+
* }}
|
|
1002
1335
|
*/
|
|
1003
1336
|
export function takePreRestartSnapshot({
|
|
1004
1337
|
config = {},
|
|
@@ -1007,21 +1340,31 @@ export function takePreRestartSnapshot({
|
|
|
1007
1340
|
profile = null,
|
|
1008
1341
|
createBackup: createBackupFn = createBackup,
|
|
1009
1342
|
} = {}) {
|
|
1343
|
+
const startedAt = Date.now()
|
|
1010
1344
|
const info = profile && typeof profile === 'object'
|
|
1011
1345
|
? { dir: profile.dir ?? null, source: profile.source ?? 'unknown' }
|
|
1012
1346
|
: { dir: null, source: 'unknown' }
|
|
1013
1347
|
const profileDir = typeof info.dir === 'string' && info.dir.trim() !== '' ? info.dir : null
|
|
1014
1348
|
// The snapshot is optional: the operator can turn it off entirely.
|
|
1015
|
-
if (config.backupOnRestart !== true)
|
|
1349
|
+
if (config.backupOnRestart !== true) {
|
|
1350
|
+
return { backup: null, profileDir, profile: info, requested: false, durationMs: 0, error: null }
|
|
1351
|
+
}
|
|
1016
1352
|
if (profileDir === null) {
|
|
1017
1353
|
log.warn('no profile directory is known; skipping the pre-restart backup', { source: info.source })
|
|
1018
|
-
return {
|
|
1354
|
+
return {
|
|
1355
|
+
backup: null,
|
|
1356
|
+
profileDir: null,
|
|
1357
|
+
profile: info,
|
|
1358
|
+
requested: true,
|
|
1359
|
+
durationMs: Math.max(0, Date.now() - startedAt),
|
|
1360
|
+
error: 'no profile directory is known',
|
|
1361
|
+
}
|
|
1019
1362
|
}
|
|
1020
1363
|
let backup = null
|
|
1021
1364
|
try {
|
|
1022
1365
|
backup = createBackupFn({
|
|
1023
1366
|
profileDir,
|
|
1024
|
-
backupsDir:
|
|
1367
|
+
backupsDir: backupsDirFor(stateDir),
|
|
1025
1368
|
includePlugins: config.backupIncludePlugins !== false,
|
|
1026
1369
|
maxBackups: config.maxBackups,
|
|
1027
1370
|
})
|
|
@@ -1029,9 +1372,14 @@ export function takePreRestartSnapshot({
|
|
|
1029
1372
|
// A backup that throws must not take the restart down with it.
|
|
1030
1373
|
backup = { ok: false, error: String(error) }
|
|
1031
1374
|
}
|
|
1375
|
+
const durationMs = Math.max(0, Date.now() - startedAt)
|
|
1032
1376
|
if (backup && backup.ok === true) {
|
|
1033
1377
|
log.warn('pre-restart backup written', {
|
|
1034
1378
|
dir: backup.dir,
|
|
1379
|
+
stamp: backup.stamp,
|
|
1380
|
+
at: backup.at,
|
|
1381
|
+
atLocal: backup.atLocal,
|
|
1382
|
+
durationMs,
|
|
1035
1383
|
bytes: backup.bytes,
|
|
1036
1384
|
includePlugins: backup.includePlugins,
|
|
1037
1385
|
pruned: backup.pruned,
|
|
@@ -1044,7 +1392,14 @@ export function takePreRestartSnapshot({
|
|
|
1044
1392
|
error: backup && backup.error ? backup.error : 'unknown error',
|
|
1045
1393
|
})
|
|
1046
1394
|
}
|
|
1047
|
-
return {
|
|
1395
|
+
return {
|
|
1396
|
+
backup,
|
|
1397
|
+
profileDir,
|
|
1398
|
+
profile: info,
|
|
1399
|
+
requested: true,
|
|
1400
|
+
durationMs,
|
|
1401
|
+
error: backup && backup.ok === true ? null : (backup && backup.error ? backup.error : 'unknown error'),
|
|
1402
|
+
}
|
|
1048
1403
|
}
|
|
1049
1404
|
|
|
1050
1405
|
/** A logger that never throws and never requires a particular Cordis logger shape. */
|
|
@@ -1158,7 +1513,9 @@ function findOwnEntry(editor) {
|
|
|
1158
1513
|
* snapshot?: Function,
|
|
1159
1514
|
* runCommand?: Function,
|
|
1160
1515
|
* scheduleDetached?: Function,
|
|
1161
|
-
*
|
|
1516
|
+
* launchWatcher?: Function,
|
|
1517
|
+
* probeInstance?: Function,
|
|
1518
|
+
* }} [options] `env` replaces parts of the detected environment and the
|
|
1162
1519
|
* functions replace the side-effecting primitives of the restart path. They
|
|
1163
1520
|
* exist so a restart can be exercised without spawning a process or touching
|
|
1164
1521
|
* the real profile; production never passes them.
|
|
@@ -1175,6 +1532,16 @@ export function createRuntime(ctx, config, options = {}) {
|
|
|
1175
1532
|
const scheduleDetachedFn = typeof options.scheduleDetached === 'function'
|
|
1176
1533
|
? options.scheduleDetached
|
|
1177
1534
|
: scheduleDetachedRestart
|
|
1535
|
+
// The rollback watcher is started OUTSIDE the host process, so it survives the
|
|
1536
|
+
// death of the host — which is what a hosted restart does.
|
|
1537
|
+
const launchWatcherFn = typeof options.launchWatcher === 'function'
|
|
1538
|
+
? options.launchWatcher
|
|
1539
|
+
: launchDetachedWatcher
|
|
1540
|
+
// The health confirmation of the new instance (in-process; the detached
|
|
1541
|
+
// watcher has its own, outside the host).
|
|
1542
|
+
const probeInstanceFn = typeof options.probeInstance === 'function'
|
|
1543
|
+
? options.probeInstance
|
|
1544
|
+
: waitForHealthyInstance
|
|
1178
1545
|
const stateFile = resolveStateFile(config)
|
|
1179
1546
|
|
|
1180
1547
|
const loaded = readState(stateFile, Date.now())
|
|
@@ -1188,6 +1555,20 @@ export function createRuntime(ctx, config, options = {}) {
|
|
|
1188
1555
|
state = boot.state
|
|
1189
1556
|
const bootId = state.bootId
|
|
1190
1557
|
|
|
1558
|
+
// THE INSTANCE RECORD exists from the first moment of this boot: it carries
|
|
1559
|
+
// the boot id (which tells the rollback watcher that this instance is the
|
|
1560
|
+
// one being replaced). As soon as the web server is known, the record is
|
|
1561
|
+
// written again — with the port of this instance and its access link.
|
|
1562
|
+
try {
|
|
1563
|
+
writeAccessRecord({
|
|
1564
|
+
stateDir: path.dirname(stateFile),
|
|
1565
|
+
bootId,
|
|
1566
|
+
pid: process.pid,
|
|
1567
|
+
})
|
|
1568
|
+
} catch (error) {
|
|
1569
|
+
log.warn('could not write the instance record:', String(error))
|
|
1570
|
+
}
|
|
1571
|
+
|
|
1191
1572
|
let pollTimer
|
|
1192
1573
|
let recoverTimer
|
|
1193
1574
|
let disposed = false
|
|
@@ -1197,6 +1578,12 @@ export function createRuntime(ctx, config, options = {}) {
|
|
|
1197
1578
|
* logged, never thrown: the plugin still loads and the other tools still work.
|
|
1198
1579
|
*/
|
|
1199
1580
|
const toolErrors = []
|
|
1581
|
+
/**
|
|
1582
|
+
* The routes of this plugin that the web server accepted, and the ones it
|
|
1583
|
+
* refused. The boot check of the restart record reports them.
|
|
1584
|
+
*/
|
|
1585
|
+
const registeredRoutes = []
|
|
1586
|
+
const routeErrors = []
|
|
1200
1587
|
|
|
1201
1588
|
const persist = () => {
|
|
1202
1589
|
try {
|
|
@@ -1233,7 +1620,43 @@ export function createRuntime(ctx, config, options = {}) {
|
|
|
1233
1620
|
/** When the last conversation activity was observed; the settle window starts here. */
|
|
1234
1621
|
let lastActivityAt = Date.now()
|
|
1235
1622
|
|
|
1236
|
-
/**
|
|
1623
|
+
/**
|
|
1624
|
+
* When a busy conversation was seen for the last time.
|
|
1625
|
+
*
|
|
1626
|
+
* The quiet window measures the time since the last REAL work: a running
|
|
1627
|
+
* turn or a queued message. The host polls the agent list every few
|
|
1628
|
+
* seconds, so this marker follows the work exactly — and it is NOT reset by
|
|
1629
|
+
* the routine session events the interface produces while it synchronises
|
|
1630
|
+
* (those used to reset the window every 2-3 seconds, so a window longer
|
|
1631
|
+
* than that interval could never fill and the restart never fired).
|
|
1632
|
+
*/
|
|
1633
|
+
let lastBusyAt = 0
|
|
1634
|
+
|
|
1635
|
+
/**
|
|
1636
|
+
* When the "the restart is waiting for a long time" line was logged last.
|
|
1637
|
+
*
|
|
1638
|
+
* The gate polls every few seconds, so the line is rate limited: at most one
|
|
1639
|
+
* per minute for the SAME restart request. A new request (`requestedAt`
|
|
1640
|
+
* changed) logs right away.
|
|
1641
|
+
*/
|
|
1642
|
+
let longWaitLog = { at: 0, requestedAt: 0 }
|
|
1643
|
+
|
|
1644
|
+
/**
|
|
1645
|
+
* When the diagnostic line of the restart gate was logged last.
|
|
1646
|
+
*
|
|
1647
|
+
* The gate is evaluated every few seconds, so the line is rate limited to
|
|
1648
|
+
* once a minute: it says what the gate decided and why, which is the first
|
|
1649
|
+
* thing to look at when "the restart never happens" is reported.
|
|
1650
|
+
*/
|
|
1651
|
+
let lastDecisionLogAt = 0
|
|
1652
|
+
|
|
1653
|
+
/**
|
|
1654
|
+
* Record REAL conversation activity (an agent turn that started or ended, a
|
|
1655
|
+
* message that was queued, a session event that says work happened).
|
|
1656
|
+
*
|
|
1657
|
+
* The routine session events of the interface do NOT go through here: they
|
|
1658
|
+
* would keep the quiet window from ever filling.
|
|
1659
|
+
*/
|
|
1237
1660
|
const noteActivity = () => {
|
|
1238
1661
|
lastActivityAt = Date.now()
|
|
1239
1662
|
}
|
|
@@ -1253,6 +1676,53 @@ export function createRuntime(ctx, config, options = {}) {
|
|
|
1253
1676
|
return count
|
|
1254
1677
|
}
|
|
1255
1678
|
|
|
1679
|
+
/**
|
|
1680
|
+
* A short human-readable duration for the log ("45s", "2m 5s", "1h 3m").
|
|
1681
|
+
*
|
|
1682
|
+
* @param {number} ms
|
|
1683
|
+
* @returns {string}
|
|
1684
|
+
*/
|
|
1685
|
+
const describeWait = (ms) => {
|
|
1686
|
+
const total = Math.max(0, Math.round(Number(ms) || 0))
|
|
1687
|
+
const seconds = Math.floor(total / 1000)
|
|
1688
|
+
if (seconds < 60) return `${seconds}s`
|
|
1689
|
+
const minutes = Math.floor(seconds / 60)
|
|
1690
|
+
if (minutes < 60) {
|
|
1691
|
+
const rest = seconds % 60
|
|
1692
|
+
return rest === 0 ? `${minutes}m` : `${minutes}m ${rest}s`
|
|
1693
|
+
}
|
|
1694
|
+
const hours = Math.floor(minutes / 60)
|
|
1695
|
+
const restMinutes = minutes % 60
|
|
1696
|
+
return restMinutes === 0 ? `${hours}h` : `${hours}h ${restMinutes}m`
|
|
1697
|
+
}
|
|
1698
|
+
|
|
1699
|
+
/**
|
|
1700
|
+
* The readable names of the sessions that hold the restart.
|
|
1701
|
+
*
|
|
1702
|
+
* A session without a title falls back to its id: the log has to say WHO
|
|
1703
|
+
* holds the restart, and an empty name would say nothing.
|
|
1704
|
+
*
|
|
1705
|
+
* @param {any[]} blockedBy
|
|
1706
|
+
* @returns {string}
|
|
1707
|
+
*/
|
|
1708
|
+
const describeBlockers = (blockedBy) => {
|
|
1709
|
+
const list = Array.isArray(blockedBy) ? blockedBy : []
|
|
1710
|
+
const names = []
|
|
1711
|
+
for (const entry of list) {
|
|
1712
|
+
if (!entry || typeof entry !== 'object') continue
|
|
1713
|
+
const name = typeof entry.name === 'string' && entry.name.trim() !== ''
|
|
1714
|
+
? entry.name.trim()
|
|
1715
|
+
: (typeof entry.label === 'string' && entry.label.trim() !== ''
|
|
1716
|
+
? entry.label.trim()
|
|
1717
|
+
: String(entry.sessionId || ''))
|
|
1718
|
+
if (name === '') continue
|
|
1719
|
+
names.push(name)
|
|
1720
|
+
}
|
|
1721
|
+
if (names.length === 0) return 'another conversation'
|
|
1722
|
+
if (names.length === 1) return names[0]
|
|
1723
|
+
return names.join(', ')
|
|
1724
|
+
}
|
|
1725
|
+
|
|
1256
1726
|
/**
|
|
1257
1727
|
* Whether this agent is a subagent (a delegated child), and which session
|
|
1258
1728
|
* created it.
|
|
@@ -1329,10 +1799,28 @@ export function createRuntime(ctx, config, options = {}) {
|
|
|
1329
1799
|
}
|
|
1330
1800
|
}
|
|
1331
1801
|
|
|
1802
|
+
/**
|
|
1803
|
+
* The tracker of the QUEUED messages.
|
|
1804
|
+
*
|
|
1805
|
+
* It remembers for how long a session has had an unclaimed inbox message, so
|
|
1806
|
+
* a message that nobody claims stops blocking the restart instead of
|
|
1807
|
+
* blocking it for ever. A session that is RUNNING is never judged here: a
|
|
1808
|
+
* live turn always blocks, under every grace setting.
|
|
1809
|
+
*/
|
|
1810
|
+
const queuedTracker = createQueuedTracker()
|
|
1811
|
+
|
|
1812
|
+
/** The sessions the tracker saw in the last round (for the cleanup). */
|
|
1813
|
+
const queuedSessions = new Set()
|
|
1814
|
+
|
|
1332
1815
|
/**
|
|
1333
1816
|
* Every conversation that is still working, subagents and teammates included.
|
|
1334
1817
|
* 'running' is a live turn; 'queued' is inbox work that has not started yet.
|
|
1335
1818
|
*
|
|
1819
|
+
* A queued message only blocks the restart until the grace window
|
|
1820
|
+
* (`queuedGraceMs`) is over: after that it is stuck — nobody will ever claim
|
|
1821
|
+
* it — so it stops blocking the restart and is named in the log. A running
|
|
1822
|
+
* conversation ALWAYS blocks.
|
|
1823
|
+
*
|
|
1336
1824
|
* Every entry carries the kind of the blocker and a readable name, so the
|
|
1337
1825
|
* interface can tell a session from a subagent and never has to show a raw
|
|
1338
1826
|
* session id.
|
|
@@ -1345,15 +1833,52 @@ export function createRuntime(ctx, config, options = {}) {
|
|
|
1345
1833
|
} catch {
|
|
1346
1834
|
agents = []
|
|
1347
1835
|
}
|
|
1836
|
+
const seen = new Set()
|
|
1348
1837
|
for (const agent of agents) {
|
|
1349
1838
|
try {
|
|
1350
1839
|
if (!agent || !agent.id) continue
|
|
1351
1840
|
const sessionId = String(agent.id)
|
|
1841
|
+
seen.add(sessionId)
|
|
1352
1842
|
const described = describeBlocker(sessionId, agent)
|
|
1353
|
-
if (agent.status === 'running')
|
|
1354
|
-
|
|
1843
|
+
if (agent.status === 'running') {
|
|
1844
|
+
// A live turn ALWAYS blocks the restart — under every grace
|
|
1845
|
+
// setting. The queued timer of that session is dropped: the work is
|
|
1846
|
+
// being done, so it is not a stuck queue.
|
|
1847
|
+
queuedTracker.forget(sessionId)
|
|
1848
|
+
out.push({ sessionId, reason: 'running', ...described })
|
|
1849
|
+
} else if (pendingInboxCount(agent) > 0) {
|
|
1850
|
+
const observed = queuedTracker.observe(sessionId, true, config.queuedGraceMs)
|
|
1851
|
+
if (observed.becameStale) {
|
|
1852
|
+
// The message has been waiting longer than the grace window and
|
|
1853
|
+
// nobody claimed it: it is stuck. The session is named here, so
|
|
1854
|
+
// the user can see WHICH conversation holds the queue.
|
|
1855
|
+
log.warn(
|
|
1856
|
+
`the queued message of session ${sessionId} has been waiting ${describeWait(observed.ageMs)} and nobody claimed it; it no longer blocks the restart`,
|
|
1857
|
+
{
|
|
1858
|
+
sessionId,
|
|
1859
|
+
ageMs: observed.ageMs,
|
|
1860
|
+
queuedGraceMs: config.queuedGraceMs,
|
|
1861
|
+
},
|
|
1862
|
+
)
|
|
1863
|
+
}
|
|
1864
|
+
if (observed.blocking) out.push({ sessionId, reason: 'queued', ...described })
|
|
1865
|
+
} else {
|
|
1866
|
+
// No queued work any more: the timer of that session is dropped, so
|
|
1867
|
+
// a NEW queue starts counting from zero.
|
|
1868
|
+
queuedTracker.observe(sessionId, false, config.queuedGraceMs)
|
|
1869
|
+
}
|
|
1355
1870
|
} catch {}
|
|
1356
1871
|
}
|
|
1872
|
+
// A session that is gone (closed) drops its record as well.
|
|
1873
|
+
for (const sessionId of queuedSessions) {
|
|
1874
|
+
if (!seen.has(sessionId)) queuedTracker.forget(sessionId)
|
|
1875
|
+
}
|
|
1876
|
+
queuedSessions.clear()
|
|
1877
|
+
for (const sessionId of seen) queuedSessions.add(sessionId)
|
|
1878
|
+
// Real work is still going on: the quiet window starts over. This is the
|
|
1879
|
+
// marker that makes the quiet window measure "since the last real work"
|
|
1880
|
+
// instead of "since the last session event of any kind".
|
|
1881
|
+
if (out.length > 0) lastBusyAt = Date.now()
|
|
1357
1882
|
return out
|
|
1358
1883
|
}
|
|
1359
1884
|
|
|
@@ -1695,13 +2220,33 @@ export function createRuntime(ctx, config, options = {}) {
|
|
|
1695
2220
|
*/
|
|
1696
2221
|
const pendingRestart = () => state.restart !== null && state.restart !== undefined
|
|
1697
2222
|
|
|
1698
|
-
/**
|
|
1699
|
-
|
|
2223
|
+
/**
|
|
2224
|
+
* The moment the quiet window started.
|
|
2225
|
+
*
|
|
2226
|
+
* It is the LATER of the two markers:
|
|
2227
|
+
* - `lastActivityAt`: the last real activity signal (a turn that started
|
|
2228
|
+
* or ended, a message that was queued);
|
|
2229
|
+
* - `lastBusyAt`: the last moment a conversation was seen working (a
|
|
2230
|
+
* running turn or a queued message).
|
|
2231
|
+
*
|
|
2232
|
+
* The routine session events of the interface (a title, a projection, a
|
|
2233
|
+
* context request) are NOT activity: they arrive every two to three
|
|
2234
|
+
* seconds, so counting them would keep the quiet window from ever filling
|
|
2235
|
+
* — and the restart would never fire.
|
|
2236
|
+
*
|
|
2237
|
+
* @returns {number} a timestamp in milliseconds
|
|
2238
|
+
*/
|
|
2239
|
+
const quietSince = () => Math.max(lastActivityAt, lastBusyAt)
|
|
2240
|
+
|
|
2241
|
+
/** How long the host has been quiet (no real conversation activity at all). */
|
|
2242
|
+
const quietForMs = () => Math.max(0, Date.now() - quietSince())
|
|
1700
2243
|
|
|
1701
2244
|
/**
|
|
1702
|
-
* The
|
|
1703
|
-
* conversation finished
|
|
1704
|
-
*
|
|
2245
|
+
* The RESTART reserve: how much longer the host waits after the last
|
|
2246
|
+
* conversation finished before it is really restarted.
|
|
2247
|
+
*
|
|
2248
|
+
* A restart is a daily operation, so the default is none: the host restarts
|
|
2249
|
+
* as soon as the conversations are done and the settle window has passed.
|
|
1705
2250
|
*/
|
|
1706
2251
|
const graceMs = () => config.restartGraceMs
|
|
1707
2252
|
|
|
@@ -1716,11 +2261,45 @@ export function createRuntime(ctx, config, options = {}) {
|
|
|
1716
2261
|
/** The two-layer restart gate: no busy conversation AND a quiet window. */
|
|
1717
2262
|
const gate = () => restartGate(state, {
|
|
1718
2263
|
busyAgents: busyAgents(),
|
|
1719
|
-
lastActivityAt,
|
|
2264
|
+
lastActivityAt: quietSince(),
|
|
1720
2265
|
now: Date.now(),
|
|
1721
2266
|
settleMs: effectiveSettleMs(),
|
|
1722
2267
|
})
|
|
1723
2268
|
|
|
2269
|
+
/**
|
|
2270
|
+
* The description of a LONG wait, or null when the restart is not waiting
|
|
2271
|
+
* for the long time yet.
|
|
2272
|
+
*
|
|
2273
|
+
* A pending restart that passed the hard deadline while a conversation was
|
|
2274
|
+
* still running is not "just a little longer": it is a long wait, and the
|
|
2275
|
+
* user has to SEE it — how long it has been waiting, why, and who holds it.
|
|
2276
|
+
* The request itself is never dropped: the restart stays pending and fires
|
|
2277
|
+
* as soon as the gate is quiet again.
|
|
2278
|
+
*
|
|
2279
|
+
* @param {number} [now]
|
|
2280
|
+
* @returns {{ since: number, reason: string, waitedMs: number, blockedBy: any[] }|null}
|
|
2281
|
+
*/
|
|
2282
|
+
const longWaitInfo = (now = Date.now()) => {
|
|
2283
|
+
const restart = state.restart
|
|
2284
|
+
if (!restart || typeof restart !== 'object') return null
|
|
2285
|
+
const requestedAt = Number.isFinite(restart.requestedAt) ? restart.requestedAt : 0
|
|
2286
|
+
if (requestedAt <= 0) return null
|
|
2287
|
+
const hardMs = Number.isFinite(config.deferHardMs) ? config.deferHardMs : 0
|
|
2288
|
+
// The long wait begins when the hard deadline is reached.
|
|
2289
|
+
const since = requestedAt + hardMs
|
|
2290
|
+
if (now < since) return null
|
|
2291
|
+
const verdict = gate()
|
|
2292
|
+
if (verdict.allow) return null
|
|
2293
|
+
const blockedBy = goalAwareBlockingDetails(runtime, verdict.blockedBy)
|
|
2294
|
+
if (blockedBy.length === 0) return null
|
|
2295
|
+
return {
|
|
2296
|
+
since,
|
|
2297
|
+
reason: 'hard-deadline-waiting-for-conversations',
|
|
2298
|
+
waitedMs: Math.max(0, now - requestedAt),
|
|
2299
|
+
blockedBy,
|
|
2300
|
+
}
|
|
2301
|
+
}
|
|
2302
|
+
|
|
1724
2303
|
/**
|
|
1725
2304
|
* Cached session titles, used to label the blocking sessions in the
|
|
1726
2305
|
* client's "pending restart" details panel.
|
|
@@ -1853,20 +2432,31 @@ const titleOfSummary = (item) => {
|
|
|
1853
2432
|
}
|
|
1854
2433
|
|
|
1855
2434
|
/**
|
|
1856
|
-
* Release the continuation request of
|
|
2435
|
+
* Release the continuation request of the sessions that asked for the current
|
|
1857
2436
|
* restart (cancel / give-up paths). Requests are coalesced into ONE restart, so
|
|
1858
2437
|
* releasing only the last requester would leave the others pinned as pending.
|
|
2438
|
+
*
|
|
2439
|
+
* `only` releases EXACTLY those sessions — that is how one session withdraws
|
|
2440
|
+
* its own request without touching the requests of the other sessions.
|
|
2441
|
+
*
|
|
2442
|
+
* @param {string[]|null} [only] the sessions to release; null = every requester
|
|
1859
2443
|
*/
|
|
1860
|
-
const releaseRequesters = () => {
|
|
1861
|
-
const restart = state.restart
|
|
1862
|
-
if (!restart) return
|
|
2444
|
+
const releaseRequesters = (only = null) => {
|
|
1863
2445
|
const ids = new Set()
|
|
1864
|
-
if (Array.isArray(
|
|
1865
|
-
for (const
|
|
1866
|
-
if (
|
|
2446
|
+
if (Array.isArray(only)) {
|
|
2447
|
+
for (const sessionId of only) {
|
|
2448
|
+
if (typeof sessionId === 'string' && sessionId) ids.add(sessionId)
|
|
1867
2449
|
}
|
|
2450
|
+
} else {
|
|
2451
|
+
const restart = state.restart
|
|
2452
|
+
if (!restart) return
|
|
2453
|
+
if (Array.isArray(restart.requesters)) {
|
|
2454
|
+
for (const requester of restart.requesters) {
|
|
2455
|
+
if (requester && typeof requester.sessionId === 'string' && requester.sessionId) ids.add(requester.sessionId)
|
|
2456
|
+
}
|
|
2457
|
+
}
|
|
2458
|
+
if (typeof restart.bySessionId === 'string' && restart.bySessionId) ids.add(restart.bySessionId)
|
|
1868
2459
|
}
|
|
1869
|
-
if (typeof restart.bySessionId === 'string' && restart.bySessionId) ids.add(restart.bySessionId)
|
|
1870
2460
|
for (const sessionId of ids) {
|
|
1871
2461
|
const entry = state.sessions ? state.sessions[sessionId] : undefined
|
|
1872
2462
|
if (!entry) continue
|
|
@@ -1937,6 +2527,644 @@ const titleOfSummary = (item) => {
|
|
|
1937
2527
|
return 'http://127.0.0.1:' + port + resolveHealthPath(config)
|
|
1938
2528
|
}
|
|
1939
2529
|
|
|
2530
|
+
/**
|
|
2531
|
+
* The state directory of this plugin (the checkpoint, the backups, the
|
|
2532
|
+
* instance record and the helper scripts live next to each other).
|
|
2533
|
+
*/
|
|
2534
|
+
const stateDirOf = () => path.dirname(stateFile)
|
|
2535
|
+
|
|
2536
|
+
/**
|
|
2537
|
+
* The durable record of the last restart: the snapshot that was taken, the
|
|
2538
|
+
* verdict of the new instance and its four boot checks.
|
|
2539
|
+
*
|
|
2540
|
+
* @returns {object|null} the summary (null when no restart happened yet)
|
|
2541
|
+
*/
|
|
2542
|
+
const lastRestartVerdict = () => summarizeRestartVerdict(readRestartVerdict(stateDirOf()))
|
|
2543
|
+
|
|
2544
|
+
/**
|
|
2545
|
+
* The version of THIS plugin (the one the user runs).
|
|
2546
|
+
*
|
|
2547
|
+
* @returns {string}
|
|
2548
|
+
*/
|
|
2549
|
+
const pluginVersion = () => {
|
|
2550
|
+
try {
|
|
2551
|
+
const raw = fs.readFileSync(new URL('../package.json', import.meta.url), 'utf8')
|
|
2552
|
+
const parsed = JSON.parse(raw)
|
|
2553
|
+
return typeof parsed.version === 'string' && parsed.version !== '' ? parsed.version : 'unknown'
|
|
2554
|
+
} catch {
|
|
2555
|
+
return 'unknown'
|
|
2556
|
+
}
|
|
2557
|
+
}
|
|
2558
|
+
|
|
2559
|
+
/**
|
|
2560
|
+
* Write (or update) the durable record of the restart.
|
|
2561
|
+
*
|
|
2562
|
+
* The record is written by BOTH halves of a restart: the instance that is
|
|
2563
|
+
* replaced writes the `pending` half (with the snapshot), the instance that
|
|
2564
|
+
* replaces it writes the final verdict. `skipIfFinalized` keeps the older
|
|
2565
|
+
* instance from overwriting a verdict that the new one already wrote.
|
|
2566
|
+
*
|
|
2567
|
+
* @param {object} [patch] the fields to change
|
|
2568
|
+
* @param {{ skipIfFinalized?: boolean }} [options]
|
|
2569
|
+
* @returns {object|null} the record that is on disk now
|
|
2570
|
+
*/
|
|
2571
|
+
const recordRestartVerdict = (patch = {}, options = {}) => {
|
|
2572
|
+
const dir = stateDirOf()
|
|
2573
|
+
let current = null
|
|
2574
|
+
try {
|
|
2575
|
+
current = readRestartVerdict(dir)
|
|
2576
|
+
} catch {
|
|
2577
|
+
current = null
|
|
2578
|
+
}
|
|
2579
|
+
if (options.skipIfFinalized === true
|
|
2580
|
+
&& current !== null
|
|
2581
|
+
&& current.bootId !== null
|
|
2582
|
+
&& current.result !== RESTART_RESULTS.PENDING) {
|
|
2583
|
+
return current
|
|
2584
|
+
}
|
|
2585
|
+
const merged = buildRestartVerdict({ ...(current ?? {}), ...patch })
|
|
2586
|
+
const written = writeRestartVerdict(dir, merged)
|
|
2587
|
+
if (!written.ok) log.warn('could not write the restart record:', written.error)
|
|
2588
|
+
return merged
|
|
2589
|
+
}
|
|
2590
|
+
|
|
2591
|
+
/**
|
|
2592
|
+
* WHICH MACHINE IS THIS?
|
|
2593
|
+
*
|
|
2594
|
+
* The same answer the graceful shutdown uses (`lib/host-kind.js`): Windows,
|
|
2595
|
+
* WSL, a real Linux, a macOS. It matters for the autostart: inside WSL the
|
|
2596
|
+
* "start at login" is a different thing (a systemd user unit + the linger,
|
|
2597
|
+
* or a task on the Windows side), and the Docker/Compose answer of a
|
|
2598
|
+
* container is simply wrong there.
|
|
2599
|
+
*
|
|
2600
|
+
* @returns {object} the host (kind, evidence, suspect, distro)
|
|
2601
|
+
*/
|
|
2602
|
+
const hostKindOf = () => detectHostKind({ platform: env.platform, env: process.env })
|
|
2603
|
+
|
|
2604
|
+
/**
|
|
2605
|
+
* THE PLATFORM GUIDE: how "start at login" works on THIS machine.
|
|
2606
|
+
*
|
|
2607
|
+
* Windows (a scheduled task, with the startup folder as the fallback),
|
|
2608
|
+
* Linux (a systemd user unit + the linger hint), WSL (the same, or the two
|
|
2609
|
+
* ways out when there is no systemd), a container (the orchestrator) and the
|
|
2610
|
+
* unknown host (nothing, and no guessing). The answer is computed ONCE
|
|
2611
|
+
* (the machine does not change while the host runs) and the settings page
|
|
2612
|
+
* reads it.
|
|
2613
|
+
*
|
|
2614
|
+
* @returns {object} the advice (kind, reason, notes, commands, options)
|
|
2615
|
+
*/
|
|
2616
|
+
let cachedAutostart = null
|
|
2617
|
+
const autostartAdviceOf = () => {
|
|
2618
|
+
if (cachedAutostart !== null) return cachedAutostart
|
|
2619
|
+
const host = hostKindOf()
|
|
2620
|
+
const init = detectInitSystems({ platform: env.platform, homeDir: os.homedir() })
|
|
2621
|
+
cachedAutostart = planAutostartAdvice({
|
|
2622
|
+
host,
|
|
2623
|
+
platform: env.platform,
|
|
2624
|
+
hasSystemd: init.systemdUser,
|
|
2625
|
+
container: detectContainer({ env: process.env }).container,
|
|
2626
|
+
user: typeof process.env.USER === 'string' ? process.env.USER : '',
|
|
2627
|
+
distro: host.distro,
|
|
2628
|
+
})
|
|
2629
|
+
return cachedAutostart
|
|
2630
|
+
}
|
|
2631
|
+
|
|
2632
|
+
/**
|
|
2633
|
+
* WHAT IS ON DISK RIGHT NOW: the backups of this host.
|
|
2634
|
+
*
|
|
2635
|
+
* The settings page shows it ("what is backed up right now"), so the facts
|
|
2636
|
+
* have to come from the disk and not from the browser: the browser may not
|
|
2637
|
+
* even see the disk. Only the DIRECTORY is read — the manifest of every
|
|
2638
|
+
* copy — because one copy can be 150 MB and its contents are never walked.
|
|
2639
|
+
*
|
|
2640
|
+
* @returns {{ dir: string, max: number, count: number, items: Array<object> }}
|
|
2641
|
+
*/
|
|
2642
|
+
const backupsInfo = () => {
|
|
2643
|
+
const dir = backupsDirFor(stateDirOf())
|
|
2644
|
+
const items = listBackups(dir).map((item) => ({
|
|
2645
|
+
name: item.name,
|
|
2646
|
+
dir: item.dir,
|
|
2647
|
+
at: item.at,
|
|
2648
|
+
atLocal: item.atLocal,
|
|
2649
|
+
bytes: item.bytes,
|
|
2650
|
+
files: item.files,
|
|
2651
|
+
includePlugins: item.includePlugins,
|
|
2652
|
+
}))
|
|
2653
|
+
return {
|
|
2654
|
+
dir,
|
|
2655
|
+
max: config.maxBackups,
|
|
2656
|
+
count: items.length,
|
|
2657
|
+
items,
|
|
2658
|
+
}
|
|
2659
|
+
}
|
|
2660
|
+
|
|
2661
|
+
/**
|
|
2662
|
+
* THE DIAGNOSTICS of "report a problem".
|
|
2663
|
+
*
|
|
2664
|
+
* One generator, on the host: the browser copies the text it gets, so the
|
|
2665
|
+
* two can never drift. It carries the facts a bug report needs — and NOT
|
|
2666
|
+
* the access token of this instance (the generator only copies the fields
|
|
2667
|
+
* it knows, and everything it copies is redacted again).
|
|
2668
|
+
*
|
|
2669
|
+
* @returns {string}
|
|
2670
|
+
*/
|
|
2671
|
+
const diagnosticsText = () => buildDiagnostics({
|
|
2672
|
+
version: pluginVersion(),
|
|
2673
|
+
platform: env.platform,
|
|
2674
|
+
hostKind: hostKindOf().kind,
|
|
2675
|
+
isDesktopApp: env.isDesktopApp === true,
|
|
2676
|
+
startup: startupModeOf(),
|
|
2677
|
+
supervisor: supervisorStatusOf(),
|
|
2678
|
+
restartMode: config.restartMode,
|
|
2679
|
+
lastRestart: lastRestartVerdict(),
|
|
2680
|
+
stateDir: stateDirOf(),
|
|
2681
|
+
})
|
|
2682
|
+
|
|
2683
|
+
/**
|
|
2684
|
+
* HOW THIS HOST WAS STARTED (the startup matrix).
|
|
2685
|
+
*
|
|
2686
|
+
* The answer decides what the plugin may do: only a `bare` host (nobody
|
|
2687
|
+
* manages it) may get the supervisor of this plugin or a self-replay
|
|
2688
|
+
* command. Everything else — an external supervisor, a container, the
|
|
2689
|
+
* desktop app, or a host whose starter cannot be established — must not
|
|
2690
|
+
* lead to a second host.
|
|
2691
|
+
*
|
|
2692
|
+
* @returns {object} the classification (mode, kind, evidence, …)
|
|
2693
|
+
*/
|
|
2694
|
+
const startupModeOf = () => classifyStartup({
|
|
2695
|
+
platform: env.platform,
|
|
2696
|
+
env: process.env,
|
|
2697
|
+
ppid: process.ppid,
|
|
2698
|
+
isDesktopApp: env.isDesktopApp === true,
|
|
2699
|
+
supervised: detectSupervisor({ env: process.env, stateDir: stateDirOf() }).supervised,
|
|
2700
|
+
container: detectContainer({ env: process.env }).container,
|
|
2701
|
+
argv: process.argv,
|
|
2702
|
+
execPath: process.execPath,
|
|
2703
|
+
readChain: readProcessChain,
|
|
2704
|
+
})
|
|
2705
|
+
|
|
2706
|
+
/**
|
|
2707
|
+
* What the plugin knows about the supervisor.
|
|
2708
|
+
*
|
|
2709
|
+
* The parent chain of this very process is part of the answer: a host that
|
|
2710
|
+
* somebody ELSE supervises (systemd, pm2, a `supervise-*.sh` script) must
|
|
2711
|
+
* not be restarted by a self-replay — the supervisor starts it again, and a
|
|
2712
|
+
* second host on the same port is a double start.
|
|
2713
|
+
*
|
|
2714
|
+
* @returns {object} the status (mode, kind, installed, running, restartCapable)
|
|
2715
|
+
*/
|
|
2716
|
+
const supervisorStatusOf = () => supervisorStatus({
|
|
2717
|
+
stateDir: stateDirOf(),
|
|
2718
|
+
env: process.env,
|
|
2719
|
+
platform: env.platform,
|
|
2720
|
+
mode: config.supervisorMode,
|
|
2721
|
+
ppid: process.ppid,
|
|
2722
|
+
// The desktop app brings ITSELF back up (`restartMode: app`): it is not
|
|
2723
|
+
// "nobody knows", and the restart is available.
|
|
2724
|
+
isDesktopApp: env.isDesktopApp === true,
|
|
2725
|
+
// HOW this host was started (the startup matrix), with its evidence.
|
|
2726
|
+
startup: startupModeOf(),
|
|
2727
|
+
// THE PLATFORM GUIDE: how "start at login" works here (Windows / Linux /
|
|
2728
|
+
// WSL / container), with the commands the user can copy.
|
|
2729
|
+
autostart: autostartAdviceOf(),
|
|
2730
|
+
})
|
|
2731
|
+
|
|
2732
|
+
/**
|
|
2733
|
+
* The start command the supervisor would use.
|
|
2734
|
+
*
|
|
2735
|
+
* This is the PLAIN command line of this very process: the interpreter, its
|
|
2736
|
+
* own arguments and the script it runs. The full derivation (with the
|
|
2737
|
+
* refusal rules) belongs to the "E" step; this is the honest minimum the
|
|
2738
|
+
* supervisor needs to be able to start the host again.
|
|
2739
|
+
*
|
|
2740
|
+
* @returns {{ execPath: string, execArgv: string[], argv: string[], cwd: string }}
|
|
2741
|
+
*/
|
|
2742
|
+
const supervisorStartCommand = () => ({
|
|
2743
|
+
execPath: process.execPath,
|
|
2744
|
+
execArgv: Array.isArray(process.execArgv) ? process.execArgv.map(String) : [],
|
|
2745
|
+
argv: Array.isArray(process.argv) ? process.argv.slice(1).map(String) : [],
|
|
2746
|
+
cwd: process.cwd(),
|
|
2747
|
+
})
|
|
2748
|
+
|
|
2749
|
+
/**
|
|
2750
|
+
* The environment the supervisor hands to the host it starts.
|
|
2751
|
+
*
|
|
2752
|
+
* Only a WHITELIST is inherited (the daemon does that too): the host's own
|
|
2753
|
+
* internal variables must not leak into the next start, or the profile would
|
|
2754
|
+
* drift.
|
|
2755
|
+
*
|
|
2756
|
+
* @returns {Record<string, string>}
|
|
2757
|
+
*/
|
|
2758
|
+
const supervisorEnv = () => {
|
|
2759
|
+
const env = {}
|
|
2760
|
+
const source = process.env || {}
|
|
2761
|
+
for (const key of [
|
|
2762
|
+
'PATH', 'HOME', 'USER', 'LOGNAME', 'SHELL', 'LANG', 'LC_ALL', 'SystemRoot',
|
|
2763
|
+
'TEMP', 'TMP', 'APPDATA', 'LOCALAPPDATA', 'ProgramData',
|
|
2764
|
+
'NUMBER_OF_PROCESSORS', 'WSLENV', 'WSL_DISTRO_NAME', 'WSL_INTEROP',
|
|
2765
|
+
]) {
|
|
2766
|
+
const value = source[key]
|
|
2767
|
+
if (typeof value === 'string' && value !== '') env[key] = value
|
|
2768
|
+
}
|
|
2769
|
+
// The two variables that say WHICH profile this host belongs to.
|
|
2770
|
+
if (typeof source.DSH_HOME === 'string' && source.DSH_HOME !== '') env.DSH_HOME = source.DSH_HOME
|
|
2771
|
+
if (typeof source.DSH_PROFILE === 'string' && source.DSH_PROFILE !== '') env.DSH_PROFILE = source.DSH_PROFILE
|
|
2772
|
+
return env
|
|
2773
|
+
}
|
|
2774
|
+
|
|
2775
|
+
/**
|
|
2776
|
+
* Install the supervisor.
|
|
2777
|
+
*
|
|
2778
|
+
* @param {{ confirmed?: boolean, dryRun?: boolean }} [options]
|
|
2779
|
+
* @returns {object} the outcome (ok, kind, steps, error, plan)
|
|
2780
|
+
*/
|
|
2781
|
+
const installSupervisorNow = (options = {}) => {
|
|
2782
|
+
const stateDir = stateDirOf()
|
|
2783
|
+
const profile = resolveProfileDirDetailed()
|
|
2784
|
+
const start = supervisorStartCommand()
|
|
2785
|
+
const result = installSupervisor({
|
|
2786
|
+
stateDir,
|
|
2787
|
+
profileDir: profile.dir,
|
|
2788
|
+
platform: env.platform,
|
|
2789
|
+
confirmed: options.confirmed === true,
|
|
2790
|
+
dryRun: options.dryRun === true,
|
|
2791
|
+
// The desktop app manages itself: a login record would start a SECOND
|
|
2792
|
+
// client at every login, so the installation is refused there.
|
|
2793
|
+
isDesktopApp: env.isDesktopApp === true,
|
|
2794
|
+
// The startup matrix: only a `bare` host (nobody manages it) may get
|
|
2795
|
+
// the supervisor of this plugin.
|
|
2796
|
+
startup: startupModeOf(),
|
|
2797
|
+
// WHICH MACHINE IS THIS? (WSL is not a Linux: the autostart there is a
|
|
2798
|
+
// different thing, and the Docker/Compose answer is a wrong one.)
|
|
2799
|
+
host: hostKindOf(),
|
|
2800
|
+
nodePath: process.execPath,
|
|
2801
|
+
execPath: start.execPath,
|
|
2802
|
+
execArgv: start.execArgv,
|
|
2803
|
+
argv: start.argv,
|
|
2804
|
+
cwd: start.cwd,
|
|
2805
|
+
env: supervisorEnv(),
|
|
2806
|
+
profileEnv: {
|
|
2807
|
+
...(process.env.DSH_HOME ? { DSH_HOME: process.env.DSH_HOME } : {}),
|
|
2808
|
+
...(process.env.DSH_PROFILE ? { DSH_PROFILE: process.env.DSH_PROFILE } : {}),
|
|
2809
|
+
},
|
|
2810
|
+
port: config.supervisorPort > 0 ? config.supervisorPort : (runtime.webPort || 0),
|
|
2811
|
+
healthUrl: restartHealthUrl() ?? '',
|
|
2812
|
+
ownerUser: typeof process.env.USER === 'string' ? process.env.USER : (typeof process.env.LOGNAME === 'string' ? process.env.LOGNAME : ''),
|
|
2813
|
+
timings: {
|
|
2814
|
+
restartDelayMs: config.supervisorRestartDelayMs,
|
|
2815
|
+
restartDelayMaxMs: config.supervisorRestartDelayMaxMs,
|
|
2816
|
+
startTimeoutMs: config.supervisorStartTimeoutMs,
|
|
2817
|
+
killGraceMs: config.supervisorKillGraceMs,
|
|
2818
|
+
},
|
|
2819
|
+
kind: config.supervisorKind,
|
|
2820
|
+
// The previous way of starting the host: the uninstall puts it back.
|
|
2821
|
+
previous: { restartMode: config.restartMode, restartCommand: config.restartCommand },
|
|
2822
|
+
log: (message) => log.warn(message),
|
|
2823
|
+
})
|
|
2824
|
+
if (result.ok) {
|
|
2825
|
+
log.warn('the supervisor was installed', { kind: result.kind, degraded: result.degraded })
|
|
2826
|
+
} else {
|
|
2827
|
+
log.error('the supervisor was not installed:', result.error)
|
|
2828
|
+
}
|
|
2829
|
+
return result
|
|
2830
|
+
}
|
|
2831
|
+
|
|
2832
|
+
/**
|
|
2833
|
+
* Uninstall the supervisor and give the machine back to the user.
|
|
2834
|
+
*
|
|
2835
|
+
* @returns {Promise<object>} the outcome (ok, steps, error, pending, previous)
|
|
2836
|
+
*/
|
|
2837
|
+
const uninstallSupervisorNow = async () => {
|
|
2838
|
+
const result = await uninstallSupervisor({
|
|
2839
|
+
stateDir: stateDirOf(),
|
|
2840
|
+
platform: env.platform,
|
|
2841
|
+
log: (message) => log.warn(message),
|
|
2842
|
+
})
|
|
2843
|
+
if (result.ok) {
|
|
2844
|
+
log.warn('the supervisor was uninstalled', { previous: result.previous })
|
|
2845
|
+
} else {
|
|
2846
|
+
log.error('the supervisor could not be removed completely:', result.error)
|
|
2847
|
+
}
|
|
2848
|
+
return result
|
|
2849
|
+
}
|
|
2850
|
+
|
|
2851
|
+
/**
|
|
2852
|
+
* Record the answer of the one-time question about the supervisor.
|
|
2853
|
+
*
|
|
2854
|
+
* @param {string} choice 'install' | 'external' | 'dismissed'
|
|
2855
|
+
* @param {object} [previous] the values that were in effect before
|
|
2856
|
+
* @returns {object} the recorded answer
|
|
2857
|
+
*/
|
|
2858
|
+
const recordSupervisorChoice = (choice, previous = null) => {
|
|
2859
|
+
const record = recordOnboarding(state, {
|
|
2860
|
+
feature: 'supervisor',
|
|
2861
|
+
version: SUPERVISOR_FEATURE_VERSION,
|
|
2862
|
+
choice,
|
|
2863
|
+
previous,
|
|
2864
|
+
now: Date.now(),
|
|
2865
|
+
})
|
|
2866
|
+
persist()
|
|
2867
|
+
return record
|
|
2868
|
+
}
|
|
2869
|
+
|
|
2870
|
+
/**
|
|
2871
|
+
* Derive the command that starts this host again — or say why there is
|
|
2872
|
+
* none. Nothing is written: this is the dry run (B).
|
|
2873
|
+
*
|
|
2874
|
+
* @param {{ fixedPort?: number }} [options]
|
|
2875
|
+
* @returns {Promise<object>} the derivation (the plan of what WOULD be written)
|
|
2876
|
+
*/
|
|
2877
|
+
const deriveStartCommandNow = async (options = {}) => {
|
|
2878
|
+
const detected = detectSupervisor({ env: process.env, stateDir: stateDirOf() })
|
|
2879
|
+
const external = detectExternalSupervisor({
|
|
2880
|
+
ppid: process.ppid,
|
|
2881
|
+
env: process.env,
|
|
2882
|
+
platform: env.platform,
|
|
2883
|
+
})
|
|
2884
|
+
const container = detectContainer({ env: process.env })
|
|
2885
|
+
const init = detectInitSystems({ platform: env.platform, homeDir: os.homedir() })
|
|
2886
|
+
// A random port has to be FIXED, or the cookie of the browser dies with
|
|
2887
|
+
// every restart. The port is picked once and then written down.
|
|
2888
|
+
let fixedPort = Number(options.fixedPort) > 0 ? Number(options.fixedPort) : 0
|
|
2889
|
+
if (fixedPort === 0 && config.supervisorPort > 0) fixedPort = config.supervisorPort
|
|
2890
|
+
if (fixedPort === 0 && Number(runtime.webPort) > 0 && config.supervisorPort === 0) {
|
|
2891
|
+
// The port of the running instance is the best candidate: it is the one
|
|
2892
|
+
// the browser knows.
|
|
2893
|
+
fixedPort = Number(runtime.webPort)
|
|
2894
|
+
}
|
|
2895
|
+
const derivation = deriveStartCommand({
|
|
2896
|
+
execPath: process.execPath,
|
|
2897
|
+
execArgv: process.execArgv,
|
|
2898
|
+
argv: process.argv,
|
|
2899
|
+
cwd: process.cwd(),
|
|
2900
|
+
platform: env.platform,
|
|
2901
|
+
env: process.env,
|
|
2902
|
+
pid: process.pid,
|
|
2903
|
+
isDesktopApp: env.isDesktopApp === true,
|
|
2904
|
+
webPort: runtime.webPort || 0,
|
|
2905
|
+
supervised: detected.supervised === true,
|
|
2906
|
+
externalSupervisor: external,
|
|
2907
|
+
container: container.container,
|
|
2908
|
+
// THE STARTUP MATRIX: it decides whether a self-replay is possible at
|
|
2909
|
+
// all (and whether the command must be detached from its session).
|
|
2910
|
+
startup: startupModeOf(),
|
|
2911
|
+
hasInit: env.platform === 'win32' ? true : init.systemdUser,
|
|
2912
|
+
fixedPort,
|
|
2913
|
+
currentRestartMode: config.restartMode,
|
|
2914
|
+
currentRestartCommand: config.restartCommand,
|
|
2915
|
+
healthPath: resolveHealthPath(config),
|
|
2916
|
+
})
|
|
2917
|
+
const plan = buildDryRunPlan(derivation)
|
|
2918
|
+
return {
|
|
2919
|
+
...plan,
|
|
2920
|
+
derivation,
|
|
2921
|
+
externalSupervisor: external,
|
|
2922
|
+
container: container.container,
|
|
2923
|
+
initReason: init.reason,
|
|
2924
|
+
// The fragment a container needs (there is no "at login" there).
|
|
2925
|
+
compose: derivation.reason === 'container-no-init' && derivation.argv
|
|
2926
|
+
? composeRestartFragment(derivation.argv)
|
|
2927
|
+
: null,
|
|
2928
|
+
}
|
|
2929
|
+
}
|
|
2930
|
+
|
|
2931
|
+
/**
|
|
2932
|
+
* Write the derived command into the profile configuration — only after the
|
|
2933
|
+
* user confirmed the dry run.
|
|
2934
|
+
*
|
|
2935
|
+
* @param {{ derivation?: object }} [options]
|
|
2936
|
+
* @returns {Promise<object>} the outcome
|
|
2937
|
+
*/
|
|
2938
|
+
const applyDerivedCommand = async (options = {}) => {
|
|
2939
|
+
const derivation = options.derivation && typeof options.derivation === 'object'
|
|
2940
|
+
? options.derivation
|
|
2941
|
+
: await deriveStartCommandNow()
|
|
2942
|
+
if (!derivation || derivation.ok !== true || !derivation.writes) {
|
|
2943
|
+
return {
|
|
2944
|
+
ok: false,
|
|
2945
|
+
error: 'nothing-to-write',
|
|
2946
|
+
reason: derivation ? derivation.reason : 'unknown',
|
|
2947
|
+
}
|
|
2948
|
+
}
|
|
2949
|
+
const previous = {
|
|
2950
|
+
restartMode: config.restartMode,
|
|
2951
|
+
restartCommand: config.restartCommand,
|
|
2952
|
+
}
|
|
2953
|
+
try {
|
|
2954
|
+
await runtime.applyConfigChoice({
|
|
2955
|
+
restartMode: derivation.writes.restartMode,
|
|
2956
|
+
restartCommand: derivation.writes.restartCommand,
|
|
2957
|
+
})
|
|
2958
|
+
} catch (error) {
|
|
2959
|
+
return { ok: false, error: describeError(error), reason: derivation.reason }
|
|
2960
|
+
}
|
|
2961
|
+
// The values that were in effect are kept, so the undo can put them back.
|
|
2962
|
+
recordSupervisorChoice('external', previous)
|
|
2963
|
+
log.warn('the derived restart command was written', {
|
|
2964
|
+
mode: derivation.writes.restartMode,
|
|
2965
|
+
reason: derivation.reason,
|
|
2966
|
+
})
|
|
2967
|
+
return { ok: true, writes: derivation.writes, previous, reason: derivation.reason }
|
|
2968
|
+
}
|
|
2969
|
+
|
|
2970
|
+
/**
|
|
2971
|
+
* Put the previous start command back (the undo).
|
|
2972
|
+
*
|
|
2973
|
+
* @returns {Promise<object>} the outcome
|
|
2974
|
+
*/
|
|
2975
|
+
const revertDerivedCommand = async () => {
|
|
2976
|
+
const record = state.onboarding
|
|
2977
|
+
const previous = record && typeof record === 'object' ? record.previous : null
|
|
2978
|
+
if (!previous || typeof previous.restartMode !== 'string') {
|
|
2979
|
+
return { ok: false, error: 'nothing-to-revert' }
|
|
2980
|
+
}
|
|
2981
|
+
try {
|
|
2982
|
+
await runtime.applyConfigChoice({
|
|
2983
|
+
restartMode: previous.restartMode,
|
|
2984
|
+
restartCommand: previous.restartCommand === null ? '' : previous.restartCommand,
|
|
2985
|
+
})
|
|
2986
|
+
} catch (error) {
|
|
2987
|
+
return { ok: false, error: describeError(error) }
|
|
2988
|
+
}
|
|
2989
|
+
log.warn('the previous restart command was put back', previous)
|
|
2990
|
+
return { ok: true, previous }
|
|
2991
|
+
}
|
|
2992
|
+
|
|
2993
|
+
/**
|
|
2994
|
+
* Write the marker that tells the rollback watcher "the plugin has decided
|
|
2995
|
+
* on its own — stand down".
|
|
2996
|
+
*
|
|
2997
|
+
* The watcher is the fallback for the case that the plugin dies with the
|
|
2998
|
+
* host. When the plugin SURVIVES the restart command it decides alone, and
|
|
2999
|
+
* this marker keeps the watcher from rolling back a restart that was fine.
|
|
3000
|
+
*
|
|
3001
|
+
* @param {{ decided: boolean, ok?: boolean, reason?: string }} outcome
|
|
3002
|
+
* @returns {boolean} whether the marker was written
|
|
3003
|
+
*/
|
|
3004
|
+
const writeRestartOutcome = (outcome) => {
|
|
3005
|
+
const file = path.join(stateDirOf(), RESTART_OUTCOME_FILE)
|
|
3006
|
+
try {
|
|
3007
|
+
fs.writeFileSync(file, JSON.stringify({
|
|
3008
|
+
decided: true,
|
|
3009
|
+
ok: outcome && outcome.ok === true,
|
|
3010
|
+
reason: outcome && typeof outcome.reason === 'string' ? outcome.reason : '',
|
|
3011
|
+
at: new Date().toISOString(),
|
|
3012
|
+
bootId,
|
|
3013
|
+
}, null, 2), 'utf8')
|
|
3014
|
+
return true
|
|
3015
|
+
} catch (error) {
|
|
3016
|
+
log.warn('could not write the restart outcome marker:', String(error))
|
|
3017
|
+
return false
|
|
3018
|
+
}
|
|
3019
|
+
}
|
|
3020
|
+
|
|
3021
|
+
/**
|
|
3022
|
+
* Write the rollback report of a restart that FAILED, so the interface can
|
|
3023
|
+
* say what happened.
|
|
3024
|
+
*
|
|
3025
|
+
* The report has the same shape as the one the detached watcher writes, so
|
|
3026
|
+
* there is exactly ONE place that reads it.
|
|
3027
|
+
*
|
|
3028
|
+
* @param {{ rolledBack: boolean, reason: string, backupDir?: string|null, restored?: string[] }} details
|
|
3029
|
+
* @returns {boolean} whether the report was written
|
|
3030
|
+
*/
|
|
3031
|
+
const writeRestartReport = (details) => {
|
|
3032
|
+
const file = path.join(stateDirOf(), RESTART_REPORT_NAME)
|
|
3033
|
+
try {
|
|
3034
|
+
fs.writeFileSync(file, JSON.stringify({
|
|
3035
|
+
ok: false,
|
|
3036
|
+
at: new Date().toISOString(),
|
|
3037
|
+
waitedMs: 0,
|
|
3038
|
+
rolledBack: details.rolledBack === true,
|
|
3039
|
+
rollbackError: details.rolledBack === true ? null : details.reason,
|
|
3040
|
+
reason: details.reason,
|
|
3041
|
+
backupDir: details.backupDir ?? null,
|
|
3042
|
+
restored: Array.isArray(details.restored) ? details.restored : [],
|
|
3043
|
+
// The plugin restores the files; it never starts a second host.
|
|
3044
|
+
appStarted: false,
|
|
3045
|
+
}, null, 2), 'utf8')
|
|
3046
|
+
return true
|
|
3047
|
+
} catch (error) {
|
|
3048
|
+
log.warn('could not write the rollback report:', String(error))
|
|
3049
|
+
return false
|
|
3050
|
+
}
|
|
3051
|
+
}
|
|
3052
|
+
|
|
3053
|
+
/**
|
|
3054
|
+
* Put the pre-restart snapshot back over the live profile.
|
|
3055
|
+
*
|
|
3056
|
+
* This is the in-process half of the rollback: it runs when the plugin
|
|
3057
|
+
* itself can see that the restart failed (the restart command did not
|
|
3058
|
+
* start, or the new instance never answered). The detached watcher does
|
|
3059
|
+
* the same thing when the plugin died with the host.
|
|
3060
|
+
*
|
|
3061
|
+
* @param {{ backup: any, profileDir: string|null, reason: string }} details
|
|
3062
|
+
* @returns {{ rolledBack: boolean, restored: string[], error: string|null }}
|
|
3063
|
+
*/
|
|
3064
|
+
const rollbackInProcess = ({ backup, profileDir, reason }) => {
|
|
3065
|
+
const restored = []
|
|
3066
|
+
if (!backup || backup.ok !== true || typeof backup.dir !== 'string' || backup.dir === '') {
|
|
3067
|
+
log.error('the restart failed and there is no snapshot to roll back to:', reason)
|
|
3068
|
+
writeRestartReport({ rolledBack: false, reason, backupDir: null, restored })
|
|
3069
|
+
return { rolledBack: false, restored, error: 'no snapshot to roll back to' }
|
|
3070
|
+
}
|
|
3071
|
+
if (typeof profileDir !== 'string' || profileDir === '') {
|
|
3072
|
+
log.error('the restart failed and the profile directory is unknown:', reason)
|
|
3073
|
+
writeRestartReport({ rolledBack: false, reason, backupDir: backup.dir, restored })
|
|
3074
|
+
return { rolledBack: false, restored, error: 'the profile directory is unknown' }
|
|
3075
|
+
}
|
|
3076
|
+
// THE LOCK: the detached watcher may be restoring the snapshot at the same
|
|
3077
|
+
// time. Exactly ONE of them copies the files — two copies at once would
|
|
3078
|
+
// write the same files (117 MB of plugin tree) at the same time.
|
|
3079
|
+
const lock = claimRollbackLock(stateDirOf())
|
|
3080
|
+
if (!lock.ok) {
|
|
3081
|
+
log.warn('another rollback is already running; this one stands down:', lock.error)
|
|
3082
|
+
return { rolledBack: false, restored, error: lock.error }
|
|
3083
|
+
}
|
|
3084
|
+
let result = null
|
|
3085
|
+
try {
|
|
3086
|
+
result = restoreBackup({ backupDir: backup.dir, profileDir })
|
|
3087
|
+
} catch (error) {
|
|
3088
|
+
result = { ok: false, restored: [], error: String(error) }
|
|
3089
|
+
} finally {
|
|
3090
|
+
releaseRollbackLock(stateDirOf())
|
|
3091
|
+
}
|
|
3092
|
+
if (result && result.ok === true) {
|
|
3093
|
+
log.warn('the pre-restart snapshot was restored:', {
|
|
3094
|
+
reason,
|
|
3095
|
+
restored: result.restored,
|
|
3096
|
+
backupDir: backup.dir,
|
|
3097
|
+
})
|
|
3098
|
+
writeRestartReport({ rolledBack: true, reason, backupDir: backup.dir, restored: result.restored })
|
|
3099
|
+
return { rolledBack: true, restored: result.restored, error: null }
|
|
3100
|
+
}
|
|
3101
|
+
const error = (result && result.error) || 'the restore failed'
|
|
3102
|
+
log.error('the pre-restart snapshot could not be restored:', reason, error)
|
|
3103
|
+
writeRestartReport({ rolledBack: false, reason, backupDir: backup.dir, restored })
|
|
3104
|
+
return { rolledBack: false, restored, error }
|
|
3105
|
+
}
|
|
3106
|
+
|
|
3107
|
+
/**
|
|
3108
|
+
* Arm the rollback watcher: the process that confirms the new instance and
|
|
3109
|
+
* puts the snapshot back when it never comes.
|
|
3110
|
+
*
|
|
3111
|
+
* The watcher is started OUTSIDE the host process, because the restart
|
|
3112
|
+
* command usually kills the host itself — in that case the plugin cannot
|
|
3113
|
+
* confirm anything any more, and the watcher is the only safety net left.
|
|
3114
|
+
*
|
|
3115
|
+
* @param {{ backup: any, profileDir: string|null, gateScriptPath?: string|null }} details
|
|
3116
|
+
* @returns {{ ok: boolean, method: string, error: string|null, scriptPath: string|null }}
|
|
3117
|
+
*/
|
|
3118
|
+
const armRollbackWatcher = ({ backup, profileDir, gateScriptPath = null }) => {
|
|
3119
|
+
if (!backup || backup.ok !== true || typeof backup.dir !== 'string' || backup.dir === '') {
|
|
3120
|
+
log.warn('no pre-restart snapshot: the automatic rollback is not armed')
|
|
3121
|
+
return { ok: false, method: 'none', error: 'no snapshot', scriptPath: null }
|
|
3122
|
+
}
|
|
3123
|
+
if (typeof profileDir !== 'string' || profileDir === '') {
|
|
3124
|
+
log.warn('the profile directory is unknown: the automatic rollback is not armed')
|
|
3125
|
+
return { ok: false, method: 'none', error: 'the profile directory is unknown', scriptPath: null }
|
|
3126
|
+
}
|
|
3127
|
+
const stateDir = stateDirOf()
|
|
3128
|
+
const scriptPath = path.join(stateDir, 'restart-' + state.generation + '-watch.ps1')
|
|
3129
|
+
const script = buildWatchScript({
|
|
3130
|
+
// The fallback health URL (the port of the OLD instance). It is only
|
|
3131
|
+
// used when the host publishes no instance record at all.
|
|
3132
|
+
healthUrl: restartHealthUrl() ?? '',
|
|
3133
|
+
healthPath: resolveHealthPath(config),
|
|
3134
|
+
// The instance record: the port of the NEW instance and its boot id.
|
|
3135
|
+
portFile: accessFilePath(stateDir),
|
|
3136
|
+
expectedBootId: bootId,
|
|
3137
|
+
outcomeFile: path.join(stateDir, RESTART_OUTCOME_FILE),
|
|
3138
|
+
backupDir: backup.dir,
|
|
3139
|
+
profileDir,
|
|
3140
|
+
reportPath: path.join(stateDir, ROLLBACK_REPORT_NAME),
|
|
3141
|
+
lockFile: path.join(stateDir, ROLLBACK_LOCK_NAME),
|
|
3142
|
+
timeoutMs: config.rollbackTimeoutMs,
|
|
3143
|
+
rollbackTimeoutMs: config.rollbackCopyTimeoutMs,
|
|
3144
|
+
since: new Date().toISOString(),
|
|
3145
|
+
selfCleanPaths: [scriptPath, gateScriptPath].filter(Boolean),
|
|
3146
|
+
})
|
|
3147
|
+
const written = writeTextFileAtomic(scriptPath, script)
|
|
3148
|
+
if (!written.ok) {
|
|
3149
|
+
log.error('could not write the rollback watcher:', written.error)
|
|
3150
|
+
return { ok: false, method: 'none', error: written.error, scriptPath }
|
|
3151
|
+
}
|
|
3152
|
+
const result = launchWatcherFn({
|
|
3153
|
+
watchScriptPath: scriptPath,
|
|
3154
|
+
platform: env.platform,
|
|
3155
|
+
})
|
|
3156
|
+
if (!result || result.ok !== true) {
|
|
3157
|
+
log.error('could not start the rollback watcher:', (result && result.error) || 'unknown error')
|
|
3158
|
+
return { ok: false, method: (result && result.method) || 'none', error: (result && result.error) || null, scriptPath }
|
|
3159
|
+
}
|
|
3160
|
+
log.warn('the rollback watcher is armed', {
|
|
3161
|
+
scriptPath,
|
|
3162
|
+
method: result.method,
|
|
3163
|
+
timeoutMs: config.rollbackTimeoutMs,
|
|
3164
|
+
})
|
|
3165
|
+
return { ok: true, method: result.method, error: null, scriptPath }
|
|
3166
|
+
}
|
|
3167
|
+
|
|
1940
3168
|
/** Execute the restart now; the detached helper survives our own death. */
|
|
1941
3169
|
const executeRestart = async () => {
|
|
1942
3170
|
stopPolling()
|
|
@@ -1996,8 +3224,70 @@ const titleOfSummary = (item) => {
|
|
|
1996
3224
|
log,
|
|
1997
3225
|
profile: resolveProfileDirDetailed(),
|
|
1998
3226
|
})
|
|
3227
|
+
// The snapshot is the rollback point of BOTH branches.
|
|
3228
|
+
const backup = snapshot && snapshot.backup ? snapshot.backup : null
|
|
3229
|
+
const profileDir = snapshot && snapshot.profileDir ? snapshot.profileDir : null
|
|
3230
|
+
|
|
3231
|
+
// THE DURABLE RESTART RECORD. The instance that is being replaced
|
|
3232
|
+
// writes the "pending" half BEFORE it runs the restart command: the
|
|
3233
|
+
// snapshot it took (directory, local time, duration, which older
|
|
3234
|
+
// snapshots were pruned) and the generation. The instance that replaces
|
|
3235
|
+
// it rewrites the record as `ok` / `failed` after it has checked itself
|
|
3236
|
+
// (see `verifyRestartBoot`). If the new instance never comes up, the
|
|
3237
|
+
// record stays `pending` — which is exactly what has to be visible.
|
|
3238
|
+
recordRestartVerdict({
|
|
3239
|
+
result: RESTART_RESULTS.PENDING,
|
|
3240
|
+
mode: useCustomCommand ? 'command' : 'app',
|
|
3241
|
+
generation: state.generation,
|
|
3242
|
+
requestedAt: state.restart && Number.isFinite(state.restart.requestedAt)
|
|
3243
|
+
? state.restart.requestedAt
|
|
3244
|
+
: null,
|
|
3245
|
+
startedAt: Date.now(),
|
|
3246
|
+
previousBootId: bootId,
|
|
3247
|
+
bootId: null,
|
|
3248
|
+
reason: state.restart && typeof state.restart.reason === 'string' ? state.restart.reason : null,
|
|
3249
|
+
snapshot: buildSnapshotRecord({
|
|
3250
|
+
requested: snapshot && snapshot.requested === true,
|
|
3251
|
+
backup,
|
|
3252
|
+
profileDir,
|
|
3253
|
+
durationMs: snapshot && Number.isFinite(snapshot.durationMs) ? snapshot.durationMs : null,
|
|
3254
|
+
error: snapshot && snapshot.error ? snapshot.error : null,
|
|
3255
|
+
}),
|
|
3256
|
+
checks: [],
|
|
3257
|
+
})
|
|
3258
|
+
// The marker of an EARLIER decision must not confuse this restart's
|
|
3259
|
+
// watcher: it would stand down before it even looked at the new
|
|
3260
|
+
// instance. It is removed BEFORE the watcher is armed.
|
|
3261
|
+
try {
|
|
3262
|
+
fs.rmSync(path.join(stateDir, RESTART_OUTCOME_FILE), { force: true })
|
|
3263
|
+
} catch {}
|
|
1999
3264
|
|
|
2000
3265
|
if (useCustomCommand) {
|
|
3266
|
+
// (0) THE C ROUTE. When the supervisor of this plugin started this
|
|
3267
|
+
// instance, the restart is asked of the SUPERVISOR (over its control
|
|
3268
|
+
// channel) — not of a command. The supervisor kills the host, starts
|
|
3269
|
+
// it again and puts the snapshot back if it does not come.
|
|
3270
|
+
const detected = detectSupervisor({ env: process.env, stateDir })
|
|
3271
|
+
if (detected.supervised) {
|
|
3272
|
+
if (backup && backup.ok === true) {
|
|
3273
|
+
writeRollbackRequest(stateDir, {
|
|
3274
|
+
backupDir: backup.dir,
|
|
3275
|
+
profileDir: profileDir ?? '',
|
|
3276
|
+
generation: state.generation,
|
|
3277
|
+
reason: 'plugin-restart',
|
|
3278
|
+
})
|
|
3279
|
+
}
|
|
3280
|
+
const asked = await requestSupervisorAction({ stateDir, action: 'restart' })
|
|
3281
|
+
if (asked && asked.ok === true) {
|
|
3282
|
+
log.warn('the restart was handed to the supervisor of this plugin')
|
|
3283
|
+
return
|
|
3284
|
+
}
|
|
3285
|
+
log.error(
|
|
3286
|
+
'the supervisor did not accept the restart request; falling back to the restart command:',
|
|
3287
|
+
asked ? asked.error : 'unknown error',
|
|
3288
|
+
)
|
|
3289
|
+
}
|
|
3290
|
+
|
|
2001
3291
|
const command = buildCustomCommand({
|
|
2002
3292
|
template: config.restartCommand,
|
|
2003
3293
|
pid: process.pid,
|
|
@@ -2005,9 +3295,95 @@ const titleOfSummary = (item) => {
|
|
|
2005
3295
|
appExePath: env.appExePath || '',
|
|
2006
3296
|
})
|
|
2007
3297
|
log.warn('running custom restart command:', command)
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
3298
|
+
|
|
3299
|
+
// (1) THE SAFETY NET. The restart command usually kills THIS process
|
|
3300
|
+
// (that is what restarting the host means), so the plugin cannot
|
|
3301
|
+
// confirm anything once the command runs. The watcher is started
|
|
3302
|
+
// BEFORE the command: it waits for the new instance outside the host
|
|
3303
|
+
// and puts the snapshot back when the new instance never comes.
|
|
3304
|
+
//
|
|
3305
|
+
// It is NOT armed when the supervisor of this plugin is in charge:
|
|
3306
|
+
// that one restores the snapshot itself, and two of them would write
|
|
3307
|
+
// the same files at the same time.
|
|
3308
|
+
if (!detected.supervised) armRollbackWatcher({ backup, profileDir })
|
|
3309
|
+
|
|
3310
|
+
try {
|
|
3311
|
+
// (2) THE COMMAND, with a real success judgement. The old code only
|
|
3312
|
+
// looked at "the spawn did not throw", so a command that does not
|
|
3313
|
+
// exist or that exits with an error was reported as a successful
|
|
3314
|
+
// restart.
|
|
3315
|
+
const outcome = await runCommandFn({
|
|
3316
|
+
command,
|
|
3317
|
+
timeoutMs: config.restartCommandTimeoutMs,
|
|
3318
|
+
})
|
|
3319
|
+
if (!outcome || outcome.ok !== true) {
|
|
3320
|
+
const reason = (outcome && outcome.error) || 'the restart command failed'
|
|
3321
|
+
log.error('the restart command failed; the pre-restart snapshot is restored:', reason)
|
|
3322
|
+
rollbackInProcess({ backup, profileDir, reason })
|
|
3323
|
+
writeRestartOutcome({ decided: true, ok: false, reason })
|
|
3324
|
+
recordRestartVerdict({
|
|
3325
|
+
result: RESTART_RESULTS.FAILED,
|
|
3326
|
+
verifiedAt: Date.now(),
|
|
3327
|
+
reason,
|
|
3328
|
+
checks: [{ name: 'command', ok: false, detail: reason }],
|
|
3329
|
+
})
|
|
3330
|
+
return
|
|
3331
|
+
}
|
|
3332
|
+
if (outcome.timedOut === true) {
|
|
3333
|
+
log.warn(
|
|
3334
|
+
`the restart command is still running after ${config.restartCommandTimeoutMs}ms; `
|
|
3335
|
+
+ 'the new instance is confirmed by the rollback watcher',
|
|
3336
|
+
{ command, timeoutMs: config.restartCommandTimeoutMs },
|
|
3337
|
+
)
|
|
3338
|
+
} else {
|
|
3339
|
+
log.warn('the restart command finished', { command, exitCode: outcome.exitCode ?? null })
|
|
3340
|
+
}
|
|
3341
|
+
|
|
3342
|
+
// (3) THE HEALTH CONFIRMATION — only possible when this process
|
|
3343
|
+
// survived the command. The new instance publishes its own port and
|
|
3344
|
+
// its own boot id; the confirmation waits for THAT instance and not
|
|
3345
|
+
// for the one that is being replaced.
|
|
3346
|
+
const confirmed = await probeInstanceFn({
|
|
3347
|
+
accessFile: accessFilePath(stateDir),
|
|
3348
|
+
expectedBootId: bootId,
|
|
3349
|
+
healthPath: resolveHealthPath(config),
|
|
3350
|
+
timeoutMs: config.rollbackTimeoutMs,
|
|
3351
|
+
})
|
|
3352
|
+
if (confirmed && confirmed.ok === true) {
|
|
3353
|
+
log.warn('the new instance answered its health route:', confirmed)
|
|
3354
|
+
writeRestartOutcome({ decided: true, ok: true, reason: 'the new instance answered' })
|
|
3355
|
+
recordRestartVerdict({
|
|
3356
|
+
result: RESTART_RESULTS.OK,
|
|
3357
|
+
verifiedAt: Date.now(),
|
|
3358
|
+
reason: 'the new instance answered',
|
|
3359
|
+
checks: [{ name: 'command', ok: true, detail: 'the new instance answered its health route' }],
|
|
3360
|
+
}, { skipIfFinalized: true })
|
|
3361
|
+
return
|
|
3362
|
+
}
|
|
3363
|
+
const reason = (confirmed && confirmed.reason) || 'the new instance never answered'
|
|
3364
|
+
log.error('the new instance did not come up; the pre-restart snapshot is restored:', reason)
|
|
3365
|
+
rollbackInProcess({ backup, profileDir, reason })
|
|
3366
|
+
writeRestartOutcome({ decided: true, ok: false, reason })
|
|
3367
|
+
recordRestartVerdict({
|
|
3368
|
+
result: RESTART_RESULTS.FAILED,
|
|
3369
|
+
verifiedAt: Date.now(),
|
|
3370
|
+
reason,
|
|
3371
|
+
checks: [{ name: 'command', ok: false, detail: reason }],
|
|
3372
|
+
})
|
|
3373
|
+
} catch (error) {
|
|
3374
|
+
// The command has already run: a retry would restart the host a
|
|
3375
|
+
// SECOND time. The snapshot is restored and the reason is reported.
|
|
3376
|
+
const reason = 'the hosted restart failed: ' + String(error)
|
|
3377
|
+
log.error(reason)
|
|
3378
|
+
rollbackInProcess({ backup, profileDir, reason })
|
|
3379
|
+
writeRestartOutcome({ decided: true, ok: false, reason })
|
|
3380
|
+
recordRestartVerdict({
|
|
3381
|
+
result: RESTART_RESULTS.FAILED,
|
|
3382
|
+
verifiedAt: Date.now(),
|
|
3383
|
+
reason,
|
|
3384
|
+
checks: [{ name: 'command', ok: false, detail: reason }],
|
|
3385
|
+
})
|
|
3386
|
+
}
|
|
2011
3387
|
} else {
|
|
2012
3388
|
const aborts = Number(state.restart.gateAborts) || 0
|
|
2013
3389
|
const url = aborts > 20 ? null : restartGateUrl()
|
|
@@ -2017,29 +3393,41 @@ const titleOfSummary = (item) => {
|
|
|
2017
3393
|
log.warn('the web server port is unknown; scheduling an UNGATED restart (the helper cannot re-check the idle gate)')
|
|
2018
3394
|
}
|
|
2019
3395
|
// The snapshot was already taken above (the shared entry).
|
|
2020
|
-
const backup = snapshot && snapshot.backup ? snapshot.backup : null
|
|
2021
|
-
const profileDir = snapshot && snapshot.profileDir ? snapshot.profileDir : null
|
|
2022
3396
|
const healthUrl = restartHealthUrl()
|
|
2023
|
-
const
|
|
3397
|
+
const gateScriptPath = url === null ? null : path.join(stateDir, 'restart-' + state.generation + '.cmd')
|
|
3398
|
+
// The watcher does NOT need the port of the old instance (it reads the
|
|
3399
|
+
// port of the NEW one from the instance record), so it is armed as
|
|
3400
|
+
// soon as there is a snapshot and a profile directory.
|
|
3401
|
+
const watchScriptPath = backup && backup.ok === true && profileDir !== null
|
|
2024
3402
|
? path.join(stateDir, 'restart-' + state.generation + '-watch.ps1')
|
|
2025
3403
|
: null
|
|
2026
3404
|
const watchScript = watchScriptPath === null
|
|
2027
3405
|
? null
|
|
2028
3406
|
: buildWatchScript({
|
|
3407
|
+
// The fallback URL of the OLD instance. The watcher uses the port
|
|
3408
|
+
// of the NEW instance (from the instance record) — the desktop
|
|
3409
|
+
// app listens on a random port, so the old one is worthless.
|
|
2029
3410
|
healthUrl,
|
|
3411
|
+
healthPath: resolveHealthPath(config),
|
|
3412
|
+
portFile: accessFilePath(stateDir),
|
|
3413
|
+
expectedBootId: bootId,
|
|
3414
|
+
outcomeFile: path.join(stateDir, RESTART_OUTCOME_FILE),
|
|
2030
3415
|
backupDir: backup.dir,
|
|
2031
3416
|
profileDir,
|
|
2032
3417
|
reportPath: path.join(stateDir, ROLLBACK_REPORT_NAME),
|
|
2033
|
-
|
|
3418
|
+
lockFile: path.join(stateDir, ROLLBACK_LOCK_NAME),
|
|
2034
3419
|
timeoutMs: config.rollbackTimeoutMs,
|
|
3420
|
+
rollbackTimeoutMs: config.rollbackCopyTimeoutMs,
|
|
2035
3421
|
// Only a crash log written after this point belongs to this restart.
|
|
2036
3422
|
since: new Date().toISOString(),
|
|
3423
|
+
// The watcher removes the helper scripts when it ends.
|
|
3424
|
+
selfCleanPaths: [watchScriptPath, gateScriptPath].filter(Boolean),
|
|
2037
3425
|
})
|
|
2038
3426
|
const result = scheduleDetachedFn({
|
|
2039
3427
|
appExePath: env.appExePath,
|
|
2040
3428
|
relaunchDelaySeconds: config.relaunchDelaySeconds,
|
|
2041
3429
|
gateUrl: url,
|
|
2042
|
-
scriptPath:
|
|
3430
|
+
scriptPath: gateScriptPath,
|
|
2043
3431
|
watchScriptPath,
|
|
2044
3432
|
watchScript,
|
|
2045
3433
|
})
|
|
@@ -2073,7 +3461,7 @@ const titleOfSummary = (item) => {
|
|
|
2073
3461
|
const decision = restartDecision(state, {
|
|
2074
3462
|
anyAgentRunning: busy.length > 0,
|
|
2075
3463
|
busyAgents: busy,
|
|
2076
|
-
lastActivityAt,
|
|
3464
|
+
lastActivityAt: quietSince(),
|
|
2077
3465
|
settleMs: effectiveSettleMs(),
|
|
2078
3466
|
now,
|
|
2079
3467
|
softMs: config.deferSoftMs,
|
|
@@ -2082,6 +3470,23 @@ const titleOfSummary = (item) => {
|
|
|
2082
3470
|
forceRestartWhenBusy: config.forceRestartWhenBusy,
|
|
2083
3471
|
})
|
|
2084
3472
|
if (decision.log) log.info(decision.log)
|
|
3473
|
+
// The diagnostic line of the pending restart: what the gate decided and
|
|
3474
|
+
// why. It is rate limited (once a minute) so the 3s poll does not flood
|
|
3475
|
+
// the log, and it is the first thing to look at when "the restart never
|
|
3476
|
+
// happens" is reported.
|
|
3477
|
+
if (now - lastDecisionLogAt >= LONG_WAIT_LOG_INTERVAL_MS) {
|
|
3478
|
+
lastDecisionLogAt = now
|
|
3479
|
+
log.info(
|
|
3480
|
+
`restart gate: action=${decision.action} reason=${decision.reason} busyCount=${busy.length} quietForMs=${Math.max(0, now - quietSince())} effectiveSettleMs=${effectiveSettleMs()}`,
|
|
3481
|
+
{
|
|
3482
|
+
action: decision.action,
|
|
3483
|
+
reason: decision.reason,
|
|
3484
|
+
busyCount: busy.length,
|
|
3485
|
+
quietForMs: Math.max(0, now - quietSince()),
|
|
3486
|
+
effectiveSettleMs: effectiveSettleMs(),
|
|
3487
|
+
},
|
|
3488
|
+
)
|
|
3489
|
+
}
|
|
2085
3490
|
switch (decision.action) {
|
|
2086
3491
|
case 'wait':
|
|
2087
3492
|
return
|
|
@@ -2092,6 +3497,28 @@ const titleOfSummary = (item) => {
|
|
|
2092
3497
|
}
|
|
2093
3498
|
return
|
|
2094
3499
|
}
|
|
3500
|
+
case 'long-wait': {
|
|
3501
|
+
// The hard deadline is over and a conversation still holds the
|
|
3502
|
+
// restart. The request is KEPT — the user asked for a restart, it must
|
|
3503
|
+
// never be dropped silently — and the wait becomes VISIBLE: a clear
|
|
3504
|
+
// log line (rate limited, so the 3s poll does not flood the log) and
|
|
3505
|
+
// the `longWait` block of the state payload.
|
|
3506
|
+
const requestedAt = Number.isFinite(state.restart.requestedAt) ? state.restart.requestedAt : now
|
|
3507
|
+
const waitedMs = Math.max(0, now - requestedAt)
|
|
3508
|
+
const lastAt = longWaitLog.requestedAt === requestedAt ? longWaitLog.at : 0
|
|
3509
|
+
if (now - lastAt >= LONG_WAIT_LOG_INTERVAL_MS) {
|
|
3510
|
+
longWaitLog = { at: now, requestedAt }
|
|
3511
|
+
log.warn(
|
|
3512
|
+
`the restart has been waiting for ${describeWait(waitedMs)}; ${describeBlockers(decision.blockedBy)} still holds it — cancel the restart or wait for that work to finish`,
|
|
3513
|
+
{
|
|
3514
|
+
waitedMs,
|
|
3515
|
+
blockedBy: decision.blockedBy,
|
|
3516
|
+
reason: decision.reason,
|
|
3517
|
+
},
|
|
3518
|
+
)
|
|
3519
|
+
}
|
|
3520
|
+
return
|
|
3521
|
+
}
|
|
2095
3522
|
case 'restart':
|
|
2096
3523
|
case 'force-restart': {
|
|
2097
3524
|
void executeRestart()
|
|
@@ -2284,10 +3711,114 @@ const titleOfSummary = (item) => {
|
|
|
2284
3711
|
waitedMs: Number.isFinite(report.waitedMs) ? report.waitedMs : null,
|
|
2285
3712
|
rolledBack: report.rolledBack === true,
|
|
2286
3713
|
backupDir: typeof report.backupDir === 'string' ? report.backupDir : null,
|
|
3714
|
+
// WHY the restart failed. Without it the interface can only say "it was
|
|
3715
|
+
// rolled back", which is the half of the story the user needs most.
|
|
3716
|
+
reason: typeof report.reason === 'string'
|
|
3717
|
+
? report.reason
|
|
3718
|
+
: (typeof report.rollbackError === 'string' ? report.rollbackError : ''),
|
|
2287
3719
|
crashLog: typeof report.crashLog === 'string' ? report.crashLog : null,
|
|
2288
3720
|
crashTail: tail.length > 4000 ? tail.slice(-4000) : tail,
|
|
2289
3721
|
}
|
|
2290
3722
|
},
|
|
3723
|
+
/**
|
|
3724
|
+
* The instance record of the running host: the port it listens on and the
|
|
3725
|
+
* access link (with the boot token) that still works.
|
|
3726
|
+
*
|
|
3727
|
+
* After a restart the OLD link answers 401 — the boot token is created per
|
|
3728
|
+
* process — so this is the link the user needs to get back into the
|
|
3729
|
+
* interface.
|
|
3730
|
+
*/
|
|
3731
|
+
get accessRecord() {
|
|
3732
|
+
const read = readAccessRecord(path.dirname(stateFile))
|
|
3733
|
+
return read && read.ok === true ? read.record : null
|
|
3734
|
+
},
|
|
3735
|
+
/**
|
|
3736
|
+
* What the plugin knows about the self-starting supervisor: who starts
|
|
3737
|
+
* this host (the supervisor of this plugin / the user's own way / nobody)
|
|
3738
|
+
* and whether the restart can actually work.
|
|
3739
|
+
*/
|
|
3740
|
+
supervisorStatus: supervisorStatusOf,
|
|
3741
|
+
/**
|
|
3742
|
+
* HOW this host was started (the startup matrix): the mode, the evidence
|
|
3743
|
+
* and what the plugin may do about it.
|
|
3744
|
+
*/
|
|
3745
|
+
startupMode: startupModeOf,
|
|
3746
|
+
/**
|
|
3747
|
+
* WHAT IS ON DISK RIGHT NOW: the backups of this host.
|
|
3748
|
+
*/
|
|
3749
|
+
backupsInfo,
|
|
3750
|
+
/**
|
|
3751
|
+
* THE DIAGNOSTICS of "report a problem" (the text the user copies).
|
|
3752
|
+
*/
|
|
3753
|
+
diagnostics: diagnosticsText,
|
|
3754
|
+
/**
|
|
3755
|
+
* WHICH MACHINE IS THIS? (Windows / WSL / Linux / macOS) — the same
|
|
3756
|
+
* answer the graceful shutdown uses.
|
|
3757
|
+
*/
|
|
3758
|
+
hostKind: hostKindOf,
|
|
3759
|
+
/**
|
|
3760
|
+
* Install the supervisor. `confirmed` must be true: the confirmation card
|
|
3761
|
+
* is what says "yes".
|
|
3762
|
+
*/
|
|
3763
|
+
installSupervisor: installSupervisorNow,
|
|
3764
|
+
/**
|
|
3765
|
+
* Remove the supervisor and give the machine back to the user.
|
|
3766
|
+
*/
|
|
3767
|
+
uninstallSupervisor: uninstallSupervisorNow,
|
|
3768
|
+
/**
|
|
3769
|
+
* Record the answer of the one-time question about the supervisor.
|
|
3770
|
+
*/
|
|
3771
|
+
recordSupervisorChoice: recordSupervisorChoice,
|
|
3772
|
+
/**
|
|
3773
|
+
* Derive the command that starts this host again (the dry run: nothing is
|
|
3774
|
+
* written).
|
|
3775
|
+
*/
|
|
3776
|
+
deriveStartCommand: deriveStartCommandNow,
|
|
3777
|
+
/**
|
|
3778
|
+
* Write the derived command into the profile configuration (only after
|
|
3779
|
+
* the user confirmed it).
|
|
3780
|
+
*/
|
|
3781
|
+
applyDerivedCommand: applyDerivedCommand,
|
|
3782
|
+
/**
|
|
3783
|
+
* Put the previous start command back (the undo).
|
|
3784
|
+
*/
|
|
3785
|
+
revertDerivedCommand: revertDerivedCommand,
|
|
3786
|
+
/**
|
|
3787
|
+
* The answer of the one-time question about the supervisor (null when it
|
|
3788
|
+
* was never answered).
|
|
3789
|
+
*/
|
|
3790
|
+
get onboardingChoice() {
|
|
3791
|
+
const record = state.onboarding
|
|
3792
|
+
if (!record || typeof record !== 'object') return null
|
|
3793
|
+
if (record.feature !== 'supervisor') return null
|
|
3794
|
+
return {
|
|
3795
|
+
feature: record.feature,
|
|
3796
|
+
version: record.version,
|
|
3797
|
+
choice: record.choice,
|
|
3798
|
+
answeredAt: record.answeredAt,
|
|
3799
|
+
}
|
|
3800
|
+
},
|
|
3801
|
+
/**
|
|
3802
|
+
* Whether the one-time question about the supervisor may be asked.
|
|
3803
|
+
*
|
|
3804
|
+
* ONLY for the HOSTED hosts: the desktop app brings itself back up
|
|
3805
|
+
* (`restartMode: app`), so asking a hosted user "shall I install a
|
|
3806
|
+
* supervisor?" would be noise there.
|
|
3807
|
+
*
|
|
3808
|
+
* And NEVER when somebody else is already in charge: a second supervisor
|
|
3809
|
+
* would fight over the port with the first one — that is the double start
|
|
3810
|
+
* the whole external-supervisor detection exists to prevent.
|
|
3811
|
+
*/
|
|
3812
|
+
get supervisorOffer() {
|
|
3813
|
+
if (env.isDesktopApp) return false
|
|
3814
|
+
const record = state.onboarding
|
|
3815
|
+
if (record && typeof record === 'object' && record.feature === 'supervisor') return false
|
|
3816
|
+
try {
|
|
3817
|
+
const status = supervisorStatusOf()
|
|
3818
|
+
if (status && status.mode === 'external') return false
|
|
3819
|
+
} catch {}
|
|
3820
|
+
return true
|
|
3821
|
+
},
|
|
2291
3822
|
env,
|
|
2292
3823
|
stateFile,
|
|
2293
3824
|
log,
|
|
@@ -2340,6 +3871,24 @@ const titleOfSummary = (item) => {
|
|
|
2340
3871
|
return { applied: result.applied, showSleepButton: showSleepButton === true }
|
|
2341
3872
|
},
|
|
2342
3873
|
toolErrors,
|
|
3874
|
+
/**
|
|
3875
|
+
* The routes of this plugin that the web server accepted, and the ones it
|
|
3876
|
+
* refused. The boot check reports them: a host that came up without its
|
|
3877
|
+
* routes is NOT a healthy host.
|
|
3878
|
+
*/
|
|
3879
|
+
registeredRoutes,
|
|
3880
|
+
routeErrors,
|
|
3881
|
+
/**
|
|
3882
|
+
* The outcome of the boot recovery pass of THIS boot:
|
|
3883
|
+
* `{ ran, ok, detail }`. `ran` is false when the recovery is switched off.
|
|
3884
|
+
*/
|
|
3885
|
+
recovery: null,
|
|
3886
|
+
/**
|
|
3887
|
+
* The durable record of the last restart (the snapshot that was taken and
|
|
3888
|
+
* the verdict of the instance that came up).
|
|
3889
|
+
*/
|
|
3890
|
+
lastRestartVerdict,
|
|
3891
|
+
recordRestartVerdict,
|
|
2343
3892
|
startPolling,
|
|
2344
3893
|
stopPolling,
|
|
2345
3894
|
tick,
|
|
@@ -2351,13 +3900,23 @@ const titleOfSummary = (item) => {
|
|
|
2351
3900
|
*/
|
|
2352
3901
|
pendingRestart,
|
|
2353
3902
|
/**
|
|
2354
|
-
* How long the host has been quiet (no conversation activity
|
|
3903
|
+
* How long the host has been quiet (no REAL conversation activity: a
|
|
3904
|
+
* running turn, a queued message, a turn that started or ended).
|
|
3905
|
+
*
|
|
3906
|
+
* The routine session events of the interface do not count.
|
|
2355
3907
|
*
|
|
2356
3908
|
* @returns {number}
|
|
2357
3909
|
*/
|
|
2358
3910
|
quietForMs,
|
|
2359
3911
|
/**
|
|
2360
|
-
* The
|
|
3912
|
+
* The moment the quiet window started.
|
|
3913
|
+
*
|
|
3914
|
+
* @returns {number} a timestamp in milliseconds
|
|
3915
|
+
*/
|
|
3916
|
+
quietSince,
|
|
3917
|
+
/**
|
|
3918
|
+
* The RESTART reserve: the extra wait after the last conversation
|
|
3919
|
+
* finished, before the host is really restarted.
|
|
2361
3920
|
*
|
|
2362
3921
|
* @returns {number} milliseconds
|
|
2363
3922
|
*/
|
|
@@ -2378,6 +3937,19 @@ const titleOfSummary = (item) => {
|
|
|
2378
3937
|
noteActivity,
|
|
2379
3938
|
busyAgents,
|
|
2380
3939
|
gate,
|
|
3940
|
+
/**
|
|
3941
|
+
* The description of a long wait (null when there is none).
|
|
3942
|
+
*
|
|
3943
|
+
* @param {number} [now]
|
|
3944
|
+
* @returns {{ since: number, reason: string, waitedMs: number, blockedBy: any[] }|null}
|
|
3945
|
+
*/
|
|
3946
|
+
longWaitInfo,
|
|
3947
|
+
/**
|
|
3948
|
+
* The upper bound of a queued (unclaimed) inbox message.
|
|
3949
|
+
*
|
|
3950
|
+
* @returns {number} milliseconds
|
|
3951
|
+
*/
|
|
3952
|
+
queuedGraceMs: () => config.queuedGraceMs,
|
|
2381
3953
|
sessionTitle,
|
|
2382
3954
|
disarmGoal,
|
|
2383
3955
|
disarmBlockingGoals,
|
|
@@ -2929,6 +4501,137 @@ async function recoverAfterRestart(ctx, runtime, config) {
|
|
|
2929
4501
|
return { recovered: results, phase: 'running' }
|
|
2930
4502
|
}
|
|
2931
4503
|
|
|
4504
|
+
/**
|
|
4505
|
+
* How long a pending restart record stays fresh enough to be finished by
|
|
4506
|
+
* THIS boot. A record that is older belongs to a restart of another day — it
|
|
4507
|
+
* stays `pending`, because that restart really was never confirmed.
|
|
4508
|
+
*/
|
|
4509
|
+
export const RESTART_VERDICT_FRESH_MS = 30 * 60 * 1000
|
|
4510
|
+
|
|
4511
|
+
/**
|
|
4512
|
+
* The four checks the instance that replaced the old one runs on itself.
|
|
4513
|
+
*
|
|
4514
|
+
* They answer the question the operator actually has after a restart: "did
|
|
4515
|
+
* the new host really come up?" — and each of them is a fact about THIS boot,
|
|
4516
|
+
* not about the old one:
|
|
4517
|
+
*
|
|
4518
|
+
* - `listener` — the host listens on a known port;
|
|
4519
|
+
* - `routes` — the routes of this plugin were accepted by the web server;
|
|
4520
|
+
* - `plugin` — the plugin itself is alive in this boot;
|
|
4521
|
+
* - `recovery`— the boot recovery pass finished (or was not needed).
|
|
4522
|
+
*
|
|
4523
|
+
* @param {object} runtime
|
|
4524
|
+
* @returns {Array<{ name: string, ok: boolean, detail: string }>}
|
|
4525
|
+
*/
|
|
4526
|
+
export function collectBootChecks(runtime) {
|
|
4527
|
+
const checks = []
|
|
4528
|
+
|
|
4529
|
+
const port = Number(runtime && runtime.webPort)
|
|
4530
|
+
checks.push({
|
|
4531
|
+
name: 'listener',
|
|
4532
|
+
ok: Number.isFinite(port) && port > 0,
|
|
4533
|
+
detail: Number.isFinite(port) && port > 0
|
|
4534
|
+
? 'the host listens on port ' + port
|
|
4535
|
+
: 'the host does not listen on a known port',
|
|
4536
|
+
})
|
|
4537
|
+
|
|
4538
|
+
const registered = Array.isArray(runtime && runtime.registeredRoutes) ? runtime.registeredRoutes : []
|
|
4539
|
+
const routeErrors = Array.isArray(runtime && runtime.routeErrors) ? runtime.routeErrors : []
|
|
4540
|
+
checks.push({
|
|
4541
|
+
name: 'routes',
|
|
4542
|
+
ok: registered.length > 0 && routeErrors.length === 0,
|
|
4543
|
+
detail: registered.length === 0
|
|
4544
|
+
? 'no route of this plugin was registered'
|
|
4545
|
+
: registered.length + ' route(s) registered'
|
|
4546
|
+
+ (routeErrors.length > 0 ? ', ' + routeErrors.length + ' refused' : ''),
|
|
4547
|
+
})
|
|
4548
|
+
|
|
4549
|
+
const bootId = runtime && typeof runtime.bootId === 'string' ? runtime.bootId : ''
|
|
4550
|
+
const toolErrors = Array.isArray(runtime && runtime.toolErrors) ? runtime.toolErrors : []
|
|
4551
|
+
checks.push({
|
|
4552
|
+
name: 'plugin',
|
|
4553
|
+
ok: bootId !== '' && toolErrors.length === 0,
|
|
4554
|
+
detail: bootId === ''
|
|
4555
|
+
? 'the plugin did not register a boot id'
|
|
4556
|
+
: 'the plugin is alive (boot ' + bootId + ')'
|
|
4557
|
+
+ (toolErrors.length > 0 ? ', ' + toolErrors.length + ' tool(s) failed' : ''),
|
|
4558
|
+
})
|
|
4559
|
+
|
|
4560
|
+
const recovery = runtime && runtime.recovery && typeof runtime.recovery === 'object'
|
|
4561
|
+
? runtime.recovery
|
|
4562
|
+
: null
|
|
4563
|
+
checks.push({
|
|
4564
|
+
name: 'recovery',
|
|
4565
|
+
ok: recovery === null ? false : recovery.ok === true,
|
|
4566
|
+
detail: recovery === null
|
|
4567
|
+
? 'the boot recovery did not report anything'
|
|
4568
|
+
: (typeof recovery.detail === 'string' && recovery.detail !== '' ? recovery.detail : 'the boot recovery finished'),
|
|
4569
|
+
})
|
|
4570
|
+
|
|
4571
|
+
return checks
|
|
4572
|
+
}
|
|
4573
|
+
|
|
4574
|
+
/**
|
|
4575
|
+
* Finish the durable restart record.
|
|
4576
|
+
*
|
|
4577
|
+
* The instance that replaced the old one reads the `pending` record, runs the
|
|
4578
|
+
* four boot checks and writes the final verdict (`ok` / `failed`) with the
|
|
4579
|
+
* time it verified itself and how long the whole restart took.
|
|
4580
|
+
*
|
|
4581
|
+
* Nothing is written when there is no pending record (this boot was not the
|
|
4582
|
+
* result of a restart) or when the pending record belongs to another day.
|
|
4583
|
+
*
|
|
4584
|
+
* @param {object} runtime
|
|
4585
|
+
* @param {{ timeoutMs?: number, now?: number, sleep?: Function }} [options]
|
|
4586
|
+
* @returns {Promise<object|null>} the record that is on disk now
|
|
4587
|
+
*/
|
|
4588
|
+
export async function verifyRestartBoot(runtime, options = {}) {
|
|
4589
|
+
const log = runtime.log
|
|
4590
|
+
const stateDir = path.dirname(runtime.stateFile)
|
|
4591
|
+
const pending = readRestartVerdict(stateDir)
|
|
4592
|
+
if (pending === null) return null
|
|
4593
|
+
if (pending.result !== RESTART_RESULTS.PENDING) return pending
|
|
4594
|
+
const now = typeof options.now === 'function' ? options.now : () => Date.now()
|
|
4595
|
+
const timeoutMs = Number.isFinite(Number(options.timeoutMs)) && Number(options.timeoutMs) > 0
|
|
4596
|
+
? Math.trunc(Number(options.timeoutMs))
|
|
4597
|
+
: 15000
|
|
4598
|
+
const sleep = typeof options.sleep === 'function' ? options.sleep : recoveryDelay
|
|
4599
|
+
const started = now()
|
|
4600
|
+
// The web server may still be coming up: the checks need its port and its
|
|
4601
|
+
// routes, so they wait — but only for a bounded while.
|
|
4602
|
+
while (now() - started < timeoutMs) {
|
|
4603
|
+
const port = Number(runtime.webPort)
|
|
4604
|
+
const registered = Array.isArray(runtime.registeredRoutes) ? runtime.registeredRoutes.length : 0
|
|
4605
|
+
if (Number.isFinite(port) && port > 0 && registered > 0) break
|
|
4606
|
+
await sleep(250)
|
|
4607
|
+
}
|
|
4608
|
+
const checks = collectBootChecks(runtime)
|
|
4609
|
+
const evaluated = evaluateRestartChecks(checks)
|
|
4610
|
+
const verifiedAt = now()
|
|
4611
|
+
const record = runtime.recordRestartVerdict({
|
|
4612
|
+
result: evaluated.ok ? RESTART_RESULTS.OK : RESTART_RESULTS.FAILED,
|
|
4613
|
+
verifiedAt,
|
|
4614
|
+
durationMs: pending.startedAt !== null ? Math.max(0, verifiedAt - pending.startedAt) : null,
|
|
4615
|
+
bootId: runtime.bootId,
|
|
4616
|
+
checks,
|
|
4617
|
+
reason: evaluated.ok
|
|
4618
|
+
? 'the new instance came up and every check passed'
|
|
4619
|
+
: 'the new instance came up, but ' + evaluated.failed.join(', ') + ' failed',
|
|
4620
|
+
})
|
|
4621
|
+
if (record !== null) {
|
|
4622
|
+
if (evaluated.ok) {
|
|
4623
|
+
log.warn('the new instance verified itself:', { durationMs: record.durationMs, checks })
|
|
4624
|
+
} else {
|
|
4625
|
+
log.error('the new instance did NOT come up cleanly:', {
|
|
4626
|
+
durationMs: record.durationMs,
|
|
4627
|
+
failed: evaluated.failed,
|
|
4628
|
+
checks,
|
|
4629
|
+
})
|
|
4630
|
+
}
|
|
4631
|
+
}
|
|
4632
|
+
return record
|
|
4633
|
+
}
|
|
4634
|
+
|
|
2932
4635
|
/** Promise-based pause used by boot recovery (batching + resolveAgent retries). */
|
|
2933
4636
|
function recoveryDelay(ms) {
|
|
2934
4637
|
const wait = Number.isFinite(ms) && ms > 0 ? ms : 0
|
|
@@ -3107,6 +4810,7 @@ function registerTools(ctx, runtime, config) {
|
|
|
3107
4810
|
quietForMs: { type: 'number' },
|
|
3108
4811
|
graceMs: { type: 'number' },
|
|
3109
4812
|
effectiveSettleMs: { type: 'number' },
|
|
4813
|
+
queuedGraceMs: { type: 'number' },
|
|
3110
4814
|
blockedBy: {
|
|
3111
4815
|
type: 'array',
|
|
3112
4816
|
items: {
|
|
@@ -3139,6 +4843,11 @@ function registerTools(ctx, runtime, config) {
|
|
|
3139
4843
|
profileDirSource: { type: 'string' },
|
|
3140
4844
|
backupIncludePlugins: { type: 'boolean' },
|
|
3141
4845
|
lastRollback: { type: 'object' },
|
|
4846
|
+
// The last restart: the snapshot that was taken before it and the
|
|
4847
|
+
// verdict of the instance that came up.
|
|
4848
|
+
lastRestart: { type: 'object' },
|
|
4849
|
+
// The access link of the running instance (with the boot token).
|
|
4850
|
+
accessUrl: { type: 'string' },
|
|
3142
4851
|
environment: { type: 'object' },
|
|
3143
4852
|
sessions: { type: 'array', items: { type: 'object' } },
|
|
3144
4853
|
},
|
|
@@ -3158,10 +4867,12 @@ function registerTools(ctx, runtime, config) {
|
|
|
3158
4867
|
pendingRestart: state.restart !== null,
|
|
3159
4868
|
gateAllow: verdict.allow,
|
|
3160
4869
|
quietForMs: Math.round(verdict.quietForMs),
|
|
3161
|
-
// The
|
|
4870
|
+
// The RESTART reserve (the extra wait after the last conversation
|
|
3162
4871
|
// finished) and the quiet window the gate really uses.
|
|
3163
4872
|
graceMs: runtime.graceMs(),
|
|
3164
4873
|
effectiveSettleMs: runtime.effectiveSettleMs(),
|
|
4874
|
+
// The upper bound of a queued (unclaimed) inbox message.
|
|
4875
|
+
queuedGraceMs: runtime.queuedGraceMs(),
|
|
3165
4876
|
blockedBy: blockingDetails(verdict.blockedBy),
|
|
3166
4877
|
parkedGoals: parkedGoalDetails(),
|
|
3167
4878
|
recovery: recoveryList(state),
|
|
@@ -3173,6 +4884,15 @@ function registerTools(ctx, runtime, config) {
|
|
|
3173
4884
|
backupIncludePlugins: config.backupIncludePlugins !== false,
|
|
3174
4885
|
// Present only when the previous restart had to be rolled back.
|
|
3175
4886
|
...(runtime.lastRollback ? { lastRollback: runtime.lastRollback } : {}),
|
|
4887
|
+
// THE LAST RESTART: the snapshot that was taken before it (directory,
|
|
4888
|
+
// LOCAL time, duration, pruned) and the verdict of the instance that
|
|
4889
|
+
// came up (the four boot checks). Present only when a restart
|
|
4890
|
+
// happened — otherwise there is nothing to report.
|
|
4891
|
+
...(runtime.lastRestartVerdict() ? { lastRestart: runtime.lastRestartVerdict() } : {}),
|
|
4892
|
+
// The access link of THIS instance (with the boot token). The boot
|
|
4893
|
+
// token is created per process, so the link the browser holds answers
|
|
4894
|
+
// 401 after a restart — this is the link that still works.
|
|
4895
|
+
...(accessUrlOf(runtime) ? { accessUrl: accessUrlOf(runtime) } : {}),
|
|
3176
4896
|
environment: {
|
|
3177
4897
|
platform: runtime.env.platform,
|
|
3178
4898
|
isDesktopApp: runtime.env.isDesktopApp,
|
|
@@ -3348,11 +5068,19 @@ function registerTools(ctx, runtime, config) {
|
|
|
3348
5068
|
|
|
3349
5069
|
tool({
|
|
3350
5070
|
name: 'continuum_cancel_restart',
|
|
3351
|
-
description:
|
|
5071
|
+
description:
|
|
5072
|
+
'Cancel a restart requested through continuum_restart. By default ONLY the calling session\'s own request is '
|
|
5073
|
+
+ 'withdrawn (scope "mine"): the other sessions that asked for the same restart keep it. Use scope "all" to '
|
|
5074
|
+
+ 'cancel the request for EVERYONE ("nobody restarts any more"). Does nothing when no restart is pending.',
|
|
3352
5075
|
parameters: {
|
|
3353
5076
|
type: 'object',
|
|
3354
5077
|
additionalProperties: false,
|
|
3355
5078
|
properties: {
|
|
5079
|
+
scope: {
|
|
5080
|
+
type: 'string',
|
|
5081
|
+
enum: ['mine', 'all'],
|
|
5082
|
+
description: 'Whose request to cancel: "mine" (the default) withdraws only the calling session\'s own request and leaves the requests of the other sessions alone; "all" cancels the request for every session.',
|
|
5083
|
+
},
|
|
3356
5084
|
reason: { type: 'string', description: 'Optional note recorded in the log.' },
|
|
3357
5085
|
},
|
|
3358
5086
|
},
|
|
@@ -3364,20 +5092,248 @@ function registerTools(ctx, runtime, config) {
|
|
|
3364
5092
|
properties: {
|
|
3365
5093
|
ok: { type: 'boolean' },
|
|
3366
5094
|
cancelled: { type: 'boolean' },
|
|
5095
|
+
scope: { type: 'string' },
|
|
5096
|
+
remainingRequesters: { type: 'integer' },
|
|
5097
|
+
sessionId: { type: 'string' },
|
|
3367
5098
|
phase: { type: 'string' },
|
|
5099
|
+
error: { type: 'string' },
|
|
3368
5100
|
},
|
|
3369
5101
|
},
|
|
3370
5102
|
(_args, value) => [{ type: 'text', text: JSON.stringify(value) }],
|
|
3371
5103
|
),
|
|
3372
|
-
execute(args = {}) {
|
|
5104
|
+
execute(args = {}, exec) {
|
|
3373
5105
|
const state = runtime.state
|
|
3374
|
-
const
|
|
3375
|
-
|
|
3376
|
-
|
|
3377
|
-
|
|
5106
|
+
const now = Date.now()
|
|
5107
|
+
const scope = args.scope === 'all' ? 'all' : 'mine'
|
|
5108
|
+
const agent = exec && exec.agent
|
|
5109
|
+
const sessionId = agent && agent.id ? String(agent.id) : ''
|
|
5110
|
+
const reason = typeof args.reason === 'string' && args.reason.trim() !== '' ? args.reason.trim() : ''
|
|
5111
|
+
|
|
5112
|
+
// How many sessions are waiting for the pending restart.
|
|
5113
|
+
const requesterCount = () => {
|
|
5114
|
+
const restart = state.restart
|
|
5115
|
+
if (!restart) return 0
|
|
5116
|
+
const ids = new Set()
|
|
5117
|
+
if (Array.isArray(restart.requesters)) {
|
|
5118
|
+
for (const requester of restart.requesters) {
|
|
5119
|
+
if (requester && typeof requester.sessionId === 'string' && requester.sessionId) ids.add(requester.sessionId)
|
|
5120
|
+
}
|
|
5121
|
+
}
|
|
5122
|
+
if (typeof restart.bySessionId === 'string' && restart.bySessionId) ids.add(restart.bySessionId)
|
|
5123
|
+
return ids.size
|
|
5124
|
+
}
|
|
5125
|
+
|
|
5126
|
+
if (scope === 'all') {
|
|
5127
|
+
// The explicit "nobody restarts any more": the whole request goes.
|
|
5128
|
+
const hadPending = state.restart !== null
|
|
5129
|
+
runtime.releaseRequesters()
|
|
5130
|
+
cancelRestart(state, { now })
|
|
5131
|
+
runtime.stopPolling()
|
|
5132
|
+
runtime.persist()
|
|
5133
|
+
if (hadPending) log.info('pending restart cancelled for EVERYONE:', reason || '(no reason)')
|
|
5134
|
+
return { ok: true, cancelled: hadPending, scope: 'all', remainingRequesters: 0, phase: state.phase }
|
|
5135
|
+
}
|
|
5136
|
+
|
|
5137
|
+
// scope "mine": withdraw ONLY the calling session's request.
|
|
5138
|
+
if (state.restart === null) {
|
|
5139
|
+
// Nothing is pending: the tool does nothing, as before.
|
|
5140
|
+
return { ok: true, cancelled: false, scope: 'mine', remainingRequesters: 0, phase: state.phase }
|
|
5141
|
+
}
|
|
5142
|
+
if (sessionId === '') {
|
|
5143
|
+
// The calling session is unknown: refuse instead of guessing. A
|
|
5144
|
+
// silent "cancel everything" is exactly the accident this scope
|
|
5145
|
+
// exists to prevent.
|
|
5146
|
+
log.warn('the restart request of the calling session could not be withdrawn: the calling session is unknown; nothing was cancelled')
|
|
5147
|
+
return {
|
|
5148
|
+
ok: false,
|
|
5149
|
+
cancelled: false,
|
|
5150
|
+
scope: 'mine',
|
|
5151
|
+
remainingRequesters: requesterCount(),
|
|
5152
|
+
phase: state.phase,
|
|
5153
|
+
error: 'The calling session is unknown, so its own request cannot be identified. Nothing was cancelled. Call this tool from the session that requested the restart, or pass scope "all" to cancel the request for everyone.',
|
|
5154
|
+
}
|
|
5155
|
+
}
|
|
5156
|
+
const outcome = cancelOwnRestart(state, { sessionId, now })
|
|
5157
|
+
if (!outcome.removed) {
|
|
5158
|
+
log.warn(`session ${sessionId} did not request the pending restart; nothing was cancelled`)
|
|
5159
|
+
return {
|
|
5160
|
+
ok: false,
|
|
5161
|
+
cancelled: false,
|
|
5162
|
+
scope: 'mine',
|
|
5163
|
+
remainingRequesters: outcome.remaining,
|
|
5164
|
+
sessionId,
|
|
5165
|
+
phase: state.phase,
|
|
5166
|
+
error: 'This session did not request the pending restart, so nothing was cancelled. Pass scope "all" to cancel the request for every session.',
|
|
5167
|
+
}
|
|
5168
|
+
}
|
|
5169
|
+
// Release ONLY this session's continuation request: the other sessions
|
|
5170
|
+
// stay pinned for the restart they asked for.
|
|
5171
|
+
runtime.releaseRequesters([sessionId])
|
|
5172
|
+
if (outcome.cancelled) {
|
|
5173
|
+
// The last requester left: the restart is really over.
|
|
5174
|
+
runtime.stopPolling()
|
|
5175
|
+
log.info(`session ${sessionId} withdrew the last restart request; the pending restart is cancelled`, reason || '')
|
|
5176
|
+
} else {
|
|
5177
|
+
log.info(`session ${sessionId} withdrew its restart request; ${outcome.remaining} requester(s) remain`)
|
|
5178
|
+
}
|
|
3378
5179
|
runtime.persist()
|
|
3379
|
-
|
|
3380
|
-
|
|
5180
|
+
return {
|
|
5181
|
+
ok: true,
|
|
5182
|
+
cancelled: outcome.cancelled,
|
|
5183
|
+
scope: 'mine',
|
|
5184
|
+
remainingRequesters: outcome.remaining,
|
|
5185
|
+
sessionId,
|
|
5186
|
+
phase: state.phase,
|
|
5187
|
+
}
|
|
5188
|
+
},
|
|
5189
|
+
})
|
|
5190
|
+
|
|
5191
|
+
tool({
|
|
5192
|
+
name: 'continuum_supervisor',
|
|
5193
|
+
description:
|
|
5194
|
+
'Inspect and manage the self-starting supervisor of this plugin (the process that brings the host back when it '
|
|
5195
|
+
+ 'dies). action "status" reports who starts this host (the supervisor of this plugin / your own way / nobody) '
|
|
5196
|
+
+ 'and whether the restart can work at all; action "plan" reports what an installation WOULD change without '
|
|
5197
|
+
+ 'touching anything; action "install" and action "uninstall" need confirmed: true, because they really change '
|
|
5198
|
+
+ 'the machine (a scheduled task or a systemd user unit that starts the supervisor at login).',
|
|
5199
|
+
parameters: {
|
|
5200
|
+
type: 'object',
|
|
5201
|
+
additionalProperties: false,
|
|
5202
|
+
properties: {
|
|
5203
|
+
action: {
|
|
5204
|
+
type: 'string',
|
|
5205
|
+
enum: ['status', 'plan', 'dry-run', 'install', 'uninstall', 'apply', 'revert'],
|
|
5206
|
+
description: 'What to do: "status" (read-only), "plan" (what the installation would change), "dry-run" (the command this host would be restarted with, and why it may not be derivable), "install"/"uninstall" (the real thing, they need confirmed: true), "apply" (write the derived restart command after the user confirmed the dry run) and "revert" (put the previous command back).',
|
|
5207
|
+
},
|
|
5208
|
+
confirmed: {
|
|
5209
|
+
type: 'boolean',
|
|
5210
|
+
description: 'Required for "install" and "uninstall": the user confirmed the card that says what would change.',
|
|
5211
|
+
},
|
|
5212
|
+
choice: {
|
|
5213
|
+
type: 'string',
|
|
5214
|
+
enum: ['install', 'external', 'dismissed'],
|
|
5215
|
+
description: 'The answer of the one-time question about the supervisor (recorded so the question is never asked twice).',
|
|
5216
|
+
},
|
|
5217
|
+
},
|
|
5218
|
+
},
|
|
5219
|
+
output: textOutput(
|
|
5220
|
+
{
|
|
5221
|
+
type: 'object',
|
|
5222
|
+
additionalProperties: false,
|
|
5223
|
+
required: ['ok', 'action'],
|
|
5224
|
+
properties: {
|
|
5225
|
+
ok: { type: 'boolean' },
|
|
5226
|
+
action: { type: 'string' },
|
|
5227
|
+
kind: { type: 'string' },
|
|
5228
|
+
degraded: { type: 'boolean' },
|
|
5229
|
+
reason: { type: 'string' },
|
|
5230
|
+
error: { type: 'string' },
|
|
5231
|
+
supervisor: { type: 'object' },
|
|
5232
|
+
steps: { type: 'array', items: { type: 'object' } },
|
|
5233
|
+
previous: { type: 'object' },
|
|
5234
|
+
pending: { type: 'object' },
|
|
5235
|
+
},
|
|
5236
|
+
},
|
|
5237
|
+
(_args, value) => [{ type: 'text', text: JSON.stringify(value, null, 2) }],
|
|
5238
|
+
),
|
|
5239
|
+
async execute(args = {}) {
|
|
5240
|
+
const action = typeof args.action === 'string' ? args.action.trim() : 'status'
|
|
5241
|
+
const supervisor = runtime.supervisorStatus()
|
|
5242
|
+
if (action === 'status') {
|
|
5243
|
+
return { ok: true, action, supervisor }
|
|
5244
|
+
}
|
|
5245
|
+
if (action === 'dry-run' || action === 'derive') {
|
|
5246
|
+
// B: the dry run. It reports what WOULD be written; it writes nothing.
|
|
5247
|
+
const plan = await runtime.deriveStartCommand()
|
|
5248
|
+
return {
|
|
5249
|
+
ok: plan.ok === true,
|
|
5250
|
+
action,
|
|
5251
|
+
level: plan.level,
|
|
5252
|
+
mode: plan.mode,
|
|
5253
|
+
reason: plan.reason,
|
|
5254
|
+
command: plan.command,
|
|
5255
|
+
plan: plan.plan,
|
|
5256
|
+
needsConfirmation: plan.needsConfirmation !== false,
|
|
5257
|
+
writes: plan.writes,
|
|
5258
|
+
compose: plan.compose,
|
|
5259
|
+
supervisor,
|
|
5260
|
+
}
|
|
5261
|
+
}
|
|
5262
|
+
if (action === 'plan') {
|
|
5263
|
+
const plan = planSupervisorInstall({
|
|
5264
|
+
platform: runtime.env.platform,
|
|
5265
|
+
// The desktop app manages itself: the plan is refused there, and no
|
|
5266
|
+
// step is generated that could be run by accident.
|
|
5267
|
+
isDesktopApp: runtime.env.isDesktopApp === true,
|
|
5268
|
+
// The startup matrix: only a `bare` host may get a supervisor.
|
|
5269
|
+
startup: runtime.startupMode(),
|
|
5270
|
+
// WHICH MACHINE IS THIS? (WSL is not a Linux.)
|
|
5271
|
+
host: runtime.hostKind(),
|
|
5272
|
+
nodePath: process.execPath,
|
|
5273
|
+
execPath: process.execPath,
|
|
5274
|
+
supervisorPath: supervisorPaths(path.dirname(runtime.stateFile)).script,
|
|
5275
|
+
configPath: supervisorPaths(path.dirname(runtime.stateFile)).config,
|
|
5276
|
+
kind: config.supervisorKind,
|
|
5277
|
+
})
|
|
5278
|
+
return {
|
|
5279
|
+
ok: plan.error ? false : true,
|
|
5280
|
+
action,
|
|
5281
|
+
kind: plan.kind,
|
|
5282
|
+
reason: plan.reason,
|
|
5283
|
+
error: plan.error ?? undefined,
|
|
5284
|
+
steps: plan.steps,
|
|
5285
|
+
supervisor,
|
|
5286
|
+
}
|
|
5287
|
+
}
|
|
5288
|
+
if (action === 'install' && runtime.env.isDesktopApp === true) {
|
|
5289
|
+
// The desktop app manages itself: a login record would open a SECOND
|
|
5290
|
+
// client at every login. The refusal comes BEFORE the confirmation, so
|
|
5291
|
+
// the user is never asked to confirm something that cannot happen.
|
|
5292
|
+
return {
|
|
5293
|
+
ok: false,
|
|
5294
|
+
action,
|
|
5295
|
+
error: 'desktop-manages-itself',
|
|
5296
|
+
reason: 'the desktop app relaunches itself; a supervisor would start a second copy',
|
|
5297
|
+
steps: [],
|
|
5298
|
+
supervisor,
|
|
5299
|
+
}
|
|
5300
|
+
}
|
|
5301
|
+
if (args.confirmed !== true) {
|
|
5302
|
+
// The card is what says "yes": without it the machine is not touched.
|
|
5303
|
+
return {
|
|
5304
|
+
ok: false,
|
|
5305
|
+
action,
|
|
5306
|
+
error: 'confirmation-required',
|
|
5307
|
+
reason: 'the installation and the uninstallation change the machine, so the user has to confirm the card first',
|
|
5308
|
+
supervisor,
|
|
5309
|
+
}
|
|
5310
|
+
}
|
|
5311
|
+
if (action === 'install') {
|
|
5312
|
+
const choice = typeof args.choice === 'string' ? args.choice : ''
|
|
5313
|
+
if (choice !== '') runtime.recordSupervisorChoice(choice)
|
|
5314
|
+
const result = runtime.installSupervisor({ confirmed: true })
|
|
5315
|
+
return {
|
|
5316
|
+
ok: result.ok === true,
|
|
5317
|
+
action,
|
|
5318
|
+
kind: result.kind,
|
|
5319
|
+
degraded: result.degraded,
|
|
5320
|
+
reason: result.reason,
|
|
5321
|
+
error: result.error ?? undefined,
|
|
5322
|
+
steps: result.steps,
|
|
5323
|
+
supervisor: runtime.supervisorStatus(),
|
|
5324
|
+
}
|
|
5325
|
+
}
|
|
5326
|
+
const result = await runtime.uninstallSupervisor()
|
|
5327
|
+
return {
|
|
5328
|
+
ok: result.ok === true,
|
|
5329
|
+
action,
|
|
5330
|
+
reason: result.ok === true ? 'the supervisor was removed' : undefined,
|
|
5331
|
+
error: result.error ?? undefined,
|
|
5332
|
+
steps: result.steps,
|
|
5333
|
+
previous: result.previous ?? undefined,
|
|
5334
|
+
pending: result.pending ?? undefined,
|
|
5335
|
+
supervisor: runtime.supervisorStatus(),
|
|
5336
|
+
}
|
|
3381
5337
|
},
|
|
3382
5338
|
})
|
|
3383
5339
|
}
|
|
@@ -3417,10 +5373,10 @@ function injectPageHeartbeat(html, bootId, routePath) {
|
|
|
3417
5373
|
const AUTORESTART_PROMPT = [
|
|
3418
5374
|
'## Restart policy (dsh-autorestart)',
|
|
3419
5375
|
'- When DeepSeek Harness itself must restart (for example to load an installed or updated plugin bundle), do NOT kill or relaunch the app yourself with pwsh/taskkill/Stop-Process/npm scripts. A hard guard blocks such commands anyway (denials mention `blockManualRestart`).',
|
|
3420
|
-
'- Call the `continuum_restart` tool instead (optional arguments: `reason`, `resume`, `defer_seconds`, `parkGoals`). It waits until every conversation is idle (no running turn, no
|
|
5376
|
+
'- Call the `continuum_restart` tool instead (optional arguments: `reason`, `resume`, `defer_seconds`, `parkGoals`). It waits until every conversation is idle (no running turn — a running turn ALWAYS blocks; an inbox message nobody claimed blocks only until `queuedGraceMs`, default 60000 ms, after which it is treated as stuck and no longer blocks) AND the host has been quiet for `settleMs` (default 2000 ms); the detached helper re-checks that gate immediately before killing, so other conversations are never killed mid-turn. It then relaunches the app, and after boot this plugin resumes the session and continues the work automatically.',
|
|
3421
5377
|
'- A goal-driven session is DISARMED automatically before the restart (its current round still finishes, nothing is lost), because such a session would otherwise keep the restart waiting forever: disarming only removes the process-local continuation authority, so no new round starts and the session goes idle. After the restart the goal is re-armed and continues. A goal whose round budget is spent cannot be continued and is reported instead. You can disarm your own goal on demand with `continuum_park_goal`, and retry a stuck recovery with `continuum_recover_now`.',
|
|
3422
5378
|
'- Persist anything the next turn needs before restarting (workspace files, `todo_write`, goal state); do not rely on in-memory state.',
|
|
3423
|
-
'- Inspect the lifecycle with `continuum_status`; cancel a pending restart with `continuum_cancel_restart`.',
|
|
5379
|
+
'- Inspect the lifecycle with `continuum_status`; cancel a pending restart with `continuum_cancel_restart`. Its default scope is `mine`: it withdraws ONLY the calling session\'s own request and leaves the requests of the other sessions alone (a restart request is coalesced, so several sessions can wait for the same restart). Use `scope: "all"` to cancel the request for EVERYONE.',
|
|
3424
5380
|
'- A session whose context exceeds the model window can be handed off to a fresh session with `continuum_handoff` (it carries the goal objective, the recorded reasons and a bounded tail of the conversation); this plugin refuses to auto-resume such a session, so hand it off instead of retrying it.',
|
|
3425
5381
|
'- For multi-turn objectives prefer a durable goal (`create_goal`): after a restart the plugin re-arms it and the goal round driver keeps producing rounds. No goal is required for restart continuation — without one the plugin injects a single continuation message that continues the work in a fresh turn.',
|
|
3426
5382
|
].join('\n')
|
|
@@ -3753,6 +5709,124 @@ function buildParkRoute(ctx, runtime, config) {
|
|
|
3753
5709
|
}
|
|
3754
5710
|
}
|
|
3755
5711
|
|
|
5712
|
+
/**
|
|
5713
|
+
* Build the POST `/cancel` route: withdraw ONE session's own restart request.
|
|
5714
|
+
*
|
|
5715
|
+
* It is the sibling of the `/park` route next to the health route and uses the
|
|
5716
|
+
* same authentication rule (loopback OR a request the connection service
|
|
5717
|
+
* admits). Body: `{ "sessionId": string, "scope"?: "mine" }`.
|
|
5718
|
+
*
|
|
5719
|
+
* The route NEVER cancels the request of everybody: `scope: "all"` is
|
|
5720
|
+
* refused, that operation belongs to the tool (where the caller says it out
|
|
5721
|
+
* loud). The answer reports what happened: `{ ok, cancelled, reason,
|
|
5722
|
+
* remainingRequesters, sessionId }`.
|
|
5723
|
+
*
|
|
5724
|
+
* @param {object} ctx
|
|
5725
|
+
* @param {object} runtime
|
|
5726
|
+
* @param {object} config
|
|
5727
|
+
*/
|
|
5728
|
+
function buildCancelRoute(ctx, runtime, config) {
|
|
5729
|
+
const routePath = resolveCancelPath(config)
|
|
5730
|
+
return {
|
|
5731
|
+
kind: 'exact',
|
|
5732
|
+
path: routePath,
|
|
5733
|
+
handler: (req, res) => {
|
|
5734
|
+
const remote = req && req.socket ? req.socket.remoteAddress : undefined
|
|
5735
|
+
const loopback = remote === undefined
|
|
5736
|
+
|| remote === '127.0.0.1'
|
|
5737
|
+
|| remote === '::1'
|
|
5738
|
+
|| remote === '::ffff:127.0.0.1'
|
|
5739
|
+
if (!loopback && !admittedByConnection(ctx, req)) {
|
|
5740
|
+
res.statusCode = 403
|
|
5741
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
5742
|
+
res.end('{"ok":false,"error":"loopback-only"}')
|
|
5743
|
+
return
|
|
5744
|
+
}
|
|
5745
|
+
if (!req || req.method !== 'POST') {
|
|
5746
|
+
res.statusCode = 405
|
|
5747
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
5748
|
+
res.setHeader('allow', 'POST')
|
|
5749
|
+
res.end('{"ok":false,"error":"method-not-allowed"}')
|
|
5750
|
+
return
|
|
5751
|
+
}
|
|
5752
|
+
void readJsonBody(req).then((payload) => {
|
|
5753
|
+
const body = payload && typeof payload === 'object' ? payload : {}
|
|
5754
|
+
const sessionId = typeof body.sessionId === 'string' ? body.sessionId.trim() : ''
|
|
5755
|
+
const scope = typeof body.scope === 'string' && body.scope.trim() !== '' ? body.scope.trim() : 'mine'
|
|
5756
|
+
const answer = (statusCode, value) => {
|
|
5757
|
+
res.statusCode = statusCode
|
|
5758
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
5759
|
+
res.setHeader('cache-control', 'no-store')
|
|
5760
|
+
res.end(JSON.stringify(value))
|
|
5761
|
+
}
|
|
5762
|
+
if (scope !== 'mine') {
|
|
5763
|
+
// "Cancel everybody" is a tool operation: the route says no.
|
|
5764
|
+
answer(400, {
|
|
5765
|
+
ok: false,
|
|
5766
|
+
cancelled: false,
|
|
5767
|
+
reason: 'scope-not-allowed',
|
|
5768
|
+
error: 'This route can only withdraw one session\'s own request (scope "mine").',
|
|
5769
|
+
remainingRequesters: 0,
|
|
5770
|
+
sessionId,
|
|
5771
|
+
})
|
|
5772
|
+
return
|
|
5773
|
+
}
|
|
5774
|
+
if (sessionId === '') {
|
|
5775
|
+
answer(400, {
|
|
5776
|
+
ok: false,
|
|
5777
|
+
cancelled: false,
|
|
5778
|
+
reason: 'session-id-required',
|
|
5779
|
+
error: 'The session id is required: without it the own request cannot be identified.',
|
|
5780
|
+
remainingRequesters: 0,
|
|
5781
|
+
sessionId: '',
|
|
5782
|
+
})
|
|
5783
|
+
return
|
|
5784
|
+
}
|
|
5785
|
+
const outcome = cancelOwnRestart(runtime.state, { sessionId, now: Date.now() })
|
|
5786
|
+
if (!outcome.removed) {
|
|
5787
|
+
// Nothing was changed: the caller is not a requester (or the
|
|
5788
|
+
// request is already gone). The reason says which.
|
|
5789
|
+
const error = outcome.reason === 'not-a-requester'
|
|
5790
|
+
? 'This session did not request the pending restart, so nothing was cancelled.'
|
|
5791
|
+
: outcome.reason === 'no-restart-pending'
|
|
5792
|
+
? 'There is no pending restart to cancel.'
|
|
5793
|
+
: 'The own request could not be identified, so nothing was cancelled.'
|
|
5794
|
+
runtime.log.warn(`the cancel route refused the request of session ${sessionId}: ${outcome.reason}`)
|
|
5795
|
+
answer(200, {
|
|
5796
|
+
ok: false,
|
|
5797
|
+
cancelled: false,
|
|
5798
|
+
reason: outcome.reason,
|
|
5799
|
+
error,
|
|
5800
|
+
remainingRequesters: outcome.remaining,
|
|
5801
|
+
sessionId,
|
|
5802
|
+
})
|
|
5803
|
+
return
|
|
5804
|
+
}
|
|
5805
|
+
// Release ONLY this session's continuation request.
|
|
5806
|
+
runtime.releaseRequesters([sessionId])
|
|
5807
|
+
if (outcome.cancelled) runtime.stopPolling()
|
|
5808
|
+
runtime.persist()
|
|
5809
|
+
runtime.log.info(
|
|
5810
|
+
outcome.cancelled
|
|
5811
|
+
? `session ${sessionId} withdrew the last restart request through the cancel route; the pending restart is cancelled`
|
|
5812
|
+
: `session ${sessionId} withdrew its restart request through the cancel route; ${outcome.remaining} requester(s) remain`,
|
|
5813
|
+
)
|
|
5814
|
+
answer(200, {
|
|
5815
|
+
ok: true,
|
|
5816
|
+
cancelled: outcome.cancelled,
|
|
5817
|
+
reason: outcome.reason,
|
|
5818
|
+
remainingRequesters: outcome.remaining,
|
|
5819
|
+
sessionId,
|
|
5820
|
+
})
|
|
5821
|
+
}).catch((error) => {
|
|
5822
|
+
res.statusCode = 400
|
|
5823
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
5824
|
+
res.end(JSON.stringify({ ok: false, error: describeError(error) }))
|
|
5825
|
+
})
|
|
5826
|
+
},
|
|
5827
|
+
}
|
|
5828
|
+
}
|
|
5829
|
+
|
|
3756
5830
|
/**
|
|
3757
5831
|
* Build the POST `/recover` route: retry the post-restart recovery for the
|
|
3758
5832
|
* still-pending sessions right now.
|
|
@@ -4019,7 +6093,7 @@ function buildChoiceRoute(ctx, runtime, config) {
|
|
|
4019
6093
|
res.end(JSON.stringify({
|
|
4020
6094
|
ok: false,
|
|
4021
6095
|
error: 'no-choice',
|
|
4022
|
-
instruction: 'Send { "includePlugins": true | false } and/or { "backupIncludePlugins": true | false } and/or { "backupOnRestart": true | false } and/or { "showSleepButton": true | false } and/or { "sleepCountdownSeconds": 0 | 10 | 20 | 30 | 60 } and/or { "restartGraceMs": 0 |
|
|
6096
|
+
instruction: 'Send { "includePlugins": true | false } and/or { "backupIncludePlugins": true | false } and/or { "backupOnRestart": true | false } and/or { "showSleepButton": true | false } and/or { "sleepCountdownSeconds": 0 | 10 | 20 | 30 | 60 } and/or { "restartGraceMs": 0 | 5000 | 10000 | 20000 | 30000 | 60000 }.',
|
|
4023
6097
|
}))
|
|
4024
6098
|
return
|
|
4025
6099
|
}
|
|
@@ -4134,6 +6208,223 @@ function buildChoiceRoute(ctx, runtime, config) {
|
|
|
4134
6208
|
}
|
|
4135
6209
|
}
|
|
4136
6210
|
|
|
6211
|
+
/**
|
|
6212
|
+
* The route of the self-starting supervisor.
|
|
6213
|
+
*
|
|
6214
|
+
* Three actions, and NOTHING happens without the confirmation:
|
|
6215
|
+
*
|
|
6216
|
+
* - `status` — who starts this host and whether the restart can work;
|
|
6217
|
+
* - `plan` — what the installation WOULD change (the dry run: it writes
|
|
6218
|
+
* nothing and runs nothing);
|
|
6219
|
+
* - `install` / `uninstall` — the real thing, and they need `confirmed: true`
|
|
6220
|
+
* (the confirmation card is what says "yes").
|
|
6221
|
+
*
|
|
6222
|
+
* @param {object} ctx the plugin context
|
|
6223
|
+
* @param {any} runtime the runtime of this plugin
|
|
6224
|
+
* @param {object} config the normalized config
|
|
6225
|
+
* @returns {object} the route
|
|
6226
|
+
*/
|
|
6227
|
+
function buildSupervisorRoute(ctx, runtime, config) {
|
|
6228
|
+
const routePath = resolveSupervisorPath(config)
|
|
6229
|
+
return {
|
|
6230
|
+
kind: 'exact',
|
|
6231
|
+
path: routePath,
|
|
6232
|
+
handler: (req, res) => {
|
|
6233
|
+
const remote = req && req.socket ? req.socket.remoteAddress : undefined
|
|
6234
|
+
const loopback = remote === undefined
|
|
6235
|
+
|| remote === '127.0.0.1'
|
|
6236
|
+
|| remote === '::1'
|
|
6237
|
+
|| remote === '::ffff:127.0.0.1'
|
|
6238
|
+
if (!loopback && !admittedByConnection(ctx, req)) {
|
|
6239
|
+
res.statusCode = 403
|
|
6240
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6241
|
+
res.end('{"ok":false,"error":"loopback-only"}')
|
|
6242
|
+
return
|
|
6243
|
+
}
|
|
6244
|
+
if (!req || req.method !== 'POST') {
|
|
6245
|
+
res.statusCode = 405
|
|
6246
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6247
|
+
res.setHeader('allow', 'POST')
|
|
6248
|
+
res.end('{"ok":false,"error":"method-not-allowed"}')
|
|
6249
|
+
return
|
|
6250
|
+
}
|
|
6251
|
+
void readJsonBody(req).then(async (payload) => {
|
|
6252
|
+
const body = payload && typeof payload === 'object' ? payload : {}
|
|
6253
|
+
const action = typeof body.action === 'string' ? body.action.trim() : ''
|
|
6254
|
+
if (action === 'status') {
|
|
6255
|
+
res.statusCode = 200
|
|
6256
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6257
|
+
res.setHeader('cache-control', 'no-store')
|
|
6258
|
+
res.end(JSON.stringify({ ok: true, action, supervisor: runtime.supervisorStatus() }))
|
|
6259
|
+
return
|
|
6260
|
+
}
|
|
6261
|
+
if (action === 'plan' || action === 'install-plan') {
|
|
6262
|
+
// The installation plan: what the registration WOULD do. Nothing is
|
|
6263
|
+
// written and nothing is run.
|
|
6264
|
+
const plan = planSupervisorInstall({
|
|
6265
|
+
platform: runtime.env.platform,
|
|
6266
|
+
// The desktop app manages itself: there is no plan there.
|
|
6267
|
+
isDesktopApp: runtime.env.isDesktopApp === true,
|
|
6268
|
+
// The startup matrix: only a `bare` host may get a supervisor.
|
|
6269
|
+
startup: runtime.startupMode(),
|
|
6270
|
+
// WHICH MACHINE IS THIS? (WSL is not a Linux.)
|
|
6271
|
+
host: runtime.hostKind(),
|
|
6272
|
+
nodePath: process.execPath,
|
|
6273
|
+
execPath: process.execPath,
|
|
6274
|
+
supervisorPath: supervisorPaths(path.dirname(runtime.stateFile)).script,
|
|
6275
|
+
configPath: supervisorPaths(path.dirname(runtime.stateFile)).config,
|
|
6276
|
+
kind: config.supervisorKind,
|
|
6277
|
+
})
|
|
6278
|
+
res.statusCode = 200
|
|
6279
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6280
|
+
res.setHeader('cache-control', 'no-store')
|
|
6281
|
+
res.end(JSON.stringify({
|
|
6282
|
+
ok: plan.error ? false : true,
|
|
6283
|
+
action,
|
|
6284
|
+
plan,
|
|
6285
|
+
error: plan.error ?? null,
|
|
6286
|
+
reason: plan.reason,
|
|
6287
|
+
supervisor: runtime.supervisorStatus(),
|
|
6288
|
+
}))
|
|
6289
|
+
return
|
|
6290
|
+
}
|
|
6291
|
+
if (action === 'dry-run' || action === 'derive') {
|
|
6292
|
+
// B: the dry run. It shows what WOULD be written and runs nothing.
|
|
6293
|
+
const plan = await runtime.deriveStartCommand()
|
|
6294
|
+
res.statusCode = 200
|
|
6295
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6296
|
+
res.setHeader('cache-control', 'no-store')
|
|
6297
|
+
res.end(JSON.stringify({ ok: true, action, plan, supervisor: runtime.supervisorStatus() }))
|
|
6298
|
+
return
|
|
6299
|
+
}
|
|
6300
|
+
if (action === 'apply' || action === 'revert') {
|
|
6301
|
+
if (body.confirmed !== true) {
|
|
6302
|
+
// The dry run is the confirmation: without it nothing is written.
|
|
6303
|
+
res.statusCode = 400
|
|
6304
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6305
|
+
res.setHeader('cache-control', 'no-store')
|
|
6306
|
+
res.end(JSON.stringify({
|
|
6307
|
+
ok: false,
|
|
6308
|
+
error: 'confirmation-required',
|
|
6309
|
+
instruction: 'Send { "action": "' + action + '", "confirmed": true } after the user confirmed the dry run.',
|
|
6310
|
+
}))
|
|
6311
|
+
return
|
|
6312
|
+
}
|
|
6313
|
+
const result = action === 'apply'
|
|
6314
|
+
? await runtime.applyDerivedCommand()
|
|
6315
|
+
: await runtime.revertDerivedCommand()
|
|
6316
|
+
res.statusCode = result.ok === true ? 200 : 500
|
|
6317
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6318
|
+
res.setHeader('cache-control', 'no-store')
|
|
6319
|
+
res.end(JSON.stringify({
|
|
6320
|
+
ok: result.ok === true,
|
|
6321
|
+
action,
|
|
6322
|
+
error: result.error ?? null,
|
|
6323
|
+
writes: result.writes ?? null,
|
|
6324
|
+
previous: result.previous ?? null,
|
|
6325
|
+
reason: result.reason ?? null,
|
|
6326
|
+
supervisor: runtime.supervisorStatus(),
|
|
6327
|
+
}))
|
|
6328
|
+
return
|
|
6329
|
+
}
|
|
6330
|
+
if (action === 'external') {
|
|
6331
|
+
// "I keep my own way": nothing is installed, but the answer is
|
|
6332
|
+
// recorded — so the plugin knows that a supervisor is already there
|
|
6333
|
+
// and never installs a second one.
|
|
6334
|
+
runtime.recordSupervisorChoice('external')
|
|
6335
|
+
try {
|
|
6336
|
+
await runtime.applyConfigChoice({ supervisorMode: 'external' })
|
|
6337
|
+
} catch (error) {
|
|
6338
|
+
runtime.log.warn('the supervisor mode could not be written:', String(error))
|
|
6339
|
+
}
|
|
6340
|
+
res.statusCode = 200
|
|
6341
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6342
|
+
res.setHeader('cache-control', 'no-store')
|
|
6343
|
+
res.end(JSON.stringify({ ok: true, action, supervisor: runtime.supervisorStatus() }))
|
|
6344
|
+
return
|
|
6345
|
+
}
|
|
6346
|
+
if (action === 'install' || action === 'uninstall') {
|
|
6347
|
+
if (action === 'install' && runtime.env.isDesktopApp === true) {
|
|
6348
|
+
// The desktop app manages itself: a login record would open a
|
|
6349
|
+
// SECOND client at every login. The refusal comes BEFORE the
|
|
6350
|
+
// confirmation, so nothing is asked of the user.
|
|
6351
|
+
res.statusCode = 400
|
|
6352
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6353
|
+
res.setHeader('cache-control', 'no-store')
|
|
6354
|
+
res.end(JSON.stringify({
|
|
6355
|
+
ok: false,
|
|
6356
|
+
action,
|
|
6357
|
+
error: 'desktop-manages-itself',
|
|
6358
|
+
reason: 'the desktop app relaunches itself; a supervisor would start a second copy',
|
|
6359
|
+
steps: [],
|
|
6360
|
+
supervisor: runtime.supervisorStatus(),
|
|
6361
|
+
}))
|
|
6362
|
+
return
|
|
6363
|
+
}
|
|
6364
|
+
if (body.confirmed !== true) {
|
|
6365
|
+
// The confirmation card is what says "yes": without it nothing is
|
|
6366
|
+
// changed on the machine.
|
|
6367
|
+
res.statusCode = 400
|
|
6368
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6369
|
+
res.setHeader('cache-control', 'no-store')
|
|
6370
|
+
res.end(JSON.stringify({
|
|
6371
|
+
ok: false,
|
|
6372
|
+
error: 'confirmation-required',
|
|
6373
|
+
instruction: 'Send { "action": "' + action + '", "confirmed": true } after the user confirmed the card.',
|
|
6374
|
+
}))
|
|
6375
|
+
return
|
|
6376
|
+
}
|
|
6377
|
+
if (action === 'install') {
|
|
6378
|
+
const choice = typeof body.choice === 'string' ? body.choice : ''
|
|
6379
|
+
if (choice !== '') runtime.recordSupervisorChoice(choice)
|
|
6380
|
+
const result = runtime.installSupervisor({ confirmed: true })
|
|
6381
|
+
res.statusCode = result.ok ? 200 : 500
|
|
6382
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6383
|
+
res.setHeader('cache-control', 'no-store')
|
|
6384
|
+
res.end(JSON.stringify({
|
|
6385
|
+
ok: result.ok === true,
|
|
6386
|
+
action,
|
|
6387
|
+
kind: result.kind,
|
|
6388
|
+
degraded: result.degraded,
|
|
6389
|
+
steps: result.steps,
|
|
6390
|
+
error: result.error,
|
|
6391
|
+
reason: result.reason,
|
|
6392
|
+
supervisor: runtime.supervisorStatus(),
|
|
6393
|
+
}))
|
|
6394
|
+
return
|
|
6395
|
+
}
|
|
6396
|
+
const result = await runtime.uninstallSupervisor()
|
|
6397
|
+
res.statusCode = result.ok ? 200 : 500
|
|
6398
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6399
|
+
res.setHeader('cache-control', 'no-store')
|
|
6400
|
+
res.end(JSON.stringify({
|
|
6401
|
+
ok: result.ok === true,
|
|
6402
|
+
action,
|
|
6403
|
+
steps: result.steps,
|
|
6404
|
+
error: result.error,
|
|
6405
|
+
pending: result.pending,
|
|
6406
|
+
previous: result.previous,
|
|
6407
|
+
supervisor: runtime.supervisorStatus(),
|
|
6408
|
+
}))
|
|
6409
|
+
return
|
|
6410
|
+
}
|
|
6411
|
+
res.statusCode = 400
|
|
6412
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6413
|
+
res.setHeader('cache-control', 'no-store')
|
|
6414
|
+
res.end(JSON.stringify({
|
|
6415
|
+
ok: false,
|
|
6416
|
+
error: 'unknown-action',
|
|
6417
|
+
instruction: 'Send { "action": "status" | "plan" | "install" | "uninstall" }.',
|
|
6418
|
+
}))
|
|
6419
|
+
}).catch((error) => {
|
|
6420
|
+
res.statusCode = 500
|
|
6421
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6422
|
+
res.end(JSON.stringify({ ok: false, error: describeError(error) }))
|
|
6423
|
+
})
|
|
6424
|
+
},
|
|
6425
|
+
}
|
|
6426
|
+
}
|
|
6427
|
+
|
|
4137
6428
|
/**
|
|
4138
6429
|
* Is the graceful sleep a dry run?
|
|
4139
6430
|
*
|
|
@@ -4158,16 +6449,16 @@ function sleepDryRun() {
|
|
|
4158
6449
|
* happened). The sleep feature counts down on screen itself, so it only needs
|
|
4159
6450
|
* the busy check — and it keeps `settleMs` out of the picture entirely.
|
|
4160
6451
|
*
|
|
4161
|
-
* The
|
|
4162
|
-
*
|
|
4163
|
-
*
|
|
4164
|
-
*
|
|
6452
|
+
* The shutdown has NO invisible reserve of its own: the visible countdown
|
|
6453
|
+
* (`sleepCountdownSeconds`) IS the wait before the machine goes off, and it is
|
|
6454
|
+
* on screen, so the user can see it and cancel it. A second, hidden window
|
|
6455
|
+
* next to it would only say the same thing twice.
|
|
4165
6456
|
*
|
|
4166
6457
|
* The restart gate itself is untouched: the restart feature keeps its quiet
|
|
4167
|
-
* window exactly as it was (plus
|
|
6458
|
+
* window exactly as it was (plus its own restart reserve).
|
|
4168
6459
|
*
|
|
4169
6460
|
* @param {object} runtime
|
|
4170
|
-
* @returns {{ allow: boolean, reason: 'busy'|'
|
|
6461
|
+
* @returns {{ allow: boolean, reason: 'busy'|'quiet', blockedBy: any[] }}
|
|
4171
6462
|
*/
|
|
4172
6463
|
function sleepGate(runtime) {
|
|
4173
6464
|
// A pending restart goes FIRST: the shutdown waits for it — and for the
|
|
@@ -4199,16 +6490,8 @@ function sleepGate(runtime) {
|
|
|
4199
6490
|
}
|
|
4200
6491
|
const list = Array.isArray(busy) ? busy : []
|
|
4201
6492
|
if (list.length > 0) return { allow: false, reason: 'busy', blockedBy: list }
|
|
4202
|
-
//
|
|
4203
|
-
//
|
|
4204
|
-
let quiet = 0
|
|
4205
|
-
let grace = 0
|
|
4206
|
-
try { quiet = Number(runtime.quietForMs()) } catch { quiet = 0 }
|
|
4207
|
-
try { grace = Number(runtime.graceMs()) } catch { grace = 0 }
|
|
4208
|
-
if (!Number.isFinite(quiet) || quiet < 0) quiet = 0
|
|
4209
|
-
if (Number.isFinite(grace) && grace > 0 && quiet < grace) {
|
|
4210
|
-
return { allow: false, reason: 'grace', blockedBy: [] }
|
|
4211
|
-
}
|
|
6493
|
+
// Nobody is busy and no restart is pending: the countdown may run. There is
|
|
6494
|
+
// no invisible reserve — the countdown on screen is the wait.
|
|
4212
6495
|
return { allow: true, reason: 'quiet', blockedBy: [] }
|
|
4213
6496
|
}
|
|
4214
6497
|
|
|
@@ -4290,11 +6573,11 @@ function createSleepService(ctx, runtime, config, options = {}) {
|
|
|
4290
6573
|
busyAgents: () => runtime.busyAgents(),
|
|
4291
6574
|
gate,
|
|
4292
6575
|
titleOf: (sessionId) => runtime.sessionTitle(sessionId),
|
|
4293
|
-
// The
|
|
4294
|
-
//
|
|
4295
|
-
// plugin
|
|
4296
|
-
|
|
4297
|
-
graceMs: () =>
|
|
6576
|
+
// The shutdown has NO invisible reserve: the countdown on screen IS the
|
|
6577
|
+
// wait before the machine goes off. The controller keeps the parameter for
|
|
6578
|
+
// compatibility, but the plugin never injects a window — and it never
|
|
6579
|
+
// injects the RESTART reserve either.
|
|
6580
|
+
graceMs: () => 0,
|
|
4298
6581
|
// A pending restart goes first: the shutdown waits for it.
|
|
4299
6582
|
restartPending: () => {
|
|
4300
6583
|
const pending = runtime.pendingRestart()
|
|
@@ -4429,9 +6712,10 @@ function sleepStatusPayload(sleep, config, runtime) {
|
|
|
4429
6712
|
armed: status.armed,
|
|
4430
6713
|
showSleepButton: config.showSleepButton !== false,
|
|
4431
6714
|
countdownSeconds: status.countdownSeconds,
|
|
4432
|
-
// The
|
|
4433
|
-
//
|
|
4434
|
-
|
|
6715
|
+
// The RESTART reserve is reported next to the countdown, so the settings
|
|
6716
|
+
// page can show it. The shutdown itself has NO invisible reserve: the
|
|
6717
|
+
// visible countdown is the wait before the machine goes off.
|
|
6718
|
+
restartGraceMs: config.restartGraceMs,
|
|
4435
6719
|
quietForMs: runtime && typeof runtime.quietForMs === 'function' ? runtime.quietForMs() : 0,
|
|
4436
6720
|
// When the shutdown was armed: the intent has to survive a restart, so the
|
|
4437
6721
|
// time is reported for the troubleshooting.
|
|
@@ -4573,12 +6857,18 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
4573
6857
|
generation: state.generation,
|
|
4574
6858
|
pendingRestart: state.restart !== null,
|
|
4575
6859
|
settleMs: config.settleMs,
|
|
4576
|
-
// The
|
|
4577
|
-
// are needed to tell "waiting for the
|
|
6860
|
+
// The RESTART reserve and the quiet window the gate really uses:
|
|
6861
|
+
// both are needed to tell "waiting for the reserve" from "somebody
|
|
6862
|
+
// is busy".
|
|
4578
6863
|
graceMs: config.restartGraceMs,
|
|
4579
6864
|
effectiveSettleMs: runtime.effectiveSettleMs(),
|
|
6865
|
+
// The upper bound of a queued (unclaimed) inbox message: after this
|
|
6866
|
+
// window it stops blocking the restart.
|
|
6867
|
+
queuedGraceMs: config.queuedGraceMs,
|
|
4580
6868
|
quietForMs: typeof verdict.quietForMs === 'number' ? verdict.quietForMs : 0,
|
|
4581
|
-
|
|
6869
|
+
// The number of TASKS the restart waits for: a session and its own
|
|
6870
|
+
// subagents are ONE task, exactly as the left sidebar counts them.
|
|
6871
|
+
busyCount: countBlockerTasks(verdict.blockedBy),
|
|
4582
6872
|
agentCount,
|
|
4583
6873
|
})
|
|
4584
6874
|
res.statusCode = 200
|
|
@@ -4639,17 +6929,32 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
4639
6929
|
waiters,
|
|
4640
6930
|
blockedBy,
|
|
4641
6931
|
allowRestart: verdict.allow === true,
|
|
6932
|
+
// WHY the gate says what it says. Without this, "allowRestart is
|
|
6933
|
+
// false and blockedBy is empty" looks like a contradiction: the
|
|
6934
|
+
// reason tells the two apart (an empty blocker list with the
|
|
6935
|
+
// reason "settling" means the quiet window is not full yet).
|
|
6936
|
+
reason: typeof verdict.reason === 'string' ? verdict.reason : 'unknown',
|
|
4642
6937
|
quietForMs: typeof verdict.quietForMs === 'number' ? verdict.quietForMs : 0,
|
|
4643
6938
|
settleMs: config.settleMs,
|
|
4644
|
-
// The
|
|
6939
|
+
// The RESTART reserve and the quiet window the gate really uses.
|
|
4645
6940
|
graceMs: config.restartGraceMs,
|
|
4646
6941
|
effectiveSettleMs: runtime.effectiveSettleMs(),
|
|
6942
|
+
// The upper bound of a queued (unclaimed) inbox message.
|
|
6943
|
+
queuedGraceMs: config.queuedGraceMs,
|
|
6944
|
+
// The long wait: present only when the restart passed the hard
|
|
6945
|
+
// deadline while a conversation was still running. It says how
|
|
6946
|
+
// long the restart has been waiting, why, and who holds it — the
|
|
6947
|
+
// request itself is never dropped.
|
|
6948
|
+
longWait: runtime.longWaitInfo(),
|
|
4647
6949
|
updatedAt: live.updatedAt,
|
|
4648
6950
|
// The park route, so the client never has to guess the prefix, and
|
|
4649
6951
|
// the goals this plugin already disarmed (they are re-armed after
|
|
4650
6952
|
// the restart).
|
|
4651
6953
|
parkUrl: resolveParkPath(config),
|
|
4652
6954
|
recoverUrl: resolveRecoverPath(config),
|
|
6955
|
+
// The route that withdraws ONE session's own restart request (the
|
|
6956
|
+
// manual "cancel my restart" of the pill).
|
|
6957
|
+
cancelUrl: resolveCancelPath(config),
|
|
4653
6958
|
// The route that persists the one-time install choice, plus the
|
|
4654
6959
|
// value that is effective right now.
|
|
4655
6960
|
choiceUrl: resolveChoicePath(config),
|
|
@@ -4658,8 +6963,67 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
4658
6963
|
announcementsUrl: resolveAnnouncementsPath(config),
|
|
4659
6964
|
backupOnRestart: config.backupOnRestart !== false,
|
|
4660
6965
|
backupIncludePlugins: config.backupIncludePlugins !== false,
|
|
6966
|
+
// WHAT IS ON DISK RIGHT NOW: the backups of this host (the count,
|
|
6967
|
+
// the local time, the size, whether the plugins are in it and
|
|
6968
|
+
// where it lives). The browser cannot read the disk, so the Host
|
|
6969
|
+
// reads it — only the directory, never the contents.
|
|
6970
|
+
backups: runtime.backupsInfo(),
|
|
6971
|
+
// THE RESTART COMMAND that is in effect: what is written, and what
|
|
6972
|
+
// an undo would restore. The settings page shows it in a folded
|
|
6973
|
+
// "for a technical colleague" section.
|
|
6974
|
+
restart: {
|
|
6975
|
+
mode: config.restartMode,
|
|
6976
|
+
command: typeof config.restartCommand === 'string' ? config.restartCommand : '',
|
|
6977
|
+
written: typeof config.restartCommand === 'string' && config.restartCommand !== '',
|
|
6978
|
+
canRevert: Boolean(
|
|
6979
|
+
live.onboarding
|
|
6980
|
+
&& typeof live.onboarding === 'object'
|
|
6981
|
+
&& live.onboarding.previous
|
|
6982
|
+
&& typeof live.onboarding.previous.restartMode === 'string',
|
|
6983
|
+
),
|
|
6984
|
+
previous: live.onboarding && typeof live.onboarding === 'object' && live.onboarding.previous
|
|
6985
|
+
? live.onboarding.previous
|
|
6986
|
+
: null,
|
|
6987
|
+
},
|
|
4661
6988
|
parkedGoals: parkedGoalList(runtime),
|
|
4662
6989
|
recovery: recoveryList(live),
|
|
6990
|
+
// THE ACCESS LINK of this instance (with the boot token). The boot
|
|
6991
|
+
// token is created per process, so the link the browser holds
|
|
6992
|
+
// answers 401 after a restart. The client needs the new link to
|
|
6993
|
+
// get the user back into the interface.
|
|
6994
|
+
accessUrl: accessUrlOf(runtime),
|
|
6995
|
+
// The supervisor: who starts this host, and whether the restart
|
|
6996
|
+
// can work at all.
|
|
6997
|
+
supervisor: runtime.supervisorStatus(),
|
|
6998
|
+
// WHICH END this is. The browser side serves the desktop app AND
|
|
6999
|
+
// the hosted web host, so it may not assume one of them: the
|
|
7000
|
+
// desktop app brings itself back up (there is nothing to install
|
|
7001
|
+
// there), while the hosted host may ask for a supervisor.
|
|
7002
|
+
instance: {
|
|
7003
|
+
isDesktopApp: runtime.env.isDesktopApp === true,
|
|
7004
|
+
platform: runtime.env.platform,
|
|
7005
|
+
},
|
|
7006
|
+
// THE LAST RESTART: the snapshot that was taken before it (the
|
|
7007
|
+
// directory, its LOCAL time, how long it took, which older
|
|
7008
|
+
// snapshots were pruned) and the verdict of the instance that came
|
|
7009
|
+
// up (the four checks). Without it a restart has no visible
|
|
7010
|
+
// result at all.
|
|
7011
|
+
lastRestart: runtime.lastRestartVerdict(),
|
|
7012
|
+
// Whether the one-time question about the supervisor may be asked.
|
|
7013
|
+
supervisorOffer: runtime.supervisorOffer,
|
|
7014
|
+
// The answer of the one-time question about the supervisor (null
|
|
7015
|
+
// while it was never answered).
|
|
7016
|
+
onboarding: runtime.onboardingChoice,
|
|
7017
|
+
// The route that installs, uninstalls and reports the supervisor.
|
|
7018
|
+
supervisorUrl: resolveSupervisorPath(config),
|
|
7019
|
+
// "REPORT A PROBLEM": the diagnostic text (built by ONE generator on
|
|
7020
|
+
// the host, so the browser and the host can never drift) and the
|
|
7021
|
+
// two places a report can go.
|
|
7022
|
+
diagnostics: runtime.diagnostics(),
|
|
7023
|
+
report: {
|
|
7024
|
+
email: SUPPORT_EMAIL,
|
|
7025
|
+
issueUrlBase: ISSUE_URL_BASE,
|
|
7026
|
+
},
|
|
4663
7027
|
})
|
|
4664
7028
|
} catch {
|
|
4665
7029
|
// The banner must never break the health route: fall back to "no restart".
|
|
@@ -4669,9 +7033,13 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
4669
7033
|
waiters: [],
|
|
4670
7034
|
blockedBy: [],
|
|
4671
7035
|
allowRestart: false,
|
|
7036
|
+
reason: 'unknown',
|
|
4672
7037
|
quietForMs: 0,
|
|
7038
|
+
queuedGraceMs: config.queuedGraceMs,
|
|
7039
|
+
longWait: null,
|
|
4673
7040
|
parkUrl: resolveParkPath(config),
|
|
4674
7041
|
recoverUrl: resolveRecoverPath(config),
|
|
7042
|
+
cancelUrl: resolveCancelPath(config),
|
|
4675
7043
|
choiceUrl: resolveChoicePath(config),
|
|
4676
7044
|
announcementsUrl: resolveAnnouncementsPath(config),
|
|
4677
7045
|
backupOnRestart: config.backupOnRestart !== false,
|
|
@@ -4733,49 +7101,44 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
4733
7101
|
const recoverRoute = buildRecoverRoute(ctx, runtime, config)
|
|
4734
7102
|
const handoffRoute = buildHandoffRoute(ctx, runtime, config)
|
|
4735
7103
|
const choiceRoute = buildChoiceRoute(ctx, runtime, config)
|
|
7104
|
+
const cancelRoute = buildCancelRoute(ctx, runtime, config)
|
|
4736
7105
|
const announcementsRoute = buildAnnouncementsRoute(ctx, runtime, config)
|
|
7106
|
+
const supervisorRoute = buildSupervisorRoute(ctx, runtime, config)
|
|
4737
7107
|
const sleepRoute = sleep ? buildSleepRoute(ctx, runtime, config, sleep) : null
|
|
4738
7108
|
try {
|
|
4739
7109
|
const origin = typeof webServer.getOrigin === 'function' ? String(webServer.getOrigin()) : ''
|
|
4740
7110
|
const portMatch = /:(\d+)\/?$/.exec(origin)
|
|
4741
7111
|
if (portMatch) runtime.webPort = Number(portMatch[1])
|
|
7112
|
+
// THE INSTANCE RECORD. The new instance publishes its own port and its
|
|
7113
|
+
// own access link (with the boot token), so:
|
|
7114
|
+
// - the rollback watcher learns the port of the NEW instance instead
|
|
7115
|
+
// of guessing the (random) port of the old one;
|
|
7116
|
+
// - the user finds the access link of the new instance after a
|
|
7117
|
+
// restart, instead of being locked out by a 401.
|
|
7118
|
+
publishInstanceRecord(ctx, runtime, config, origin)
|
|
4742
7119
|
} catch {}
|
|
4743
|
-
|
|
4744
|
-
|
|
4745
|
-
|
|
4746
|
-
|
|
4747
|
-
|
|
4748
|
-
|
|
4749
|
-
|
|
4750
|
-
|
|
4751
|
-
|
|
4752
|
-
|
|
4753
|
-
|
|
4754
|
-
|
|
4755
|
-
|
|
4756
|
-
|
|
4757
|
-
|
|
4758
|
-
}
|
|
4759
|
-
try {
|
|
4760
|
-
target.effect(() => webServer.register(handoffRoute), 'dsh-autorestart: handoff route')
|
|
4761
|
-
} catch (error) {
|
|
4762
|
-
runtime.log.warn('could not register the handoff route:', String(error))
|
|
4763
|
-
}
|
|
4764
|
-
try {
|
|
4765
|
-
target.effect(() => webServer.register(choiceRoute), 'dsh-autorestart: choice route')
|
|
4766
|
-
} catch (error) {
|
|
4767
|
-
runtime.log.warn('could not register the choice route:', String(error))
|
|
4768
|
-
}
|
|
4769
|
-
try {
|
|
4770
|
-
target.effect(() => webServer.register(announcementsRoute), 'dsh-autorestart: announcements route')
|
|
4771
|
-
} catch (error) {
|
|
4772
|
-
runtime.log.warn('could not register the announcements route:', String(error))
|
|
4773
|
-
}
|
|
4774
|
-
if (sleepRoute !== null) {
|
|
7120
|
+
// The routes of this plugin. Every one of them is remembered: the boot
|
|
7121
|
+
// check of the restart record reports "the routes are there" — a host
|
|
7122
|
+
// that came up without them is not a healthy host.
|
|
7123
|
+
const routes = [
|
|
7124
|
+
{ name: 'health', route },
|
|
7125
|
+
{ name: 'park', route: parkRoute },
|
|
7126
|
+
{ name: 'recover', route: recoverRoute },
|
|
7127
|
+
{ name: 'handoff', route: handoffRoute },
|
|
7128
|
+
{ name: 'choice', route: choiceRoute },
|
|
7129
|
+
{ name: 'cancel', route: cancelRoute },
|
|
7130
|
+
{ name: 'announcements', route: announcementsRoute },
|
|
7131
|
+
{ name: 'supervisor', route: supervisorRoute },
|
|
7132
|
+
]
|
|
7133
|
+
if (sleepRoute !== null) routes.push({ name: 'sleep', route: sleepRoute })
|
|
7134
|
+
for (const entry of routes) {
|
|
4775
7135
|
try {
|
|
4776
|
-
target.effect(() => webServer.register(
|
|
7136
|
+
target.effect(() => webServer.register(entry.route), 'dsh-autorestart: ' + entry.name + ' route')
|
|
7137
|
+
runtime.registeredRoutes.push(entry.name)
|
|
4777
7138
|
} catch (error) {
|
|
4778
|
-
|
|
7139
|
+
// A duplicate registration means another load of this plugin already owns it.
|
|
7140
|
+
runtime.log.warn('could not register the ' + entry.name + ' route:', String(error))
|
|
7141
|
+
runtime.routeErrors.push({ route: entry.name, error: String(error) })
|
|
4779
7142
|
}
|
|
4780
7143
|
}
|
|
4781
7144
|
if (config.autoReloadPage !== false && typeof webServer.tapIndex === 'function') {
|
|
@@ -4838,9 +7201,20 @@ function trackAgents(ctx, runtime, config) {
|
|
|
4838
7201
|
}
|
|
4839
7202
|
})
|
|
4840
7203
|
|
|
4841
|
-
//
|
|
4842
|
-
|
|
7204
|
+
// A session event only restarts the settle window when it says that WORK
|
|
7205
|
+
// happened (a message, a tool, a turn). The host produces session events
|
|
7206
|
+
// every two to three seconds while the interface polls and synchronises —
|
|
7207
|
+
// counting those as activity kept the quiet window from ever filling, so a
|
|
7208
|
+
// restart with a grace window of ten seconds NEVER fired. When the type of
|
|
7209
|
+
// an event cannot be read, it is not activity: the plugin would rather
|
|
7210
|
+
// restart a little early than never restart at all.
|
|
7211
|
+
//
|
|
7212
|
+
// The hard guarantee is elsewhere and untouched: a conversation that is
|
|
7213
|
+
// really running blocks the restart through `busyAgents()`, which is never
|
|
7214
|
+
// relaxed.
|
|
7215
|
+
ctx.on('session/event', (payload) => {
|
|
4843
7216
|
try {
|
|
7217
|
+
if (!isWorkSessionEvent(payload)) return
|
|
4844
7218
|
runtime.noteActivity()
|
|
4845
7219
|
} catch {}
|
|
4846
7220
|
})
|
|
@@ -4989,15 +7363,35 @@ export function apply(ctx, rawConfig, options = {}) {
|
|
|
4989
7363
|
if (runtime.isNewBoot && config.resumeOnBoot) {
|
|
4990
7364
|
runtime.recoverTimer = ctx.timeout(() => {
|
|
4991
7365
|
recoverAfterRestart(ctx, runtime, config)
|
|
4992
|
-
.then(() => {
|
|
7366
|
+
.then(() => {
|
|
7367
|
+
runtime.recovery = { ran: true, ok: true, detail: 'the boot recovery finished' }
|
|
7368
|
+
restoreSleepIntent()
|
|
7369
|
+
})
|
|
4993
7370
|
.catch((error) => {
|
|
4994
7371
|
log.warn('boot recovery failed:', String(error))
|
|
7372
|
+
runtime.recovery = { ran: true, ok: false, detail: 'the boot recovery failed: ' + String(error) }
|
|
4995
7373
|
restoreSleepIntent()
|
|
4996
7374
|
})
|
|
7375
|
+
// THE VERDICT OF THE RESTART. The instance that replaced the old one
|
|
7376
|
+
// checks itself (listener / routes / plugin / recovery) and writes the
|
|
7377
|
+
// result into the durable restart record. Without this the four
|
|
7378
|
+
// checks would live nowhere and a failed restart would look like a
|
|
7379
|
+
// successful one.
|
|
7380
|
+
.then(() => verifyRestartBoot(runtime))
|
|
7381
|
+
.catch((error) => {
|
|
7382
|
+
log.warn('the boot verification failed:', String(error))
|
|
7383
|
+
})
|
|
4997
7384
|
}, Math.max(0, config.recoveryDelayMs))
|
|
4998
7385
|
} else {
|
|
4999
7386
|
// No recovery pass: the intent is restored right away (a hot reload must
|
|
5000
7387
|
// not lose it either).
|
|
7388
|
+
runtime.recovery = {
|
|
7389
|
+
ran: false,
|
|
7390
|
+
ok: true,
|
|
7391
|
+
detail: runtime.isNewBoot
|
|
7392
|
+
? 'the boot recovery is switched off'
|
|
7393
|
+
: 'this is not a new boot',
|
|
7394
|
+
}
|
|
5001
7395
|
restoreSleepIntent()
|
|
5002
7396
|
}
|
|
5003
7397
|
|
|
@@ -5027,4 +7421,5 @@ export {
|
|
|
5027
7421
|
pluginVersion,
|
|
5028
7422
|
PLUGIN_VERSION_FALLBACK,
|
|
5029
7423
|
SLEEP_FEATURE_VERSION,
|
|
7424
|
+
SUPERVISOR_FEATURE_VERSION,
|
|
5030
7425
|
}
|