dsh-autorestart 0.6.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +460 -12
- package/assets/screenshot-1.png +0 -0
- package/assets/screenshot-2.png +0 -0
- package/assets/screenshot-3.png +0 -0
- package/assets/screenshot-4.png +0 -0
- package/assets/screenshot-5.png +0 -0
- package/client.js +6329 -3863
- package/cordis.patch.yml +44 -1
- package/docs/README.zh.md +297 -9
- package/docs/platform-matrix.md +92 -0
- package/docs/rescue-drill.md +606 -0
- package/index.js +2867 -199
- package/lib/access.js +212 -0
- package/lib/autostart-advice.js +198 -0
- package/lib/backup.js +131 -4
- package/lib/blockers.js +228 -0
- package/lib/checkpoint.js +147 -0
- package/lib/diagnostics.js +188 -0
- package/lib/health.js +264 -0
- package/lib/host-kind.js +145 -0
- package/lib/node-runtime.js +144 -0
- package/lib/rescue-paths.js +253 -0
- package/lib/rescue.js +596 -0
- package/lib/restart-verdict.js +327 -0
- package/lib/restart.js +823 -56
- package/lib/scheduler-capability.js +300 -0
- package/lib/sleep.js +46 -49
- package/lib/startup-mode.js +491 -0
- package/lib/supervisor-config.js +735 -0
- package/lib/supervisor-install.js +557 -0
- package/lib/supervisor.js +1276 -0
- package/lib/watchdog-arbiter.js +117 -0
- package/lib/watchdog-schedule.js +310 -0
- package/lib/watchdog-tier.js +182 -0
- package/lib/watchdog.js +479 -0
- package/locale/en.json +274 -64
- package/locale/zh.json +274 -64
- package/package.json +7 -2
- package/screenshots.json +6 -4
- package/scripts/README.md +65 -0
- package/scripts/drill-rescue.ps1 +508 -0
- package/scripts/emergency-disable.cmd +27 -0
- package/scripts/emergency-toggle.ps1 +373 -0
- package/scripts/restore-profile.cmd +44 -0
- package/scripts/restore-profile.ps1 +608 -0
- package/scripts/run-tests.ps1 +46 -0
- package/supervisor/supervisor.js +884 -0
package/index.js
CHANGED
|
@@ -19,13 +19,16 @@
|
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
21
|
import crypto from 'node:crypto'
|
|
22
|
+
import { spawn } from 'node:child_process'
|
|
22
23
|
import fs from 'node:fs'
|
|
23
24
|
import os from 'node:os'
|
|
24
25
|
import path from 'node:path'
|
|
26
|
+
import { fileURLToPath } from 'node:url'
|
|
25
27
|
|
|
26
28
|
import {
|
|
27
29
|
armSleep,
|
|
28
30
|
beginBoot,
|
|
31
|
+
cancelOwnRestart,
|
|
29
32
|
cancelRestart,
|
|
30
33
|
clearSleep,
|
|
31
34
|
disarmedRecord,
|
|
@@ -37,6 +40,7 @@ import {
|
|
|
37
40
|
parkedRecord,
|
|
38
41
|
pruneSessions,
|
|
39
42
|
readState,
|
|
43
|
+
recordOnboarding,
|
|
40
44
|
recoverableSessions,
|
|
41
45
|
requestRestart,
|
|
42
46
|
setContextOverflow,
|
|
@@ -62,18 +66,112 @@ import {
|
|
|
62
66
|
} from './lib/lifecycle.js'
|
|
63
67
|
import {
|
|
64
68
|
backupsRoot,
|
|
69
|
+
claimRollbackLock,
|
|
65
70
|
clearRollbackReport,
|
|
66
71
|
createBackup,
|
|
72
|
+
listBackups,
|
|
67
73
|
readRollbackReport,
|
|
74
|
+
releaseRollbackLock,
|
|
75
|
+
restoreBackup,
|
|
76
|
+
ROLLBACK_LOCK_NAME,
|
|
68
77
|
ROLLBACK_REPORT_NAME,
|
|
69
78
|
} from './lib/backup.js'
|
|
79
|
+
import {
|
|
80
|
+
accessFilePath,
|
|
81
|
+
readAccessRecord,
|
|
82
|
+
resolveAccessUrl,
|
|
83
|
+
writeAccessRecord,
|
|
84
|
+
} from './lib/access.js'
|
|
85
|
+
import {
|
|
86
|
+
detectSupervisor,
|
|
87
|
+
installSupervisor,
|
|
88
|
+
planSupervisorInstall,
|
|
89
|
+
readSupervisorConfig,
|
|
90
|
+
requestSupervisorAction,
|
|
91
|
+
supervisorPaths,
|
|
92
|
+
supervisorStatus,
|
|
93
|
+
uninstallSupervisor,
|
|
94
|
+
writeRollbackRequest,
|
|
95
|
+
} from './lib/supervisor.js'
|
|
96
|
+
import { composeFragment, detectInitSystems } from './lib/supervisor-install.js'
|
|
97
|
+
import { classifyStartup } from './lib/startup-mode.js'
|
|
98
|
+
import {
|
|
99
|
+
createDesktopShortcut,
|
|
100
|
+
desktopDir,
|
|
101
|
+
rescueInfo,
|
|
102
|
+
rescueScriptPaths,
|
|
103
|
+
} from './lib/rescue-paths.js'
|
|
104
|
+
import {
|
|
105
|
+
emptyCapabilities,
|
|
106
|
+
isCapabilities,
|
|
107
|
+
markSchedulerWarned,
|
|
108
|
+
probeSchedulerCapability,
|
|
109
|
+
SCHEDULER_UNKNOWN,
|
|
110
|
+
shouldWarnScheduler,
|
|
111
|
+
} from './lib/scheduler-capability.js'
|
|
112
|
+
import {
|
|
113
|
+
detectWatchdogCapabilities,
|
|
114
|
+
tierProvidesRollback,
|
|
115
|
+
tierWithProof,
|
|
116
|
+
TIERS,
|
|
117
|
+
} from './lib/watchdog-tier.js'
|
|
118
|
+
import {
|
|
119
|
+
buildPending,
|
|
120
|
+
clearReady,
|
|
121
|
+
readReady,
|
|
122
|
+
readReport,
|
|
123
|
+
writePending,
|
|
124
|
+
writeReady,
|
|
125
|
+
writeReport,
|
|
126
|
+
} from './lib/watchdog.js'
|
|
127
|
+
import {
|
|
128
|
+
ARBITERS,
|
|
129
|
+
planWatchdogArbiter,
|
|
130
|
+
} from './lib/watchdog-arbiter.js'
|
|
131
|
+
import {
|
|
132
|
+
cleanupSystemdUnit,
|
|
133
|
+
detachedPidPath,
|
|
134
|
+
localIso,
|
|
135
|
+
registerDetachedProcess,
|
|
136
|
+
registerScheduledTask,
|
|
137
|
+
registerSystemdRun,
|
|
138
|
+
removeScheduledTask,
|
|
139
|
+
} from './lib/watchdog-schedule.js'
|
|
140
|
+
import { detectHostKind } from './lib/host-kind.js'
|
|
141
|
+
import { planAutostartAdvice } from './lib/autostart-advice.js'
|
|
142
|
+
import { buildDiagnostics, ISSUE_URL_BASE, SUPPORT_EMAIL } from './lib/diagnostics.js'
|
|
143
|
+
import { countBlockerTasks } from './lib/blockers.js'
|
|
144
|
+
import {
|
|
145
|
+
buildDryRunPlan,
|
|
146
|
+
composeRestartFragment,
|
|
147
|
+
deriveStartCommand,
|
|
148
|
+
detectContainer,
|
|
149
|
+
detectExternalSupervisor,
|
|
150
|
+
inspectExternalSupervisor,
|
|
151
|
+
pickFreePort,
|
|
152
|
+
} from './lib/supervisor-config.js'
|
|
153
|
+
import {
|
|
154
|
+
buildRestartVerdict,
|
|
155
|
+
buildSnapshotRecord,
|
|
156
|
+
evaluateRestartChecks,
|
|
157
|
+
readRestartVerdict,
|
|
158
|
+
RESTART_RESULTS,
|
|
159
|
+
summarizeRestartVerdict,
|
|
160
|
+
writeRestartVerdict,
|
|
161
|
+
} from './lib/restart-verdict.js'
|
|
162
|
+
import { waitForHealthyInstance } from './lib/health.js'
|
|
70
163
|
import {
|
|
71
164
|
buildCustomCommand,
|
|
72
165
|
buildWatchScript,
|
|
166
|
+
DEFAULT_COMMAND_TIMEOUT_MS,
|
|
167
|
+
DEFAULT_ROLLBACK_COPY_TIMEOUT_MS,
|
|
73
168
|
DEFAULT_ROLLBACK_TIMEOUT_MS,
|
|
74
169
|
detectEnvironment,
|
|
170
|
+
launchDetachedWatcher,
|
|
171
|
+
readProcessChain,
|
|
75
172
|
runCommandNow,
|
|
76
173
|
scheduleDetachedRestart,
|
|
174
|
+
writeTextFileAtomic,
|
|
77
175
|
} from './lib/restart.js'
|
|
78
176
|
import { compareSemver } from './lib/semver.js'
|
|
79
177
|
import { isWorkSessionEvent } from './lib/activity.js'
|
|
@@ -92,6 +190,16 @@ import {
|
|
|
92
190
|
snapCountdownChoice,
|
|
93
191
|
} from './lib/sleep.js'
|
|
94
192
|
|
|
193
|
+
/**
|
|
194
|
+
* THE PLUGIN DIRECTORY — where this very file lives.
|
|
195
|
+
*
|
|
196
|
+
* The rescue scripts (restore-profile.cmd, emergency-disable.cmd) sit next to
|
|
197
|
+
* it. The settings page shows the user that REAL path, so the path is taken
|
|
198
|
+
* from here and never guessed: nothing assumes `node_modules/dsh-autorestart`
|
|
199
|
+
* and nothing assumes a profile layout.
|
|
200
|
+
*/
|
|
201
|
+
const PLUGIN_DIR = path.dirname(fileURLToPath(import.meta.url))
|
|
202
|
+
|
|
95
203
|
/**
|
|
96
204
|
* The RESTART reserve: how long the host waits after the LAST conversation
|
|
97
205
|
* finished before it is really restarted.
|
|
@@ -125,6 +233,114 @@ const DEFAULT_RESTART_GRACE_MS = 0
|
|
|
125
233
|
*/
|
|
126
234
|
const LONG_WAIT_LOG_INTERVAL_MS = 60000
|
|
127
235
|
|
|
236
|
+
/**
|
|
237
|
+
* The marker the plugin writes when it has decided the restart ITSELF.
|
|
238
|
+
*
|
|
239
|
+
* The rollback watcher is the fallback for the case that the plugin dies
|
|
240
|
+
* with the host (a hosted restart kills the very process the plugin runs in).
|
|
241
|
+
* When the plugin survives the restart command, it decides alone — and writes
|
|
242
|
+
* this marker, so the watcher stands down instead of rolling back a restart
|
|
243
|
+
* that was fine.
|
|
244
|
+
*/
|
|
245
|
+
const RESTART_OUTCOME_FILE = 'restart-outcome.json'
|
|
246
|
+
|
|
247
|
+
/**
|
|
248
|
+
* The name of the rollback report of a restart that the plugin decided itself.
|
|
249
|
+
*
|
|
250
|
+
* It is the same file the detached watcher writes (`last-rollback.json`), so
|
|
251
|
+
* the interface shows the rollback either way.
|
|
252
|
+
*/
|
|
253
|
+
const RESTART_REPORT_NAME = ROLLBACK_REPORT_NAME
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* Resolve the connection service of the host, when it provides one.
|
|
257
|
+
*
|
|
258
|
+
* The connection service signs the access URL of the running host (the boot
|
|
259
|
+
* token). It is optional: a host without it still gets the plain origin.
|
|
260
|
+
*
|
|
261
|
+
* @param {any} ctx
|
|
262
|
+
* @returns {any|null}
|
|
263
|
+
*/
|
|
264
|
+
function resolveConnectionService(ctx) {
|
|
265
|
+
try {
|
|
266
|
+
if (ctx && ctx.connection && typeof ctx.connection.authenticatedUrl === 'function') return ctx.connection
|
|
267
|
+
} catch {}
|
|
268
|
+
try {
|
|
269
|
+
if (ctx && typeof ctx.get === 'function') {
|
|
270
|
+
const service = ctx.get('connection')
|
|
271
|
+
if (service && typeof service.authenticatedUrl === 'function') return service
|
|
272
|
+
}
|
|
273
|
+
} catch {}
|
|
274
|
+
return null
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* The access link of the running instance, or null.
|
|
279
|
+
*
|
|
280
|
+
* @param {any} runtime
|
|
281
|
+
* @returns {string|null}
|
|
282
|
+
*/
|
|
283
|
+
function accessUrlOf(runtime) {
|
|
284
|
+
try {
|
|
285
|
+
const record = runtime.accessRecord
|
|
286
|
+
if (record && typeof record.url === 'string' && record.url !== '') return record.url
|
|
287
|
+
} catch {}
|
|
288
|
+
return null
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/**
|
|
292
|
+
* Publish the instance record of the running host.
|
|
293
|
+
*
|
|
294
|
+
* The record carries three things that nobody else knows:
|
|
295
|
+
*
|
|
296
|
+
* - the PORT this instance listens on (the desktop app uses a random one,
|
|
297
|
+
* so the port of the previous instance is worthless);
|
|
298
|
+
* - the BOOT ID of this instance, so a watcher can tell "the new instance"
|
|
299
|
+
* from "the old one";
|
|
300
|
+
* - the ACCESS LINK with the boot token, so the user finds the working link
|
|
301
|
+
* after a restart instead of being locked out by a 401.
|
|
302
|
+
*
|
|
303
|
+
* The file is written with the mode 0600: it holds a secret.
|
|
304
|
+
*
|
|
305
|
+
* @param {any} ctx the plugin context
|
|
306
|
+
* @param {any} runtime the runtime of this plugin
|
|
307
|
+
* @param {object} config the normalized config
|
|
308
|
+
* @param {string} origin the origin of the web server (`protocol//host:port`)
|
|
309
|
+
* @returns {{ ok: boolean, path: string|null, url: string|null, error: string|null }}
|
|
310
|
+
*/
|
|
311
|
+
function publishInstanceRecord(ctx, runtime, config, origin) {
|
|
312
|
+
const stateDir = path.dirname(runtime.stateFile)
|
|
313
|
+
try {
|
|
314
|
+
const { url, token } = resolveAccessUrl({
|
|
315
|
+
origin,
|
|
316
|
+
connection: resolveConnectionService(ctx),
|
|
317
|
+
})
|
|
318
|
+
const written = writeAccessRecord({
|
|
319
|
+
stateDir,
|
|
320
|
+
bootId: runtime.bootId,
|
|
321
|
+
pid: process.pid,
|
|
322
|
+
port: runtime.webPort,
|
|
323
|
+
url,
|
|
324
|
+
token,
|
|
325
|
+
})
|
|
326
|
+
if (!written.ok) {
|
|
327
|
+
runtime.log.warn('could not publish the instance record:', written.error)
|
|
328
|
+
return { ok: false, path: written.path, url, error: written.error }
|
|
329
|
+
}
|
|
330
|
+
runtime.log.info('the instance record was published', {
|
|
331
|
+
path: written.path,
|
|
332
|
+
port: runtime.webPort,
|
|
333
|
+
// The token itself is a secret: the record carries it, the log only
|
|
334
|
+
// says that there is one.
|
|
335
|
+
hasToken: Boolean(token),
|
|
336
|
+
})
|
|
337
|
+
return { ok: true, path: written.path, url, error: null }
|
|
338
|
+
} catch (error) {
|
|
339
|
+
runtime.log.warn('could not publish the instance record:', String(error))
|
|
340
|
+
return { ok: false, path: null, url: null, error: String(error) }
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
|
|
128
344
|
/**
|
|
129
345
|
* Snap a reserve value onto the nearest offered choice.
|
|
130
346
|
*
|
|
@@ -158,7 +374,7 @@ function snapGraceChoice(value, fallback = DEFAULT_RESTART_GRACE_MS) {
|
|
|
158
374
|
* `pluginVersion()`), but a bundled installation may not ship that file — so
|
|
159
375
|
* the constant below is the last resort. A test keeps the two in sync.
|
|
160
376
|
*/
|
|
161
|
-
const PLUGIN_VERSION_FALLBACK = '0.
|
|
377
|
+
const PLUGIN_VERSION_FALLBACK = '0.8.0'
|
|
162
378
|
|
|
163
379
|
/**
|
|
164
380
|
* The version of the graceful-shutdown FEATURE, not the version of the package.
|
|
@@ -179,6 +395,14 @@ const PLUGIN_VERSION_FALLBACK = '0.6.2'
|
|
|
179
395
|
*/
|
|
180
396
|
const SLEEP_FEATURE_VERSION = '0.5.0'
|
|
181
397
|
|
|
398
|
+
/**
|
|
399
|
+
* The version of the self-starting supervisor ("开机自启守护").
|
|
400
|
+
*
|
|
401
|
+
* The one-time question about it is bound to this version: a release never
|
|
402
|
+
* brings an answered question back.
|
|
403
|
+
*/
|
|
404
|
+
const SUPERVISOR_FEATURE_VERSION = '0.7.0'
|
|
405
|
+
|
|
182
406
|
/** The cached version of this plugin (read once per process). */
|
|
183
407
|
let pluginVersionCache = null
|
|
184
408
|
|
|
@@ -226,6 +450,8 @@ function featureAnnouncements() {
|
|
|
226
450
|
return [
|
|
227
451
|
// The graceful shutdown ("收工关机") and its moon button.
|
|
228
452
|
{ feature: 'sleep', version: SLEEP_FEATURE_VERSION },
|
|
453
|
+
// The self-starting supervisor ("开机自启守护").
|
|
454
|
+
{ feature: 'supervisor', version: SUPERVISOR_FEATURE_VERSION },
|
|
229
455
|
]
|
|
230
456
|
}
|
|
231
457
|
|
|
@@ -442,6 +668,36 @@ function buildConfigSchema() {
|
|
|
442
668
|
.description('How many snapshots are kept; older ones are deleted automatically.'),
|
|
443
669
|
rollbackTimeoutMs: schemastery.number().min(1000).default(90000)
|
|
444
670
|
.description('Milliseconds the detached watcher waits for the relaunched host before it restores the snapshot.'),
|
|
671
|
+
// The watchdog: the process that waits for the relaunched host and puts
|
|
672
|
+
// the snapshot back when it never answers. It has to be triggered
|
|
673
|
+
// LATER, and the machine decides how (a one-off scheduled task, a
|
|
674
|
+
// systemd timer, or a detached process of our own).
|
|
675
|
+
watchdogEnabled: schemastery.boolean().default(true)
|
|
676
|
+
.description('Whether the watchdog may be armed before a restart: it waits for the new host and puts the snapshot back when it never answers. With it off there is no automatic rollback at all.'),
|
|
677
|
+
// The self-starting supervisor: a separate process that brings the host
|
|
678
|
+
// back when it dies. See the "开机自启" section of the settings.
|
|
679
|
+
supervisorMode: schemastery.union(['auto', 'installed', 'external', 'off']).default('auto')
|
|
680
|
+
.description('Who starts this host: auto = nobody decided yet, installed = the supervisor of this plugin, external = your own way, off = no automatic start.'),
|
|
681
|
+
supervisorKind: schemastery.union([
|
|
682
|
+
schemastery.const(''),
|
|
683
|
+
schemastery.const('windows-schtasks'),
|
|
684
|
+
schemastery.const('windows-startup'),
|
|
685
|
+
schemastery.const('systemd-user'),
|
|
686
|
+
schemastery.const('systemd-system'),
|
|
687
|
+
]).default('')
|
|
688
|
+
.description('Which form of the self-start record was installed (empty = the default of this platform).'),
|
|
689
|
+
supervisorRestartDelayMs: schemastery.number().min(0).default(3000)
|
|
690
|
+
.description('Milliseconds the supervisor waits before it starts the host again after it died.'),
|
|
691
|
+
supervisorRestartDelayMaxMs: schemastery.number().min(0).default(30000)
|
|
692
|
+
.description('The upper bound of that delay: the supervisor backs off further with every failed start, but never longer than this.'),
|
|
693
|
+
supervisorStartTimeoutMs: schemastery.number().min(1000).default(90000)
|
|
694
|
+
.description('Milliseconds the supervisor waits for the host to answer its health route before it restores the snapshot.'),
|
|
695
|
+
supervisorKillGraceMs: schemastery.number().min(0).default(15000)
|
|
696
|
+
.description('Milliseconds the supervisor waits between asking the host to stop and killing it.'),
|
|
697
|
+
supervisorPort: schemastery.number().min(0).default(0)
|
|
698
|
+
.description('The port the supervisor waits for to become free before it starts the host (0 = the port of the current instance).'),
|
|
699
|
+
supervisorAutoInstall: schemastery.boolean().default(false)
|
|
700
|
+
.description('Whether the one-time question may install the supervisor without asking again.'),
|
|
445
701
|
// graceful shutdown ("收工关机"): wait for every conversation, count down, then
|
|
446
702
|
// switch the REAL host machine off (a WSL host shuts Windows down, not the distro)
|
|
447
703
|
showSleepButton: schemastery.boolean().default(true)
|
|
@@ -499,7 +755,25 @@ const DEFAULTS = {
|
|
|
499
755
|
backupOnRestart: true,
|
|
500
756
|
backupIncludePlugins: true,
|
|
501
757
|
maxBackups: 2,
|
|
758
|
+
// The WATCHDOG: the process that waits for the relaunched host and puts the
|
|
759
|
+
// snapshot back when it never answers. It needs a way to be triggered
|
|
760
|
+
// later (a scheduled task, a systemd timer, a detached process) — the
|
|
761
|
+
// machine decides which one, and the settings page says which.
|
|
762
|
+
watchdogEnabled: true,
|
|
502
763
|
rollbackTimeoutMs: 90000,
|
|
764
|
+
// How long the restart command may run before the plugin stops waiting for
|
|
765
|
+
// it, and how long ONE restore step may take before the watcher gives up.
|
|
766
|
+
restartCommandTimeoutMs: DEFAULT_COMMAND_TIMEOUT_MS,
|
|
767
|
+
rollbackCopyTimeoutMs: DEFAULT_ROLLBACK_COPY_TIMEOUT_MS,
|
|
768
|
+
// The self-starting supervisor.
|
|
769
|
+
supervisorMode: 'auto',
|
|
770
|
+
supervisorKind: '',
|
|
771
|
+
supervisorRestartDelayMs: 3000,
|
|
772
|
+
supervisorRestartDelayMaxMs: 30000,
|
|
773
|
+
supervisorStartTimeoutMs: 90000,
|
|
774
|
+
supervisorKillGraceMs: 15000,
|
|
775
|
+
supervisorPort: 0,
|
|
776
|
+
supervisorAutoInstall: false,
|
|
503
777
|
// Graceful shutdown ("收工关机"): the button at the foot of the left sidebar,
|
|
504
778
|
// the countdown before the machine is switched off and the optional custom
|
|
505
779
|
// shutdown command (empty = the host is detected automatically).
|
|
@@ -530,6 +804,35 @@ function normalizeConfig(raw) {
|
|
|
530
804
|
const rollbackTimeout = Number(config.rollbackTimeoutMs)
|
|
531
805
|
config.rollbackTimeoutMs =
|
|
532
806
|
Number.isFinite(rollbackTimeout) && rollbackTimeout >= 1000 ? Math.trunc(rollbackTimeout) : 90000
|
|
807
|
+
// How long the restart command may run, and how long ONE restore step may
|
|
808
|
+
// take. Both are bounded, so neither a hung command nor a hung `robocopy`
|
|
809
|
+
// can hold the restart for ever.
|
|
810
|
+
const commandTimeout = Number(config.restartCommandTimeoutMs)
|
|
811
|
+
config.restartCommandTimeoutMs =
|
|
812
|
+
Number.isFinite(commandTimeout) && commandTimeout >= 1000 ? Math.trunc(commandTimeout) : DEFAULT_COMMAND_TIMEOUT_MS
|
|
813
|
+
const copyTimeout = Number(config.rollbackCopyTimeoutMs)
|
|
814
|
+
config.rollbackCopyTimeoutMs =
|
|
815
|
+
Number.isFinite(copyTimeout) && copyTimeout >= 1000 ? Math.trunc(copyTimeout) : DEFAULT_ROLLBACK_COPY_TIMEOUT_MS
|
|
816
|
+
// The supervisor knobs: a wrong value must not break the boot, it falls back
|
|
817
|
+
// to the documented default.
|
|
818
|
+
if (!['auto', 'installed', 'external', 'off'].includes(config.supervisorMode)) config.supervisorMode = 'auto'
|
|
819
|
+
if (!['', 'windows-schtasks', 'windows-startup', 'systemd-user', 'systemd-system'].includes(config.supervisorKind)) {
|
|
820
|
+
config.supervisorKind = ''
|
|
821
|
+
}
|
|
822
|
+
const supervisorDelay = Number(config.supervisorRestartDelayMs)
|
|
823
|
+
config.supervisorRestartDelayMs = Number.isFinite(supervisorDelay) && supervisorDelay >= 0 ? Math.trunc(supervisorDelay) : 3000
|
|
824
|
+
const supervisorDelayMax = Number(config.supervisorRestartDelayMaxMs)
|
|
825
|
+
config.supervisorRestartDelayMaxMs =
|
|
826
|
+
Number.isFinite(supervisorDelayMax) && supervisorDelayMax >= 0 ? Math.trunc(supervisorDelayMax) : 30000
|
|
827
|
+
const supervisorStartTimeout = Number(config.supervisorStartTimeoutMs)
|
|
828
|
+
config.supervisorStartTimeoutMs =
|
|
829
|
+
Number.isFinite(supervisorStartTimeout) && supervisorStartTimeout >= 1000 ? Math.trunc(supervisorStartTimeout) : 90000
|
|
830
|
+
const supervisorKillGrace = Number(config.supervisorKillGraceMs)
|
|
831
|
+
config.supervisorKillGraceMs =
|
|
832
|
+
Number.isFinite(supervisorKillGrace) && supervisorKillGrace >= 0 ? Math.trunc(supervisorKillGrace) : 15000
|
|
833
|
+
const supervisorPort = Number(config.supervisorPort)
|
|
834
|
+
config.supervisorPort = Number.isFinite(supervisorPort) && supervisorPort >= 0 ? Math.trunc(supervisorPort) : 0
|
|
835
|
+
if (typeof config.supervisorAutoInstall !== 'boolean') config.supervisorAutoInstall = false
|
|
533
836
|
// The countdown of the graceful shutdown is one of the offered choices: a
|
|
534
837
|
// profile that still carries an old value (15, 45, …) is snapped onto the
|
|
535
838
|
// nearest one instead of breaking, and 0 means "no countdown at all".
|
|
@@ -608,6 +911,21 @@ function resolveChoicePath(config) {
|
|
|
608
911
|
return `${prefix}/choice`
|
|
609
912
|
}
|
|
610
913
|
|
|
914
|
+
/**
|
|
915
|
+
* Resolve the supervisor-route path: a sibling of the health route, under the
|
|
916
|
+
* same prefix (`POST /supervisor` installs, uninstalls and reports the
|
|
917
|
+
* self-starting supervisor).
|
|
918
|
+
*
|
|
919
|
+
* @param {object} config
|
|
920
|
+
* @returns {string}
|
|
921
|
+
*/
|
|
922
|
+
function resolveSupervisorPath(config) {
|
|
923
|
+
const healthPath = resolveHealthPath(config)
|
|
924
|
+
const index = healthPath.lastIndexOf('/')
|
|
925
|
+
const prefix = index > 0 ? healthPath.slice(0, index) : '/__dsh_autorestart'
|
|
926
|
+
return `${prefix}/supervisor`
|
|
927
|
+
}
|
|
928
|
+
|
|
611
929
|
/**
|
|
612
930
|
* Resolve the sleep-route path: a sibling of the health route, under the same
|
|
613
931
|
* prefix (`GET /sleep` reports the state of the graceful sleep, `POST /sleep`
|
|
@@ -623,6 +941,20 @@ function resolveSleepPath(config) {
|
|
|
623
941
|
return `${prefix}/sleep`
|
|
624
942
|
}
|
|
625
943
|
|
|
944
|
+
/**
|
|
945
|
+
* Resolve the cancel-route path: a sibling of the health route, under the same
|
|
946
|
+
* prefix (`POST /cancel` withdraws ONE session's own restart request).
|
|
947
|
+
*
|
|
948
|
+
* @param {object} config
|
|
949
|
+
* @returns {string}
|
|
950
|
+
*/
|
|
951
|
+
function resolveCancelPath(config) {
|
|
952
|
+
const healthPath = resolveHealthPath(config)
|
|
953
|
+
const index = healthPath.lastIndexOf('/')
|
|
954
|
+
const prefix = index > 0 ? healthPath.slice(0, index) : '/__dsh_autorestart'
|
|
955
|
+
return `${prefix}/cancel`
|
|
956
|
+
}
|
|
957
|
+
|
|
626
958
|
/**
|
|
627
959
|
* Resolve the announcements-route path: a sibling of the health route, under
|
|
628
960
|
* the same prefix (`GET /announcements` reports the feature announcements the
|
|
@@ -1019,6 +1351,20 @@ function resolveProfileDir() {
|
|
|
1019
1351
|
return resolveProfileDirDetailed().dir
|
|
1020
1352
|
}
|
|
1021
1353
|
|
|
1354
|
+
/**
|
|
1355
|
+
* THE BACKUPS DIRECTORY.
|
|
1356
|
+
*
|
|
1357
|
+
* It is built in exactly ONE place: the snapshot writes here, and the
|
|
1358
|
+
* settings page reads here. Two places would mean two answers to "where do
|
|
1359
|
+
* the backups live".
|
|
1360
|
+
*
|
|
1361
|
+
* @param {string} dir the state directory
|
|
1362
|
+
* @returns {string}
|
|
1363
|
+
*/
|
|
1364
|
+
function backupsDirFor(dir) {
|
|
1365
|
+
return backupsRoot(typeof dir === 'string' ? dir : '')
|
|
1366
|
+
}
|
|
1367
|
+
|
|
1022
1368
|
/**
|
|
1023
1369
|
* The pre-restart snapshot: copy the profile into `<stateDir>/backups/<stamp>/`
|
|
1024
1370
|
* and clear the rollback report an older boot may have left behind.
|
|
@@ -1031,7 +1377,10 @@ function resolveProfileDir() {
|
|
|
1031
1377
|
* rollback possible at all.
|
|
1032
1378
|
*
|
|
1033
1379
|
* A failing backup never blocks the restart: the reason is logged and the
|
|
1034
|
-
* caller continues without a rollback point.
|
|
1380
|
+
* caller continues without a rollback point. It is NOT silent though: the
|
|
1381
|
+
* facts of the snapshot (was one asked for, was one taken, which directory,
|
|
1382
|
+
* how long it took, which older snapshots were pruned) travel with the answer,
|
|
1383
|
+
* so the restart record can tell the operator what the rollback net is.
|
|
1035
1384
|
*
|
|
1036
1385
|
* @param {{
|
|
1037
1386
|
* config?: object,
|
|
@@ -1040,7 +1389,14 @@ function resolveProfileDir() {
|
|
|
1040
1389
|
* profile?: { dir?: string|null, source?: string },
|
|
1041
1390
|
* createBackup?: Function,
|
|
1042
1391
|
* }} [options]
|
|
1043
|
-
* @returns {{
|
|
1392
|
+
* @returns {{
|
|
1393
|
+
* backup: object|null,
|
|
1394
|
+
* profileDir: string|null,
|
|
1395
|
+
* profile: { dir: string|null, source: string },
|
|
1396
|
+
* requested: boolean,
|
|
1397
|
+
* durationMs: number,
|
|
1398
|
+
* error: string|null,
|
|
1399
|
+
* }}
|
|
1044
1400
|
*/
|
|
1045
1401
|
export function takePreRestartSnapshot({
|
|
1046
1402
|
config = {},
|
|
@@ -1049,21 +1405,31 @@ export function takePreRestartSnapshot({
|
|
|
1049
1405
|
profile = null,
|
|
1050
1406
|
createBackup: createBackupFn = createBackup,
|
|
1051
1407
|
} = {}) {
|
|
1408
|
+
const startedAt = Date.now()
|
|
1052
1409
|
const info = profile && typeof profile === 'object'
|
|
1053
1410
|
? { dir: profile.dir ?? null, source: profile.source ?? 'unknown' }
|
|
1054
1411
|
: { dir: null, source: 'unknown' }
|
|
1055
1412
|
const profileDir = typeof info.dir === 'string' && info.dir.trim() !== '' ? info.dir : null
|
|
1056
1413
|
// The snapshot is optional: the operator can turn it off entirely.
|
|
1057
|
-
if (config.backupOnRestart !== true)
|
|
1414
|
+
if (config.backupOnRestart !== true) {
|
|
1415
|
+
return { backup: null, profileDir, profile: info, requested: false, durationMs: 0, error: null }
|
|
1416
|
+
}
|
|
1058
1417
|
if (profileDir === null) {
|
|
1059
1418
|
log.warn('no profile directory is known; skipping the pre-restart backup', { source: info.source })
|
|
1060
|
-
return {
|
|
1419
|
+
return {
|
|
1420
|
+
backup: null,
|
|
1421
|
+
profileDir: null,
|
|
1422
|
+
profile: info,
|
|
1423
|
+
requested: true,
|
|
1424
|
+
durationMs: Math.max(0, Date.now() - startedAt),
|
|
1425
|
+
error: 'no profile directory is known',
|
|
1426
|
+
}
|
|
1061
1427
|
}
|
|
1062
1428
|
let backup = null
|
|
1063
1429
|
try {
|
|
1064
1430
|
backup = createBackupFn({
|
|
1065
1431
|
profileDir,
|
|
1066
|
-
backupsDir:
|
|
1432
|
+
backupsDir: backupsDirFor(stateDir),
|
|
1067
1433
|
includePlugins: config.backupIncludePlugins !== false,
|
|
1068
1434
|
maxBackups: config.maxBackups,
|
|
1069
1435
|
})
|
|
@@ -1071,9 +1437,14 @@ export function takePreRestartSnapshot({
|
|
|
1071
1437
|
// A backup that throws must not take the restart down with it.
|
|
1072
1438
|
backup = { ok: false, error: String(error) }
|
|
1073
1439
|
}
|
|
1440
|
+
const durationMs = Math.max(0, Date.now() - startedAt)
|
|
1074
1441
|
if (backup && backup.ok === true) {
|
|
1075
1442
|
log.warn('pre-restart backup written', {
|
|
1076
1443
|
dir: backup.dir,
|
|
1444
|
+
stamp: backup.stamp,
|
|
1445
|
+
at: backup.at,
|
|
1446
|
+
atLocal: backup.atLocal,
|
|
1447
|
+
durationMs,
|
|
1077
1448
|
bytes: backup.bytes,
|
|
1078
1449
|
includePlugins: backup.includePlugins,
|
|
1079
1450
|
pruned: backup.pruned,
|
|
@@ -1086,7 +1457,14 @@ export function takePreRestartSnapshot({
|
|
|
1086
1457
|
error: backup && backup.error ? backup.error : 'unknown error',
|
|
1087
1458
|
})
|
|
1088
1459
|
}
|
|
1089
|
-
return {
|
|
1460
|
+
return {
|
|
1461
|
+
backup,
|
|
1462
|
+
profileDir,
|
|
1463
|
+
profile: info,
|
|
1464
|
+
requested: true,
|
|
1465
|
+
durationMs,
|
|
1466
|
+
error: backup && backup.ok === true ? null : (backup && backup.error ? backup.error : 'unknown error'),
|
|
1467
|
+
}
|
|
1090
1468
|
}
|
|
1091
1469
|
|
|
1092
1470
|
/** A logger that never throws and never requires a particular Cordis logger shape. */
|
|
@@ -1200,7 +1578,9 @@ function findOwnEntry(editor) {
|
|
|
1200
1578
|
* snapshot?: Function,
|
|
1201
1579
|
* runCommand?: Function,
|
|
1202
1580
|
* scheduleDetached?: Function,
|
|
1203
|
-
*
|
|
1581
|
+
* launchWatcher?: Function,
|
|
1582
|
+
* probeInstance?: Function,
|
|
1583
|
+
* }} [options] `env` replaces parts of the detected environment and the
|
|
1204
1584
|
* functions replace the side-effecting primitives of the restart path. They
|
|
1205
1585
|
* exist so a restart can be exercised without spawning a process or touching
|
|
1206
1586
|
* the real profile; production never passes them.
|
|
@@ -1217,6 +1597,23 @@ export function createRuntime(ctx, config, options = {}) {
|
|
|
1217
1597
|
const scheduleDetachedFn = typeof options.scheduleDetached === 'function'
|
|
1218
1598
|
? options.scheduleDetached
|
|
1219
1599
|
: scheduleDetachedRestart
|
|
1600
|
+
// The rollback watcher is started OUTSIDE the host process, so it survives the
|
|
1601
|
+
// death of the host — which is what a hosted restart does.
|
|
1602
|
+
const launchWatcherFn = typeof options.launchWatcher === 'function'
|
|
1603
|
+
? options.launchWatcher
|
|
1604
|
+
: launchDetachedWatcher
|
|
1605
|
+
// The health confirmation of the new instance (in-process; the detached
|
|
1606
|
+
// watcher has its own, outside the host).
|
|
1607
|
+
const probeInstanceFn = typeof options.probeInstance === 'function'
|
|
1608
|
+
? options.probeInstance
|
|
1609
|
+
: waitForHealthyInstance
|
|
1610
|
+
// The NEW watchdog (v2). It is injected so a test can decide who is the
|
|
1611
|
+
// arbiter of the restart WITHOUT registering anything on the machine.
|
|
1612
|
+
// (The fallback is a wrapper: `armWatchdog` is declared further down, so it
|
|
1613
|
+
// must not be READ here.)
|
|
1614
|
+
const armWatchdogFn = typeof options.armWatchdog === 'function'
|
|
1615
|
+
? options.armWatchdog
|
|
1616
|
+
: (...args) => armWatchdog(...args)
|
|
1220
1617
|
const stateFile = resolveStateFile(config)
|
|
1221
1618
|
|
|
1222
1619
|
const loaded = readState(stateFile, Date.now())
|
|
@@ -1230,6 +1627,20 @@ export function createRuntime(ctx, config, options = {}) {
|
|
|
1230
1627
|
state = boot.state
|
|
1231
1628
|
const bootId = state.bootId
|
|
1232
1629
|
|
|
1630
|
+
// THE INSTANCE RECORD exists from the first moment of this boot: it carries
|
|
1631
|
+
// the boot id (which tells the rollback watcher that this instance is the
|
|
1632
|
+
// one being replaced). As soon as the web server is known, the record is
|
|
1633
|
+
// written again — with the port of this instance and its access link.
|
|
1634
|
+
try {
|
|
1635
|
+
writeAccessRecord({
|
|
1636
|
+
stateDir: path.dirname(stateFile),
|
|
1637
|
+
bootId,
|
|
1638
|
+
pid: process.pid,
|
|
1639
|
+
})
|
|
1640
|
+
} catch (error) {
|
|
1641
|
+
log.warn('could not write the instance record:', String(error))
|
|
1642
|
+
}
|
|
1643
|
+
|
|
1233
1644
|
let pollTimer
|
|
1234
1645
|
let recoverTimer
|
|
1235
1646
|
let disposed = false
|
|
@@ -1239,6 +1650,12 @@ export function createRuntime(ctx, config, options = {}) {
|
|
|
1239
1650
|
* logged, never thrown: the plugin still loads and the other tools still work.
|
|
1240
1651
|
*/
|
|
1241
1652
|
const toolErrors = []
|
|
1653
|
+
/**
|
|
1654
|
+
* The routes of this plugin that the web server accepted, and the ones it
|
|
1655
|
+
* refused. The boot check of the restart record reports them.
|
|
1656
|
+
*/
|
|
1657
|
+
const registeredRoutes = []
|
|
1658
|
+
const routeErrors = []
|
|
1242
1659
|
|
|
1243
1660
|
const persist = () => {
|
|
1244
1661
|
try {
|
|
@@ -2087,20 +2504,31 @@ const titleOfSummary = (item) => {
|
|
|
2087
2504
|
}
|
|
2088
2505
|
|
|
2089
2506
|
/**
|
|
2090
|
-
* Release the continuation request of
|
|
2507
|
+
* Release the continuation request of the sessions that asked for the current
|
|
2091
2508
|
* restart (cancel / give-up paths). Requests are coalesced into ONE restart, so
|
|
2092
2509
|
* releasing only the last requester would leave the others pinned as pending.
|
|
2510
|
+
*
|
|
2511
|
+
* `only` releases EXACTLY those sessions — that is how one session withdraws
|
|
2512
|
+
* its own request without touching the requests of the other sessions.
|
|
2513
|
+
*
|
|
2514
|
+
* @param {string[]|null} [only] the sessions to release; null = every requester
|
|
2093
2515
|
*/
|
|
2094
|
-
const releaseRequesters = () => {
|
|
2095
|
-
const restart = state.restart
|
|
2096
|
-
if (!restart) return
|
|
2516
|
+
const releaseRequesters = (only = null) => {
|
|
2097
2517
|
const ids = new Set()
|
|
2098
|
-
if (Array.isArray(
|
|
2099
|
-
for (const
|
|
2100
|
-
if (
|
|
2518
|
+
if (Array.isArray(only)) {
|
|
2519
|
+
for (const sessionId of only) {
|
|
2520
|
+
if (typeof sessionId === 'string' && sessionId) ids.add(sessionId)
|
|
2521
|
+
}
|
|
2522
|
+
} else {
|
|
2523
|
+
const restart = state.restart
|
|
2524
|
+
if (!restart) return
|
|
2525
|
+
if (Array.isArray(restart.requesters)) {
|
|
2526
|
+
for (const requester of restart.requesters) {
|
|
2527
|
+
if (requester && typeof requester.sessionId === 'string' && requester.sessionId) ids.add(requester.sessionId)
|
|
2528
|
+
}
|
|
2101
2529
|
}
|
|
2530
|
+
if (typeof restart.bySessionId === 'string' && restart.bySessionId) ids.add(restart.bySessionId)
|
|
2102
2531
|
}
|
|
2103
|
-
if (typeof restart.bySessionId === 'string' && restart.bySessionId) ids.add(restart.bySessionId)
|
|
2104
2532
|
for (const sessionId of ids) {
|
|
2105
2533
|
const entry = state.sessions ? state.sessions[sessionId] : undefined
|
|
2106
2534
|
if (!entry) continue
|
|
@@ -2171,134 +2599,1242 @@ const titleOfSummary = (item) => {
|
|
|
2171
2599
|
return 'http://127.0.0.1:' + port + resolveHealthPath(config)
|
|
2172
2600
|
}
|
|
2173
2601
|
|
|
2174
|
-
/**
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
|
|
2602
|
+
/**
|
|
2603
|
+
* The state directory of this plugin (the checkpoint, the backups, the
|
|
2604
|
+
* instance record and the helper scripts live next to each other).
|
|
2605
|
+
*/
|
|
2606
|
+
const stateDirOf = () => path.dirname(stateFile)
|
|
2607
|
+
|
|
2608
|
+
/**
|
|
2609
|
+
* The durable record of the last restart: the snapshot that was taken, the
|
|
2610
|
+
* verdict of the new instance and its four boot checks.
|
|
2611
|
+
*
|
|
2612
|
+
* @returns {object|null} the summary (null when no restart happened yet)
|
|
2613
|
+
*/
|
|
2614
|
+
const lastRestartVerdict = () => summarizeRestartVerdict(readRestartVerdict(stateDirOf()))
|
|
2615
|
+
|
|
2616
|
+
/**
|
|
2617
|
+
* The version of THIS plugin (the one the user runs).
|
|
2618
|
+
*
|
|
2619
|
+
* @returns {string}
|
|
2620
|
+
*/
|
|
2621
|
+
const pluginVersion = () => {
|
|
2178
2622
|
try {
|
|
2179
|
-
|
|
2180
|
-
|
|
2181
|
-
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2623
|
+
const raw = fs.readFileSync(new URL('../package.json', import.meta.url), 'utf8')
|
|
2624
|
+
const parsed = JSON.parse(raw)
|
|
2625
|
+
return typeof parsed.version === 'string' && parsed.version !== '' ? parsed.version : 'unknown'
|
|
2626
|
+
} catch {
|
|
2627
|
+
return 'unknown'
|
|
2628
|
+
}
|
|
2629
|
+
}
|
|
2630
|
+
|
|
2631
|
+
/**
|
|
2632
|
+
* Write (or update) the durable record of the restart.
|
|
2633
|
+
*
|
|
2634
|
+
* The record is written by BOTH halves of a restart: the instance that is
|
|
2635
|
+
* replaced writes the `pending` half (with the snapshot), the instance that
|
|
2636
|
+
* replaces it writes the final verdict. `skipIfFinalized` keeps the older
|
|
2637
|
+
* instance from overwriting a verdict that the new one already wrote.
|
|
2638
|
+
*
|
|
2639
|
+
* @param {object} [patch] the fields to change
|
|
2640
|
+
* @param {{ skipIfFinalized?: boolean }} [options]
|
|
2641
|
+
* @returns {object|null} the record that is on disk now
|
|
2642
|
+
*/
|
|
2643
|
+
const recordRestartVerdict = (patch = {}, options = {}) => {
|
|
2644
|
+
const dir = stateDirOf()
|
|
2645
|
+
let current = null
|
|
2188
2646
|
try {
|
|
2189
|
-
|
|
2190
|
-
} catch
|
|
2191
|
-
|
|
2647
|
+
current = readRestartVerdict(dir)
|
|
2648
|
+
} catch {
|
|
2649
|
+
current = null
|
|
2192
2650
|
}
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
quietForMs: verdict.quietForMs,
|
|
2199
|
-
})
|
|
2200
|
-
persist()
|
|
2201
|
-
startPolling()
|
|
2202
|
-
return
|
|
2651
|
+
if (options.skipIfFinalized === true
|
|
2652
|
+
&& current !== null
|
|
2653
|
+
&& current.bootId !== null
|
|
2654
|
+
&& current.result !== RESTART_RESULTS.PENDING) {
|
|
2655
|
+
return current
|
|
2203
2656
|
}
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
// How the host is restarted: the custom command, or the desktop app.
|
|
2210
|
-
const useCustomCommand = config.restartMode === 'command' && config.restartCommand.trim() !== ''
|
|
2211
|
-
const canRelaunchDesktop = Boolean(env.isDesktopApp && env.appExePath)
|
|
2212
|
-
if (!useCustomCommand && !canRelaunchDesktop) {
|
|
2213
|
-
// No restart method at all (the host is not the desktop app and there is
|
|
2214
|
-
// no custom command): nothing is restarted, so nothing is snapshotted
|
|
2215
|
-
// either — there is no rollback point for a restart that never happens.
|
|
2216
|
-
log.warn('no restart method available (not the desktop app and restartCommand is empty); leaving the checkpoint in place')
|
|
2217
|
-
rollbackRestart('no restart method available')
|
|
2218
|
-
return
|
|
2219
|
-
}
|
|
2657
|
+
const merged = buildRestartVerdict({ ...(current ?? {}), ...patch })
|
|
2658
|
+
const written = writeRestartVerdict(dir, merged)
|
|
2659
|
+
if (!written.ok) log.warn('could not write the restart record:', written.error)
|
|
2660
|
+
return merged
|
|
2661
|
+
}
|
|
2220
2662
|
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2663
|
+
/**
|
|
2664
|
+
* WHICH MACHINE IS THIS?
|
|
2665
|
+
*
|
|
2666
|
+
* The same answer the graceful shutdown uses (`lib/host-kind.js`): Windows,
|
|
2667
|
+
* WSL, a real Linux, a macOS. It matters for the autostart: inside WSL the
|
|
2668
|
+
* "start at login" is a different thing (a systemd user unit + the linger,
|
|
2669
|
+
* or a task on the Windows side), and the Docker/Compose answer of a
|
|
2670
|
+
* container is simply wrong there.
|
|
2671
|
+
*
|
|
2672
|
+
* @returns {object} the host (kind, evidence, suspect, distro)
|
|
2673
|
+
*/
|
|
2674
|
+
const hostKindOf = () => detectHostKind({ platform: env.platform, env: process.env })
|
|
2233
2675
|
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
|
|
2246
|
-
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
const watchScript = watchScriptPath === null
|
|
2261
|
-
? null
|
|
2262
|
-
: buildWatchScript({
|
|
2263
|
-
healthUrl,
|
|
2264
|
-
backupDir: backup.dir,
|
|
2265
|
-
profileDir,
|
|
2266
|
-
reportPath: path.join(stateDir, ROLLBACK_REPORT_NAME),
|
|
2267
|
-
appExePath: env.appExePath,
|
|
2268
|
-
timeoutMs: config.rollbackTimeoutMs,
|
|
2269
|
-
// Only a crash log written after this point belongs to this restart.
|
|
2270
|
-
since: new Date().toISOString(),
|
|
2271
|
-
})
|
|
2272
|
-
const result = scheduleDetachedFn({
|
|
2273
|
-
appExePath: env.appExePath,
|
|
2274
|
-
relaunchDelaySeconds: config.relaunchDelaySeconds,
|
|
2275
|
-
gateUrl: url,
|
|
2276
|
-
scriptPath: url === null ? null : path.join(stateDir, 'restart-' + state.generation + '.cmd'),
|
|
2277
|
-
watchScriptPath,
|
|
2278
|
-
watchScript,
|
|
2279
|
-
})
|
|
2280
|
-
log.warn('scheduled detached restart:', result)
|
|
2281
|
-
if (!result || result.ok !== true) rollbackRestart((result && result.error) || 'detached restart refused')
|
|
2282
|
-
else armGateWatchdog()
|
|
2283
|
-
}
|
|
2284
|
-
} catch (error) {
|
|
2285
|
-
rollbackRestart(String(error))
|
|
2286
|
-
}
|
|
2676
|
+
/**
|
|
2677
|
+
* THE PLATFORM GUIDE: how "start at login" works on THIS machine.
|
|
2678
|
+
*
|
|
2679
|
+
* Windows (a scheduled task, with the startup folder as the fallback),
|
|
2680
|
+
* Linux (a systemd user unit + the linger hint), WSL (the same, or the two
|
|
2681
|
+
* ways out when there is no systemd), a container (the orchestrator) and the
|
|
2682
|
+
* unknown host (nothing, and no guessing). The answer is computed ONCE
|
|
2683
|
+
* (the machine does not change while the host runs) and the settings page
|
|
2684
|
+
* reads it.
|
|
2685
|
+
*
|
|
2686
|
+
* @returns {object} the advice (kind, reason, notes, commands, options)
|
|
2687
|
+
*/
|
|
2688
|
+
let cachedAutostart = null
|
|
2689
|
+
const autostartAdviceOf = () => {
|
|
2690
|
+
if (cachedAutostart !== null) return cachedAutostart
|
|
2691
|
+
const host = hostKindOf()
|
|
2692
|
+
const init = detectInitSystems({ platform: env.platform, homeDir: os.homedir() })
|
|
2693
|
+
cachedAutostart = planAutostartAdvice({
|
|
2694
|
+
host,
|
|
2695
|
+
platform: env.platform,
|
|
2696
|
+
hasSystemd: init.systemdUser,
|
|
2697
|
+
container: detectContainer({ env: process.env }).container,
|
|
2698
|
+
user: typeof process.env.USER === 'string' ? process.env.USER : '',
|
|
2699
|
+
distro: host.distro,
|
|
2700
|
+
})
|
|
2701
|
+
return cachedAutostart
|
|
2287
2702
|
}
|
|
2288
2703
|
|
|
2289
|
-
/**
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
|
|
2293
|
-
|
|
2294
|
-
|
|
2704
|
+
/**
|
|
2705
|
+
* WHAT IS ON DISK RIGHT NOW: the backups of this host.
|
|
2706
|
+
*
|
|
2707
|
+
* The settings page shows it ("what is backed up right now"), so the facts
|
|
2708
|
+
* have to come from the disk and not from the browser: the browser may not
|
|
2709
|
+
* even see the disk. Only the DIRECTORY is read — the manifest of every
|
|
2710
|
+
* copy — because one copy can be 150 MB and its contents are never walked.
|
|
2711
|
+
*
|
|
2712
|
+
* @returns {{ dir: string, max: number, count: number, items: Array<object> }}
|
|
2713
|
+
*/
|
|
2714
|
+
const backupsInfo = () => {
|
|
2715
|
+
const dir = backupsDirFor(stateDirOf())
|
|
2716
|
+
const items = listBackups(dir).map((item) => ({
|
|
2717
|
+
name: item.name,
|
|
2718
|
+
dir: item.dir,
|
|
2719
|
+
at: item.at,
|
|
2720
|
+
atLocal: item.atLocal,
|
|
2721
|
+
bytes: item.bytes,
|
|
2722
|
+
files: item.files,
|
|
2723
|
+
includePlugins: item.includePlugins,
|
|
2724
|
+
}))
|
|
2725
|
+
return {
|
|
2726
|
+
dir,
|
|
2727
|
+
max: config.maxBackups,
|
|
2728
|
+
count: items.length,
|
|
2729
|
+
items,
|
|
2295
2730
|
}
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2731
|
+
}
|
|
2732
|
+
|
|
2733
|
+
/**
|
|
2734
|
+
* WHERE THE RESCUE SCRIPTS ARE.
|
|
2735
|
+
*
|
|
2736
|
+
* The rescue kit is for everybody who installed the plugin: when DSH does
|
|
2737
|
+
* not come up, the user cannot read a README — they need the path, and they
|
|
2738
|
+
* need to be able to copy it. The browser cannot read the disk, so the Host
|
|
2739
|
+
* resolves it: the plugin directory is where THIS file lives, and the
|
|
2740
|
+
* scripts sit next to it.
|
|
2741
|
+
*
|
|
2742
|
+
* @returns {object}
|
|
2743
|
+
*/
|
|
2744
|
+
const rescueKitInfo = () => rescueInfo({
|
|
2745
|
+
pluginDir: PLUGIN_DIR,
|
|
2746
|
+
env: process.env,
|
|
2747
|
+
platform: env.platform,
|
|
2748
|
+
exists: (target) => fs.existsSync(target),
|
|
2749
|
+
})
|
|
2750
|
+
|
|
2751
|
+
/**
|
|
2752
|
+
* THE CAPABILITIES of this machine: can it create a one-off scheduled
|
|
2753
|
+
* task?
|
|
2754
|
+
*
|
|
2755
|
+
* The automatic rollback hangs on that answer: the host is restarted and a
|
|
2756
|
+
* task that fires a minute later brings it back if it did not come up. On a
|
|
2757
|
+
* machine where such a task cannot be created the safety net is silently
|
|
2758
|
+
* missing — so the user is told what they lose.
|
|
2759
|
+
*
|
|
2760
|
+
* The answer is checked ONCE — the first time the plugin runs here — and
|
|
2761
|
+
* then kept in a file of its own next to the state. It is NOT checked when
|
|
2762
|
+
* the settings page opens: that would register a task every time somebody
|
|
2763
|
+
* looks at the page.
|
|
2764
|
+
*
|
|
2765
|
+
* A failure of the probe is NEVER a failure of the plugin: the plugin
|
|
2766
|
+
* works, only the safety net may be missing.
|
|
2767
|
+
*
|
|
2768
|
+
* @returns {object} the capabilities record
|
|
2769
|
+
*/
|
|
2770
|
+
const capabilitiesInfo = () => {
|
|
2771
|
+
const file = path.join(stateDirOf(), 'capabilities.json')
|
|
2772
|
+
const stored = readCapabilitiesRecord(file)
|
|
2773
|
+
if (stored) return stored
|
|
2774
|
+
let checked
|
|
2775
|
+
try {
|
|
2776
|
+
checked = probeSchedulerCapability({ platform: env.platform })
|
|
2777
|
+
} catch (error) {
|
|
2778
|
+
checked = { scheduler: SCHEDULER_UNKNOWN, checkedAt: Date.now(), evidence: { error: String(error) } }
|
|
2779
|
+
}
|
|
2780
|
+
// The tier is decided by ONE function (lib/watchdog-tier.js), so the
|
|
2781
|
+
// warning in the settings page and the watchdog that is really registered
|
|
2782
|
+
// cannot drift apart.
|
|
2783
|
+
let picked = { tier: TIERS.NONE, reason: 'this machine was not checked yet' }
|
|
2784
|
+
try {
|
|
2785
|
+
const caps = detectWatchdogCapabilities({ platform: env.platform })
|
|
2786
|
+
picked = tierWithProof({
|
|
2787
|
+
platform: env.platform,
|
|
2788
|
+
probeOk: checked.scheduler === 'ok',
|
|
2789
|
+
hasSystemdRun: caps.hasSystemdRun,
|
|
2790
|
+
hasAt: caps.hasAt,
|
|
2791
|
+
canDetach: caps.canDetach,
|
|
2792
|
+
})
|
|
2793
|
+
} catch (error) {
|
|
2794
|
+
picked = { tier: TIERS.NONE, reason: 'the machine could not be checked: ' + String(error) }
|
|
2795
|
+
}
|
|
2796
|
+
const record = { ...emptyCapabilities(), ...checked, tier: picked.tier, tierReason: picked.reason }
|
|
2797
|
+
// The warning is only for the TRANSITION into blocked: a machine that was
|
|
2798
|
+
// blocked and stays blocked does not nag.
|
|
2799
|
+
if (shouldWarnScheduler(emptyCapabilities(), record) && !tierProvidesRollback(picked.tier)) {
|
|
2800
|
+
runtime.log.warn('this machine cannot run the watchdog: the automatic rollback is not available')
|
|
2801
|
+
}
|
|
2802
|
+
writeCapabilitiesRecord(file, record)
|
|
2803
|
+
return record
|
|
2804
|
+
}
|
|
2805
|
+
|
|
2806
|
+
/**
|
|
2807
|
+
* THE WATCHDOG: which tier this machine uses, and what the last run did.
|
|
2808
|
+
*
|
|
2809
|
+
* @returns {{ enabled: boolean, tier: string, tierReason: string, available: boolean, report: object|null }}
|
|
2810
|
+
*/
|
|
2811
|
+
const watchdogInfo = () => {
|
|
2812
|
+
const capabilities = capabilitiesInfo()
|
|
2813
|
+
const tier = typeof capabilities.tier === 'string' && capabilities.tier !== ''
|
|
2814
|
+
? capabilities.tier
|
|
2815
|
+
: TIERS.NONE
|
|
2816
|
+
const report = readReport(stateDirOf())
|
|
2817
|
+
return {
|
|
2818
|
+
enabled: config.watchdogEnabled !== false,
|
|
2819
|
+
tier,
|
|
2820
|
+
tierReason: typeof capabilities.tierReason === 'string' ? capabilities.tierReason : '',
|
|
2821
|
+
available: tierProvidesRollback(tier),
|
|
2822
|
+
report,
|
|
2823
|
+
}
|
|
2824
|
+
}
|
|
2825
|
+
|
|
2826
|
+
/**
|
|
2827
|
+
* The capabilities with the warning marked as seen (the user read it).
|
|
2828
|
+
*
|
|
2829
|
+
* @returns {object}
|
|
2830
|
+
*/
|
|
2831
|
+
const acknowledgeSchedulerWarning = () => {
|
|
2832
|
+
const file = path.join(stateDirOf(), 'capabilities.json')
|
|
2833
|
+
const stored = readCapabilitiesRecord(file) || { ...emptyCapabilities() }
|
|
2834
|
+
const marked = markSchedulerWarned(stored)
|
|
2835
|
+
writeCapabilitiesRecord(file, marked)
|
|
2836
|
+
return marked
|
|
2837
|
+
}
|
|
2838
|
+
|
|
2839
|
+
/**
|
|
2840
|
+
* ARM THE WATCHDOG: the one-off thing that waits for the relaunched host.
|
|
2841
|
+
*
|
|
2842
|
+
* The tier is decided by ONE function (lib/watchdog-tier.js) — the same one
|
|
2843
|
+
* that the settings page and the diagnostics ask — so the warning and the
|
|
2844
|
+
* behaviour cannot drift apart. When no tier is available NOTHING is
|
|
2845
|
+
* registered: there is no watchdog, and the report says so.
|
|
2846
|
+
*
|
|
2847
|
+
* @param {{
|
|
2848
|
+
* snapshotDir?: string,
|
|
2849
|
+
* profileDir?: string|null,
|
|
2850
|
+
* relaunchCommand?: object|null,
|
|
2851
|
+
* healthUrl?: string,
|
|
2852
|
+
* portFile?: string,
|
|
2853
|
+
* expectedBootId?: string|null,
|
|
2854
|
+
* }} options
|
|
2855
|
+
* @returns {{ ok: boolean, tier: string, reason: string, error: string|null }}
|
|
2856
|
+
*/
|
|
2857
|
+
const armWatchdog = ({
|
|
2858
|
+
snapshotDir = '',
|
|
2859
|
+
profileDir = null,
|
|
2860
|
+
relaunchCommand = null,
|
|
2861
|
+
healthUrl = '',
|
|
2862
|
+
portFile = '',
|
|
2863
|
+
expectedBootId = null,
|
|
2864
|
+
} = {}) => {
|
|
2865
|
+
const stateDir = stateDirOf()
|
|
2866
|
+
if (config.watchdogEnabled === false) {
|
|
2867
|
+
return { ok: false, tier: TIERS.NONE, reason: 'the watchdog is switched off', error: null }
|
|
2868
|
+
}
|
|
2869
|
+
if (profileDir === null || snapshotDir === '') {
|
|
2870
|
+
return { ok: false, tier: TIERS.NONE, reason: 'there is no snapshot to watch', error: null }
|
|
2871
|
+
}
|
|
2872
|
+
const info = watchdogInfo()
|
|
2873
|
+
if (info.tier === TIERS.NONE) {
|
|
2874
|
+
return { ok: false, tier: TIERS.NONE, reason: info.tierReason || 'this machine cannot run the watchdog', error: null }
|
|
2875
|
+
}
|
|
2876
|
+
const timeoutMs = Number(config.rollbackTimeoutMs) > 0 ? Number(config.rollbackTimeoutMs) : 180000
|
|
2877
|
+
// ONE name for the scheduling item: a new registration replaces the old
|
|
2878
|
+
// one, and there is never more than one of them on the machine.
|
|
2879
|
+
const taskName = 'dsh-autorestart-watchdog'
|
|
2880
|
+
const unitId = 'dsh-autorestart-watchdog'
|
|
2881
|
+
const pidPath = detachedPidPath(stateDir)
|
|
2882
|
+
const watchdogCommand = '"' + process.execPath + '" "' + path.join(PLUGIN_DIR, 'scripts', 'watchdog.mjs') + '" --state-dir "' + stateDir + '"'
|
|
2883
|
+
const pending = buildPending({
|
|
2884
|
+
tier: info.tier,
|
|
2885
|
+
tierReason: info.tierReason,
|
|
2886
|
+
taskName,
|
|
2887
|
+
unitId,
|
|
2888
|
+
pidPath,
|
|
2889
|
+
snapshotDir,
|
|
2890
|
+
profileDir,
|
|
2891
|
+
relaunchCommand,
|
|
2892
|
+
healthUrl,
|
|
2893
|
+
healthPath: resolveHealthPath(config),
|
|
2894
|
+
portFile,
|
|
2895
|
+
expectedBootId,
|
|
2896
|
+
timeoutMs,
|
|
2897
|
+
recheckMs: 20000,
|
|
2898
|
+
readyTimeoutMs: 90000,
|
|
2899
|
+
hostKind: hostKindOf().kind,
|
|
2900
|
+
now: Date.now(),
|
|
2901
|
+
})
|
|
2902
|
+
writePending(stateDir, pending)
|
|
2903
|
+
|
|
2904
|
+
let registered = { ok: false, error: 'the watchdog was not registered' }
|
|
2905
|
+
try {
|
|
2906
|
+
if (info.tier === TIERS.SCHEDULED_TASK) {
|
|
2907
|
+
const start = new Date(Date.now() + Math.ceil(timeoutMs / 1000) * 1000);
|
|
2908
|
+
const end = new Date(start.getTime() + 30 * 60 * 1000);
|
|
2909
|
+
registered = registerScheduledTask({
|
|
2910
|
+
taskName,
|
|
2911
|
+
startIso: localIso(start),
|
|
2912
|
+
endIso: localIso(end),
|
|
2913
|
+
command: watchdogCommand,
|
|
2914
|
+
xmlPath: path.join(stateDir, 'watchdog-task.xml'),
|
|
2915
|
+
});
|
|
2916
|
+
} else if (info.tier === TIERS.SYSTEMD_RUN) {
|
|
2917
|
+
// The unit of an earlier run must be gone before a new one can take
|
|
2918
|
+
// the name.
|
|
2919
|
+
cleanupSystemdUnit({ unitId });
|
|
2920
|
+
registered = registerSystemdRun({
|
|
2921
|
+
unitId,
|
|
2922
|
+
delaySeconds: Math.ceil(timeoutMs / 1000),
|
|
2923
|
+
command: [process.execPath, path.join(PLUGIN_DIR, 'scripts', 'watchdog.mjs'), '--state-dir', stateDir],
|
|
2924
|
+
});
|
|
2925
|
+
} else if (info.tier === TIERS.DETACHED_PROCESS) {
|
|
2926
|
+
registered = registerDetachedProcess({
|
|
2927
|
+
delaySeconds: Math.ceil(timeoutMs / 1000),
|
|
2928
|
+
command: process.execPath + ' "' + path.join(PLUGIN_DIR, 'scripts', 'watchdog.mjs') + '" --state-dir "' + stateDir + '"',
|
|
2929
|
+
pidPath,
|
|
2930
|
+
});
|
|
2931
|
+
}
|
|
2932
|
+
} catch (error) {
|
|
2933
|
+
registered = { ok: false, error: error && error.message ? error.message : String(error) };
|
|
2934
|
+
}
|
|
2935
|
+
if (!registered || registered.ok !== true) {
|
|
2936
|
+
// Nothing half-armed: the marker goes, and so does the scheduling item.
|
|
2937
|
+
clearPending(stateDir);
|
|
2938
|
+
try {
|
|
2939
|
+
if (info.tier === TIERS.SCHEDULED_TASK) removeScheduledTask({ taskName });
|
|
2940
|
+
else if (info.tier === TIERS.SYSTEMD_RUN) cleanupSystemdUnit({ unitId });
|
|
2941
|
+
} catch {
|
|
2942
|
+
// The next run cleans up.
|
|
2943
|
+
}
|
|
2944
|
+
const reason = (registered && registered.error) || 'the watchdog could not be registered';
|
|
2945
|
+
log.warn('the watchdog was not armed: ' + reason);
|
|
2946
|
+
return { ok: false, tier: info.tier, reason: info.tierReason, error: reason };
|
|
2947
|
+
}
|
|
2948
|
+
log.warn('the watchdog is armed (' + info.tier + '): ' + info.tierReason);
|
|
2949
|
+
return { ok: true, tier: info.tier, reason: info.tierReason, error: null };
|
|
2950
|
+
}
|
|
2951
|
+
|
|
2952
|
+
/**
|
|
2953
|
+
* THE DIAGNOSTICS of "report a problem".
|
|
2954
|
+
*
|
|
2955
|
+
* One generator, on the host: the browser copies the text it gets, so the
|
|
2956
|
+
* two can never drift. It carries the facts a bug report needs — and NOT
|
|
2957
|
+
* the access token of this instance (the generator only copies the fields
|
|
2958
|
+
* it knows, and everything it copies is redacted again).
|
|
2959
|
+
*
|
|
2960
|
+
* @returns {string}
|
|
2961
|
+
*/
|
|
2962
|
+
const diagnosticsText = () => buildDiagnostics({
|
|
2963
|
+
version: pluginVersion(),
|
|
2964
|
+
platform: env.platform,
|
|
2965
|
+
hostKind: hostKindOf().kind,
|
|
2966
|
+
isDesktopApp: env.isDesktopApp === true,
|
|
2967
|
+
startup: startupModeOf(),
|
|
2968
|
+
supervisor: supervisorStatusOf(),
|
|
2969
|
+
restartMode: config.restartMode,
|
|
2970
|
+
lastRestart: lastRestartVerdict(),
|
|
2971
|
+
stateDir: stateDirOf(),
|
|
2972
|
+
watchdog: watchdogInfo(),
|
|
2973
|
+
})
|
|
2974
|
+
|
|
2975
|
+
/**
|
|
2976
|
+
* HOW THIS HOST WAS STARTED (the startup matrix).
|
|
2977
|
+
*
|
|
2978
|
+
* The answer decides what the plugin may do: only a `bare` host (nobody
|
|
2979
|
+
* manages it) may get the supervisor of this plugin or a self-replay
|
|
2980
|
+
* command. Everything else — an external supervisor, a container, the
|
|
2981
|
+
* desktop app, or a host whose starter cannot be established — must not
|
|
2982
|
+
* lead to a second host.
|
|
2983
|
+
*
|
|
2984
|
+
* @returns {object} the classification (mode, kind, evidence, …)
|
|
2985
|
+
*/
|
|
2986
|
+
const startupModeOf = () => classifyStartup({
|
|
2987
|
+
platform: env.platform,
|
|
2988
|
+
env: process.env,
|
|
2989
|
+
ppid: process.ppid,
|
|
2990
|
+
isDesktopApp: env.isDesktopApp === true,
|
|
2991
|
+
supervised: detectSupervisor({ env: process.env, stateDir: stateDirOf() }).supervised,
|
|
2992
|
+
container: detectContainer({ env: process.env }).container,
|
|
2993
|
+
argv: process.argv,
|
|
2994
|
+
execPath: process.execPath,
|
|
2995
|
+
readChain: readProcessChain,
|
|
2996
|
+
})
|
|
2997
|
+
|
|
2998
|
+
/**
|
|
2999
|
+
* What the plugin knows about the supervisor.
|
|
3000
|
+
*
|
|
3001
|
+
* The parent chain of this very process is part of the answer: a host that
|
|
3002
|
+
* somebody ELSE supervises (systemd, pm2, a `supervise-*.sh` script) must
|
|
3003
|
+
* not be restarted by a self-replay — the supervisor starts it again, and a
|
|
3004
|
+
* second host on the same port is a double start.
|
|
3005
|
+
*
|
|
3006
|
+
* @returns {object} the status (mode, kind, installed, running, restartCapable)
|
|
3007
|
+
*/
|
|
3008
|
+
const supervisorStatusOf = () => supervisorStatus({
|
|
3009
|
+
stateDir: stateDirOf(),
|
|
3010
|
+
env: process.env,
|
|
3011
|
+
platform: env.platform,
|
|
3012
|
+
mode: config.supervisorMode,
|
|
3013
|
+
ppid: process.ppid,
|
|
3014
|
+
// The desktop app brings ITSELF back up (`restartMode: app`): it is not
|
|
3015
|
+
// "nobody knows", and the restart is available.
|
|
3016
|
+
isDesktopApp: env.isDesktopApp === true,
|
|
3017
|
+
// HOW this host was started (the startup matrix), with its evidence.
|
|
3018
|
+
startup: startupModeOf(),
|
|
3019
|
+
// THE PLATFORM GUIDE: how "start at login" works here (Windows / Linux /
|
|
3020
|
+
// WSL / container), with the commands the user can copy.
|
|
3021
|
+
autostart: autostartAdviceOf(),
|
|
3022
|
+
})
|
|
3023
|
+
|
|
3024
|
+
/**
|
|
3025
|
+
* The start command the supervisor would use.
|
|
3026
|
+
*
|
|
3027
|
+
* This is the PLAIN command line of this very process: the interpreter, its
|
|
3028
|
+
* own arguments and the script it runs. The full derivation (with the
|
|
3029
|
+
* refusal rules) belongs to the "E" step; this is the honest minimum the
|
|
3030
|
+
* supervisor needs to be able to start the host again.
|
|
3031
|
+
*
|
|
3032
|
+
* @returns {{ execPath: string, execArgv: string[], argv: string[], cwd: string }}
|
|
3033
|
+
*/
|
|
3034
|
+
const supervisorStartCommand = () => ({
|
|
3035
|
+
execPath: process.execPath,
|
|
3036
|
+
execArgv: Array.isArray(process.execArgv) ? process.execArgv.map(String) : [],
|
|
3037
|
+
argv: Array.isArray(process.argv) ? process.argv.slice(1).map(String) : [],
|
|
3038
|
+
cwd: process.cwd(),
|
|
3039
|
+
})
|
|
3040
|
+
|
|
3041
|
+
/**
|
|
3042
|
+
* The environment the supervisor hands to the host it starts.
|
|
3043
|
+
*
|
|
3044
|
+
* Only a WHITELIST is inherited (the daemon does that too): the host's own
|
|
3045
|
+
* internal variables must not leak into the next start, or the profile would
|
|
3046
|
+
* drift.
|
|
3047
|
+
*
|
|
3048
|
+
* @returns {Record<string, string>}
|
|
3049
|
+
*/
|
|
3050
|
+
const supervisorEnv = () => {
|
|
3051
|
+
const env = {}
|
|
3052
|
+
const source = process.env || {}
|
|
3053
|
+
for (const key of [
|
|
3054
|
+
'PATH', 'HOME', 'USER', 'LOGNAME', 'SHELL', 'LANG', 'LC_ALL', 'SystemRoot',
|
|
3055
|
+
'TEMP', 'TMP', 'APPDATA', 'LOCALAPPDATA', 'ProgramData',
|
|
3056
|
+
'NUMBER_OF_PROCESSORS', 'WSLENV', 'WSL_DISTRO_NAME', 'WSL_INTEROP',
|
|
3057
|
+
]) {
|
|
3058
|
+
const value = source[key]
|
|
3059
|
+
if (typeof value === 'string' && value !== '') env[key] = value
|
|
3060
|
+
}
|
|
3061
|
+
// The two variables that say WHICH profile this host belongs to.
|
|
3062
|
+
if (typeof source.DSH_HOME === 'string' && source.DSH_HOME !== '') env.DSH_HOME = source.DSH_HOME
|
|
3063
|
+
if (typeof source.DSH_PROFILE === 'string' && source.DSH_PROFILE !== '') env.DSH_PROFILE = source.DSH_PROFILE
|
|
3064
|
+
return env
|
|
3065
|
+
}
|
|
3066
|
+
|
|
3067
|
+
/**
|
|
3068
|
+
* Install the supervisor.
|
|
3069
|
+
*
|
|
3070
|
+
* @param {{ confirmed?: boolean, dryRun?: boolean }} [options]
|
|
3071
|
+
* @returns {object} the outcome (ok, kind, steps, error, plan)
|
|
3072
|
+
*/
|
|
3073
|
+
const installSupervisorNow = (options = {}) => {
|
|
3074
|
+
const stateDir = stateDirOf()
|
|
3075
|
+
const profile = resolveProfileDirDetailed()
|
|
3076
|
+
const start = supervisorStartCommand()
|
|
3077
|
+
const result = installSupervisor({
|
|
3078
|
+
stateDir,
|
|
3079
|
+
profileDir: profile.dir,
|
|
3080
|
+
platform: env.platform,
|
|
3081
|
+
confirmed: options.confirmed === true,
|
|
3082
|
+
dryRun: options.dryRun === true,
|
|
3083
|
+
// The desktop app manages itself: a login record would start a SECOND
|
|
3084
|
+
// client at every login, so the installation is refused there.
|
|
3085
|
+
isDesktopApp: env.isDesktopApp === true,
|
|
3086
|
+
// The startup matrix: only a `bare` host (nobody manages it) may get
|
|
3087
|
+
// the supervisor of this plugin.
|
|
3088
|
+
startup: startupModeOf(),
|
|
3089
|
+
// WHICH MACHINE IS THIS? (WSL is not a Linux: the autostart there is a
|
|
3090
|
+
// different thing, and the Docker/Compose answer is a wrong one.)
|
|
3091
|
+
host: hostKindOf(),
|
|
3092
|
+
nodePath: process.execPath,
|
|
3093
|
+
execPath: start.execPath,
|
|
3094
|
+
execArgv: start.execArgv,
|
|
3095
|
+
argv: start.argv,
|
|
3096
|
+
cwd: start.cwd,
|
|
3097
|
+
env: supervisorEnv(),
|
|
3098
|
+
profileEnv: {
|
|
3099
|
+
...(process.env.DSH_HOME ? { DSH_HOME: process.env.DSH_HOME } : {}),
|
|
3100
|
+
...(process.env.DSH_PROFILE ? { DSH_PROFILE: process.env.DSH_PROFILE } : {}),
|
|
3101
|
+
},
|
|
3102
|
+
port: config.supervisorPort > 0 ? config.supervisorPort : (runtime.webPort || 0),
|
|
3103
|
+
healthUrl: restartHealthUrl() ?? '',
|
|
3104
|
+
ownerUser: typeof process.env.USER === 'string' ? process.env.USER : (typeof process.env.LOGNAME === 'string' ? process.env.LOGNAME : ''),
|
|
3105
|
+
timings: {
|
|
3106
|
+
restartDelayMs: config.supervisorRestartDelayMs,
|
|
3107
|
+
restartDelayMaxMs: config.supervisorRestartDelayMaxMs,
|
|
3108
|
+
startTimeoutMs: config.supervisorStartTimeoutMs,
|
|
3109
|
+
killGraceMs: config.supervisorKillGraceMs,
|
|
3110
|
+
},
|
|
3111
|
+
kind: config.supervisorKind,
|
|
3112
|
+
// The previous way of starting the host: the uninstall puts it back.
|
|
3113
|
+
previous: { restartMode: config.restartMode, restartCommand: config.restartCommand },
|
|
3114
|
+
log: (message) => log.warn(message),
|
|
3115
|
+
})
|
|
3116
|
+
if (result.ok) {
|
|
3117
|
+
log.warn('the supervisor was installed', { kind: result.kind, degraded: result.degraded })
|
|
3118
|
+
} else {
|
|
3119
|
+
log.error('the supervisor was not installed:', result.error)
|
|
3120
|
+
}
|
|
3121
|
+
return result
|
|
3122
|
+
}
|
|
3123
|
+
|
|
3124
|
+
/**
|
|
3125
|
+
* Uninstall the supervisor and give the machine back to the user.
|
|
3126
|
+
*
|
|
3127
|
+
* @returns {Promise<object>} the outcome (ok, steps, error, pending, previous)
|
|
3128
|
+
*/
|
|
3129
|
+
const uninstallSupervisorNow = async () => {
|
|
3130
|
+
const result = await uninstallSupervisor({
|
|
3131
|
+
stateDir: stateDirOf(),
|
|
3132
|
+
platform: env.platform,
|
|
3133
|
+
log: (message) => log.warn(message),
|
|
3134
|
+
})
|
|
3135
|
+
if (result.ok) {
|
|
3136
|
+
log.warn('the supervisor was uninstalled', { previous: result.previous })
|
|
3137
|
+
} else {
|
|
3138
|
+
log.error('the supervisor could not be removed completely:', result.error)
|
|
3139
|
+
}
|
|
3140
|
+
return result
|
|
3141
|
+
}
|
|
3142
|
+
|
|
3143
|
+
/**
|
|
3144
|
+
* Record the answer of the one-time question about the supervisor.
|
|
3145
|
+
*
|
|
3146
|
+
* @param {string} choice 'install' | 'external' | 'dismissed'
|
|
3147
|
+
* @param {object} [previous] the values that were in effect before
|
|
3148
|
+
* @returns {object} the recorded answer
|
|
3149
|
+
*/
|
|
3150
|
+
const recordSupervisorChoice = (choice, previous = null) => {
|
|
3151
|
+
const record = recordOnboarding(state, {
|
|
3152
|
+
feature: 'supervisor',
|
|
3153
|
+
version: SUPERVISOR_FEATURE_VERSION,
|
|
3154
|
+
choice,
|
|
3155
|
+
previous,
|
|
3156
|
+
now: Date.now(),
|
|
3157
|
+
})
|
|
3158
|
+
persist()
|
|
3159
|
+
return record
|
|
3160
|
+
}
|
|
3161
|
+
|
|
3162
|
+
/**
|
|
3163
|
+
* Derive the command that starts this host again — or say why there is
|
|
3164
|
+
* none. Nothing is written: this is the dry run (B).
|
|
3165
|
+
*
|
|
3166
|
+
* @param {{ fixedPort?: number }} [options]
|
|
3167
|
+
* @returns {Promise<object>} the derivation (the plan of what WOULD be written)
|
|
3168
|
+
*/
|
|
3169
|
+
const deriveStartCommandNow = async (options = {}) => {
|
|
3170
|
+
const detected = detectSupervisor({ env: process.env, stateDir: stateDirOf() })
|
|
3171
|
+
const external = detectExternalSupervisor({
|
|
3172
|
+
ppid: process.ppid,
|
|
3173
|
+
env: process.env,
|
|
3174
|
+
platform: env.platform,
|
|
3175
|
+
})
|
|
3176
|
+
const container = detectContainer({ env: process.env })
|
|
3177
|
+
const init = detectInitSystems({ platform: env.platform, homeDir: os.homedir() })
|
|
3178
|
+
// A random port has to be FIXED, or the cookie of the browser dies with
|
|
3179
|
+
// every restart. The port is picked once and then written down.
|
|
3180
|
+
let fixedPort = Number(options.fixedPort) > 0 ? Number(options.fixedPort) : 0
|
|
3181
|
+
if (fixedPort === 0 && config.supervisorPort > 0) fixedPort = config.supervisorPort
|
|
3182
|
+
if (fixedPort === 0 && Number(runtime.webPort) > 0 && config.supervisorPort === 0) {
|
|
3183
|
+
// The port of the running instance is the best candidate: it is the one
|
|
3184
|
+
// the browser knows.
|
|
3185
|
+
fixedPort = Number(runtime.webPort)
|
|
3186
|
+
}
|
|
3187
|
+
const derivation = deriveStartCommand({
|
|
3188
|
+
execPath: process.execPath,
|
|
3189
|
+
execArgv: process.execArgv,
|
|
3190
|
+
argv: process.argv,
|
|
3191
|
+
cwd: process.cwd(),
|
|
3192
|
+
platform: env.platform,
|
|
3193
|
+
env: process.env,
|
|
3194
|
+
pid: process.pid,
|
|
3195
|
+
isDesktopApp: env.isDesktopApp === true,
|
|
3196
|
+
webPort: runtime.webPort || 0,
|
|
3197
|
+
supervised: detected.supervised === true,
|
|
3198
|
+
externalSupervisor: external,
|
|
3199
|
+
container: container.container,
|
|
3200
|
+
// THE STARTUP MATRIX: it decides whether a self-replay is possible at
|
|
3201
|
+
// all (and whether the command must be detached from its session).
|
|
3202
|
+
startup: startupModeOf(),
|
|
3203
|
+
hasInit: env.platform === 'win32' ? true : init.systemdUser,
|
|
3204
|
+
fixedPort,
|
|
3205
|
+
currentRestartMode: config.restartMode,
|
|
3206
|
+
currentRestartCommand: config.restartCommand,
|
|
3207
|
+
healthPath: resolveHealthPath(config),
|
|
3208
|
+
})
|
|
3209
|
+
const plan = buildDryRunPlan(derivation)
|
|
3210
|
+
return {
|
|
3211
|
+
...plan,
|
|
3212
|
+
derivation,
|
|
3213
|
+
externalSupervisor: external,
|
|
3214
|
+
container: container.container,
|
|
3215
|
+
initReason: init.reason,
|
|
3216
|
+
// The fragment a container needs (there is no "at login" there).
|
|
3217
|
+
compose: derivation.reason === 'container-no-init' && derivation.argv
|
|
3218
|
+
? composeRestartFragment(derivation.argv)
|
|
3219
|
+
: null,
|
|
3220
|
+
}
|
|
3221
|
+
}
|
|
3222
|
+
|
|
3223
|
+
/**
|
|
3224
|
+
* Write the derived command into the profile configuration — only after the
|
|
3225
|
+
* user confirmed the dry run.
|
|
3226
|
+
*
|
|
3227
|
+
* @param {{ derivation?: object }} [options]
|
|
3228
|
+
* @returns {Promise<object>} the outcome
|
|
3229
|
+
*/
|
|
3230
|
+
const applyDerivedCommand = async (options = {}) => {
|
|
3231
|
+
const derivation = options.derivation && typeof options.derivation === 'object'
|
|
3232
|
+
? options.derivation
|
|
3233
|
+
: await deriveStartCommandNow()
|
|
3234
|
+
if (!derivation || derivation.ok !== true || !derivation.writes) {
|
|
3235
|
+
return {
|
|
3236
|
+
ok: false,
|
|
3237
|
+
error: 'nothing-to-write',
|
|
3238
|
+
reason: derivation ? derivation.reason : 'unknown',
|
|
3239
|
+
}
|
|
3240
|
+
}
|
|
3241
|
+
const previous = {
|
|
3242
|
+
restartMode: config.restartMode,
|
|
3243
|
+
restartCommand: config.restartCommand,
|
|
3244
|
+
}
|
|
3245
|
+
try {
|
|
3246
|
+
await runtime.applyConfigChoice({
|
|
3247
|
+
restartMode: derivation.writes.restartMode,
|
|
3248
|
+
restartCommand: derivation.writes.restartCommand,
|
|
3249
|
+
})
|
|
3250
|
+
} catch (error) {
|
|
3251
|
+
return { ok: false, error: describeError(error), reason: derivation.reason }
|
|
3252
|
+
}
|
|
3253
|
+
// The values that were in effect are kept, so the undo can put them back.
|
|
3254
|
+
recordSupervisorChoice('external', previous)
|
|
3255
|
+
log.warn('the derived restart command was written', {
|
|
3256
|
+
mode: derivation.writes.restartMode,
|
|
3257
|
+
reason: derivation.reason,
|
|
3258
|
+
})
|
|
3259
|
+
return { ok: true, writes: derivation.writes, previous, reason: derivation.reason }
|
|
3260
|
+
}
|
|
3261
|
+
|
|
3262
|
+
/**
|
|
3263
|
+
* Put the previous start command back (the undo).
|
|
3264
|
+
*
|
|
3265
|
+
* @returns {Promise<object>} the outcome
|
|
3266
|
+
*/
|
|
3267
|
+
const revertDerivedCommand = async () => {
|
|
3268
|
+
const record = state.onboarding
|
|
3269
|
+
const previous = record && typeof record === 'object' ? record.previous : null
|
|
3270
|
+
if (!previous || typeof previous.restartMode !== 'string') {
|
|
3271
|
+
return { ok: false, error: 'nothing-to-revert' }
|
|
3272
|
+
}
|
|
3273
|
+
try {
|
|
3274
|
+
await runtime.applyConfigChoice({
|
|
3275
|
+
restartMode: previous.restartMode,
|
|
3276
|
+
restartCommand: previous.restartCommand === null ? '' : previous.restartCommand,
|
|
3277
|
+
})
|
|
3278
|
+
} catch (error) {
|
|
3279
|
+
return { ok: false, error: describeError(error) }
|
|
3280
|
+
}
|
|
3281
|
+
log.warn('the previous restart command was put back', previous)
|
|
3282
|
+
return { ok: true, previous }
|
|
3283
|
+
}
|
|
3284
|
+
|
|
3285
|
+
/**
|
|
3286
|
+
* Write the marker that tells the rollback watcher "the plugin has decided
|
|
3287
|
+
* on its own — stand down".
|
|
3288
|
+
*
|
|
3289
|
+
* The watcher is the fallback for the case that the plugin dies with the
|
|
3290
|
+
* host. When the plugin SURVIVES the restart command it decides alone, and
|
|
3291
|
+
* this marker keeps the watcher from rolling back a restart that was fine.
|
|
3292
|
+
*
|
|
3293
|
+
* @param {{ decided: boolean, ok?: boolean, reason?: string }} outcome
|
|
3294
|
+
* @returns {boolean} whether the marker was written
|
|
3295
|
+
*/
|
|
3296
|
+
const writeRestartOutcome = (outcome) => {
|
|
3297
|
+
const file = path.join(stateDirOf(), RESTART_OUTCOME_FILE)
|
|
3298
|
+
try {
|
|
3299
|
+
fs.writeFileSync(file, JSON.stringify({
|
|
3300
|
+
decided: true,
|
|
3301
|
+
ok: outcome && outcome.ok === true,
|
|
3302
|
+
reason: outcome && typeof outcome.reason === 'string' ? outcome.reason : '',
|
|
3303
|
+
at: new Date().toISOString(),
|
|
3304
|
+
bootId,
|
|
3305
|
+
}, null, 2), 'utf8')
|
|
3306
|
+
return true
|
|
3307
|
+
} catch (error) {
|
|
3308
|
+
log.warn('could not write the restart outcome marker:', String(error))
|
|
3309
|
+
return false
|
|
3310
|
+
}
|
|
3311
|
+
}
|
|
3312
|
+
|
|
3313
|
+
/**
|
|
3314
|
+
* Write the rollback report of a restart that FAILED, so the interface can
|
|
3315
|
+
* say what happened.
|
|
3316
|
+
*
|
|
3317
|
+
* The report has the same shape as the one the detached watcher writes, so
|
|
3318
|
+
* there is exactly ONE place that reads it.
|
|
3319
|
+
*
|
|
3320
|
+
* @param {{ rolledBack: boolean, reason: string, backupDir?: string|null, restored?: string[] }} details
|
|
3321
|
+
* @returns {boolean} whether the report was written
|
|
3322
|
+
*/
|
|
3323
|
+
const writeRestartReport = (details) => {
|
|
3324
|
+
const file = path.join(stateDirOf(), RESTART_REPORT_NAME)
|
|
3325
|
+
try {
|
|
3326
|
+
fs.writeFileSync(file, JSON.stringify({
|
|
3327
|
+
ok: false,
|
|
3328
|
+
at: new Date().toISOString(),
|
|
3329
|
+
waitedMs: 0,
|
|
3330
|
+
rolledBack: details.rolledBack === true,
|
|
3331
|
+
rollbackError: details.rolledBack === true ? null : details.reason,
|
|
3332
|
+
reason: details.reason,
|
|
3333
|
+
backupDir: details.backupDir ?? null,
|
|
3334
|
+
restored: Array.isArray(details.restored) ? details.restored : [],
|
|
3335
|
+
// The plugin restores the files; it never starts a second host.
|
|
3336
|
+
appStarted: false,
|
|
3337
|
+
}, null, 2), 'utf8')
|
|
3338
|
+
return true
|
|
3339
|
+
} catch (error) {
|
|
3340
|
+
log.warn('could not write the rollback report:', String(error))
|
|
3341
|
+
return false
|
|
3342
|
+
}
|
|
3343
|
+
}
|
|
3344
|
+
|
|
3345
|
+
/**
|
|
3346
|
+
* Put the pre-restart snapshot back over the live profile.
|
|
3347
|
+
*
|
|
3348
|
+
* This is the in-process half of the rollback: it runs when the plugin
|
|
3349
|
+
* itself can see that the restart failed (the restart command did not
|
|
3350
|
+
* start, or the new instance never answered). The detached watcher does
|
|
3351
|
+
* the same thing when the plugin died with the host.
|
|
3352
|
+
*
|
|
3353
|
+
* @param {{ backup: any, profileDir: string|null, reason: string }} details
|
|
3354
|
+
* @returns {{ rolledBack: boolean, restored: string[], error: string|null }}
|
|
3355
|
+
*/
|
|
3356
|
+
const rollbackInProcess = ({ backup, profileDir, reason }) => {
|
|
3357
|
+
const restored = []
|
|
3358
|
+
if (!backup || backup.ok !== true || typeof backup.dir !== 'string' || backup.dir === '') {
|
|
3359
|
+
log.error('the restart failed and there is no snapshot to roll back to:', reason)
|
|
3360
|
+
writeRestartReport({ rolledBack: false, reason, backupDir: null, restored })
|
|
3361
|
+
return { rolledBack: false, restored, error: 'no snapshot to roll back to' }
|
|
3362
|
+
}
|
|
3363
|
+
if (typeof profileDir !== 'string' || profileDir === '') {
|
|
3364
|
+
log.error('the restart failed and the profile directory is unknown:', reason)
|
|
3365
|
+
writeRestartReport({ rolledBack: false, reason, backupDir: backup.dir, restored })
|
|
3366
|
+
return { rolledBack: false, restored, error: 'the profile directory is unknown' }
|
|
3367
|
+
}
|
|
3368
|
+
// THE LOCK: the detached watcher may be restoring the snapshot at the same
|
|
3369
|
+
// time. Exactly ONE of them copies the files — two copies at once would
|
|
3370
|
+
// write the same files (117 MB of plugin tree) at the same time.
|
|
3371
|
+
const lock = claimRollbackLock(stateDirOf())
|
|
3372
|
+
if (!lock.ok) {
|
|
3373
|
+
log.warn('another rollback is already running; this one stands down:', lock.error)
|
|
3374
|
+
return { rolledBack: false, restored, error: lock.error }
|
|
3375
|
+
}
|
|
3376
|
+
let result = null
|
|
3377
|
+
try {
|
|
3378
|
+
result = restoreBackup({ backupDir: backup.dir, profileDir })
|
|
3379
|
+
} catch (error) {
|
|
3380
|
+
result = { ok: false, restored: [], error: String(error) }
|
|
3381
|
+
} finally {
|
|
3382
|
+
releaseRollbackLock(stateDirOf())
|
|
3383
|
+
}
|
|
3384
|
+
if (result && result.ok === true) {
|
|
3385
|
+
log.warn('the pre-restart snapshot was restored:', {
|
|
3386
|
+
reason,
|
|
3387
|
+
restored: result.restored,
|
|
3388
|
+
backupDir: backup.dir,
|
|
3389
|
+
})
|
|
3390
|
+
writeRestartReport({ rolledBack: true, reason, backupDir: backup.dir, restored: result.restored })
|
|
3391
|
+
return { rolledBack: true, restored: result.restored, error: null }
|
|
3392
|
+
}
|
|
3393
|
+
const error = (result && result.error) || 'the restore failed'
|
|
3394
|
+
log.error('the pre-restart snapshot could not be restored:', reason, error)
|
|
3395
|
+
writeRestartReport({ rolledBack: false, reason, backupDir: backup.dir, restored })
|
|
3396
|
+
return { rolledBack: false, restored, error }
|
|
3397
|
+
}
|
|
3398
|
+
|
|
3399
|
+
/**
|
|
3400
|
+
* Arm the rollback watcher: the process that confirms the new instance and
|
|
3401
|
+
* puts the snapshot back when it never comes.
|
|
3402
|
+
*
|
|
3403
|
+
* The watcher is started OUTSIDE the host process, because the restart
|
|
3404
|
+
* command usually kills the host itself — in that case the plugin cannot
|
|
3405
|
+
* confirm anything any more, and the watcher is the only safety net left.
|
|
3406
|
+
*
|
|
3407
|
+
* @param {{ backup: any, profileDir: string|null, gateScriptPath?: string|null }} details
|
|
3408
|
+
* @returns {{ ok: boolean, method: string, error: string|null, scriptPath: string|null }}
|
|
3409
|
+
*/
|
|
3410
|
+
const armRollbackWatcher = ({ backup, profileDir, gateScriptPath = null }) => {
|
|
3411
|
+
if (!backup || backup.ok !== true || typeof backup.dir !== 'string' || backup.dir === '') {
|
|
3412
|
+
log.warn('no pre-restart snapshot: the automatic rollback is not armed')
|
|
3413
|
+
return { ok: false, method: 'none', error: 'no snapshot', scriptPath: null }
|
|
3414
|
+
}
|
|
3415
|
+
if (typeof profileDir !== 'string' || profileDir === '') {
|
|
3416
|
+
log.warn('the profile directory is unknown: the automatic rollback is not armed')
|
|
3417
|
+
return { ok: false, method: 'none', error: 'the profile directory is unknown', scriptPath: null }
|
|
3418
|
+
}
|
|
3419
|
+
const stateDir = stateDirOf()
|
|
3420
|
+
const scriptPath = path.join(stateDir, 'restart-' + state.generation + '-watch.ps1')
|
|
3421
|
+
const script = buildWatchScript({
|
|
3422
|
+
// The fallback health URL (the port of the OLD instance). It is only
|
|
3423
|
+
// used when the host publishes no instance record at all.
|
|
3424
|
+
healthUrl: restartHealthUrl() ?? '',
|
|
3425
|
+
healthPath: resolveHealthPath(config),
|
|
3426
|
+
// The instance record: the port of the NEW instance and its boot id.
|
|
3427
|
+
portFile: accessFilePath(stateDir),
|
|
3428
|
+
expectedBootId: bootId,
|
|
3429
|
+
outcomeFile: path.join(stateDir, RESTART_OUTCOME_FILE),
|
|
3430
|
+
backupDir: backup.dir,
|
|
3431
|
+
profileDir,
|
|
3432
|
+
reportPath: path.join(stateDir, ROLLBACK_REPORT_NAME),
|
|
3433
|
+
lockFile: path.join(stateDir, ROLLBACK_LOCK_NAME),
|
|
3434
|
+
timeoutMs: config.rollbackTimeoutMs,
|
|
3435
|
+
rollbackTimeoutMs: config.rollbackCopyTimeoutMs,
|
|
3436
|
+
since: new Date().toISOString(),
|
|
3437
|
+
selfCleanPaths: [scriptPath, gateScriptPath].filter(Boolean),
|
|
3438
|
+
})
|
|
3439
|
+
const written = writeTextFileAtomic(scriptPath, script)
|
|
3440
|
+
if (!written.ok) {
|
|
3441
|
+
log.error('could not write the rollback watcher:', written.error)
|
|
3442
|
+
return { ok: false, method: 'none', error: written.error, scriptPath }
|
|
3443
|
+
}
|
|
3444
|
+
const result = launchWatcherFn({
|
|
3445
|
+
watchScriptPath: scriptPath,
|
|
3446
|
+
platform: env.platform,
|
|
3447
|
+
})
|
|
3448
|
+
if (!result || result.ok !== true) {
|
|
3449
|
+
log.error('could not start the rollback watcher:', (result && result.error) || 'unknown error')
|
|
3450
|
+
return { ok: false, method: (result && result.method) || 'none', error: (result && result.error) || null, scriptPath }
|
|
3451
|
+
}
|
|
3452
|
+
log.warn('the rollback watcher is armed', {
|
|
3453
|
+
scriptPath,
|
|
3454
|
+
method: result.method,
|
|
3455
|
+
timeoutMs: config.rollbackTimeoutMs,
|
|
3456
|
+
})
|
|
3457
|
+
return { ok: true, method: result.method, error: null, scriptPath }
|
|
3458
|
+
}
|
|
3459
|
+
|
|
3460
|
+
/**
|
|
3461
|
+
* Say in the report WHO is watching this restart.
|
|
3462
|
+
*
|
|
3463
|
+
* The v2 watchdog writes its own report when it acts. When v1 is the
|
|
3464
|
+
* arbiter (v2 could not be armed), the host writes it right away — so
|
|
3465
|
+
* `last-watchdog-report.json` always names the arbiter, and a problem can
|
|
3466
|
+
* be read at a glance.
|
|
3467
|
+
*
|
|
3468
|
+
* @param {string} arbiter
|
|
3469
|
+
* @param {string} reason
|
|
3470
|
+
*/
|
|
3471
|
+
const writeArbiterReport = (arbiter, reason, tier = 'none') => {
|
|
3472
|
+
try {
|
|
3473
|
+
writeReport(stateDirOf(), {
|
|
3474
|
+
at: new Date().toISOString(),
|
|
3475
|
+
result: 'armed',
|
|
3476
|
+
tier,
|
|
3477
|
+
tierReason: reason || '',
|
|
3478
|
+
arbiter,
|
|
3479
|
+
waitedMs: 0,
|
|
3480
|
+
rechecked: false,
|
|
3481
|
+
restored: false,
|
|
3482
|
+
relaunched: false,
|
|
3483
|
+
ready: false,
|
|
3484
|
+
error: null,
|
|
3485
|
+
})
|
|
3486
|
+
} catch (error) {
|
|
3487
|
+
log.warn('could not write the arbiter report:', String(error))
|
|
3488
|
+
}
|
|
3489
|
+
}
|
|
3490
|
+
|
|
3491
|
+
/** Execute the restart now; the detached helper survives our own death. */
|
|
3492
|
+
const executeRestart = async () => {
|
|
3493
|
+
stopPolling()
|
|
3494
|
+
if (!state.restart) return
|
|
3495
|
+
try {
|
|
3496
|
+
await flushSessions()
|
|
3497
|
+
} catch {}
|
|
3498
|
+
// Re-check the gate AFTER the flush and BEFORE anything is killed or scheduled:
|
|
3499
|
+
// another conversation may have started while the sessions were being written.
|
|
3500
|
+
// This re-check also protects the ungated fallback (>20 gate aborts) and the
|
|
3501
|
+
// forceRestartWhenBusy path: even when the detached helper cannot re-check the
|
|
3502
|
+
// gate itself, a busy conversation is never killed here.
|
|
3503
|
+
// Last chance to park a goal-driven session: without it the detached
|
|
3504
|
+
// helper would refuse to kill the process and the restart would wait.
|
|
3505
|
+
try {
|
|
3506
|
+
disarmBlockingGoals(busyAgents())
|
|
3507
|
+
} catch (error) {
|
|
3508
|
+
log.warn('could not disarm the goals of the blocking sessions:', String(error))
|
|
3509
|
+
}
|
|
3510
|
+
const verdict = gate()
|
|
3511
|
+
if (!verdict.allow) {
|
|
3512
|
+
log.info('restart deferred: a conversation is still active or the host is settling', {
|
|
3513
|
+
reason: verdict.reason,
|
|
3514
|
+
blockedBy: verdict.blockedBy,
|
|
3515
|
+
quietForMs: verdict.quietForMs,
|
|
3516
|
+
})
|
|
3517
|
+
persist()
|
|
3518
|
+
startPolling()
|
|
3519
|
+
return
|
|
3520
|
+
}
|
|
3521
|
+
setPhase(state, 'restarting')
|
|
3522
|
+
state.restart.attempt += 1
|
|
3523
|
+
persist()
|
|
3524
|
+
log.warn('executing graceful restart', { generation: state.generation, pid: process.pid })
|
|
3525
|
+
try {
|
|
3526
|
+
// How the host is restarted: the custom command, or the desktop app.
|
|
3527
|
+
const useCustomCommand = config.restartMode === 'command' && config.restartCommand.trim() !== ''
|
|
3528
|
+
const canRelaunchDesktop = Boolean(env.isDesktopApp && env.appExePath)
|
|
3529
|
+
if (!useCustomCommand && !canRelaunchDesktop) {
|
|
3530
|
+
// No restart method at all (the host is not the desktop app and there is
|
|
3531
|
+
// no custom command): nothing is restarted, so nothing is snapshotted
|
|
3532
|
+
// either — there is no rollback point for a restart that never happens.
|
|
3533
|
+
log.warn('no restart method available (not the desktop app and restartCommand is empty); leaving the checkpoint in place')
|
|
3534
|
+
rollbackRestart('no restart method available')
|
|
3535
|
+
return
|
|
3536
|
+
}
|
|
3537
|
+
|
|
3538
|
+
// ONE snapshot for EVERY restart mode: the profile is copied BEFORE
|
|
3539
|
+
// anything is killed, no matter how the host is restarted. The desktop
|
|
3540
|
+
// app relaunch and the custom restart command share this single entry, so
|
|
3541
|
+
// a hosted host that is restarted by a command is just as recoverable as
|
|
3542
|
+
// the desktop app.
|
|
3543
|
+
const stateDir = path.dirname(stateFile)
|
|
3544
|
+
const snapshot = snapshotFn({
|
|
3545
|
+
config,
|
|
3546
|
+
stateDir,
|
|
3547
|
+
log,
|
|
3548
|
+
profile: resolveProfileDirDetailed(),
|
|
3549
|
+
})
|
|
3550
|
+
// The snapshot is the rollback point of BOTH branches.
|
|
3551
|
+
const backup = snapshot && snapshot.backup ? snapshot.backup : null
|
|
3552
|
+
const profileDir = snapshot && snapshot.profileDir ? snapshot.profileDir : null
|
|
3553
|
+
|
|
3554
|
+
// ARM THE WATCHDOG: it waits for the relaunched host, and when that
|
|
3555
|
+
// host never answers, it puts the snapshot back and starts it again.
|
|
3556
|
+
// Which way it is triggered is decided by the machine (the tier).
|
|
3557
|
+
//
|
|
3558
|
+
// The result matters: when v2 is armed, it is the ONLY watchdog. The
|
|
3559
|
+
// old one (v1) is NOT started next to it — they take the same lock, and
|
|
3560
|
+
// the one that loses it gives up. If v2 lost, the profile would be
|
|
3561
|
+
// restored but nothing would be running any more.
|
|
3562
|
+
const watchdogArm = backup && backup.ok === true && profileDir !== null
|
|
3563
|
+
? armWatchdogFn({
|
|
3564
|
+
snapshotDir: backup.dir,
|
|
3565
|
+
profileDir,
|
|
3566
|
+
relaunchCommand: useCustomCommand && typeof config.restartCommand === 'string' && config.restartCommand.trim() !== ''
|
|
3567
|
+
? {
|
|
3568
|
+
file: config.restartCommand.trim().split(/\s+/)[0],
|
|
3569
|
+
args: config.restartCommand.trim().split(/\s+/).slice(1),
|
|
3570
|
+
}
|
|
3571
|
+
: (env.appExePath ? { file: env.appExePath, args: [] } : null),
|
|
3572
|
+
healthUrl: restartHealthUrl(),
|
|
3573
|
+
portFile: accessFilePath(stateDir),
|
|
3574
|
+
expectedBootId: bootId,
|
|
3575
|
+
})
|
|
3576
|
+
: { ok: false, tier: TIERS.NONE, reason: 'there is no snapshot to watch', error: null }
|
|
3577
|
+
|
|
3578
|
+
// THE DURABLE RESTART RECORD. The instance that is being replaced
|
|
3579
|
+
// writes the "pending" half BEFORE it runs the restart command: the
|
|
3580
|
+
// snapshot it took (directory, local time, duration, which older
|
|
3581
|
+
// snapshots were pruned) and the generation. The instance that replaces
|
|
3582
|
+
// it rewrites the record as `ok` / `failed` after it has checked itself
|
|
3583
|
+
// (see `verifyRestartBoot`). If the new instance never comes up, the
|
|
3584
|
+
// record stays `pending` — which is exactly what has to be visible.
|
|
3585
|
+
recordRestartVerdict({
|
|
3586
|
+
result: RESTART_RESULTS.PENDING,
|
|
3587
|
+
mode: useCustomCommand ? 'command' : 'app',
|
|
3588
|
+
generation: state.generation,
|
|
3589
|
+
requestedAt: state.restart && Number.isFinite(state.restart.requestedAt)
|
|
3590
|
+
? state.restart.requestedAt
|
|
3591
|
+
: null,
|
|
3592
|
+
startedAt: Date.now(),
|
|
3593
|
+
previousBootId: bootId,
|
|
3594
|
+
bootId: null,
|
|
3595
|
+
reason: state.restart && typeof state.restart.reason === 'string' ? state.restart.reason : null,
|
|
3596
|
+
snapshot: buildSnapshotRecord({
|
|
3597
|
+
requested: snapshot && snapshot.requested === true,
|
|
3598
|
+
backup,
|
|
3599
|
+
profileDir,
|
|
3600
|
+
durationMs: snapshot && Number.isFinite(snapshot.durationMs) ? snapshot.durationMs : null,
|
|
3601
|
+
error: snapshot && snapshot.error ? snapshot.error : null,
|
|
3602
|
+
}),
|
|
3603
|
+
checks: [],
|
|
3604
|
+
})
|
|
3605
|
+
// The marker of an EARLIER decision must not confuse this restart's
|
|
3606
|
+
// watcher: it would stand down before it even looked at the new
|
|
3607
|
+
// instance. It is removed BEFORE the watcher is armed.
|
|
3608
|
+
try {
|
|
3609
|
+
fs.rmSync(path.join(stateDir, RESTART_OUTCOME_FILE), { force: true })
|
|
3610
|
+
} catch {}
|
|
3611
|
+
|
|
3612
|
+
if (useCustomCommand) {
|
|
3613
|
+
// (0) THE C ROUTE. When the supervisor of this plugin started this
|
|
3614
|
+
// instance, the restart is asked of the SUPERVISOR (over its control
|
|
3615
|
+
// channel) — not of a command. The supervisor kills the host, starts
|
|
3616
|
+
// it again and puts the snapshot back if it does not come.
|
|
3617
|
+
const detected = detectSupervisor({ env: process.env, stateDir })
|
|
3618
|
+
if (detected.supervised) {
|
|
3619
|
+
if (backup && backup.ok === true) {
|
|
3620
|
+
writeRollbackRequest(stateDir, {
|
|
3621
|
+
backupDir: backup.dir,
|
|
3622
|
+
profileDir: profileDir ?? '',
|
|
3623
|
+
generation: state.generation,
|
|
3624
|
+
reason: 'plugin-restart',
|
|
3625
|
+
})
|
|
3626
|
+
}
|
|
3627
|
+
const asked = await requestSupervisorAction({ stateDir, action: 'restart' })
|
|
3628
|
+
if (asked && asked.ok === true) {
|
|
3629
|
+
log.warn('the restart was handed to the supervisor of this plugin')
|
|
3630
|
+
return
|
|
3631
|
+
}
|
|
3632
|
+
log.error(
|
|
3633
|
+
'the supervisor did not accept the restart request; falling back to the restart command:',
|
|
3634
|
+
asked ? asked.error : 'unknown error',
|
|
3635
|
+
)
|
|
3636
|
+
}
|
|
3637
|
+
|
|
3638
|
+
const command = buildCustomCommand({
|
|
3639
|
+
template: config.restartCommand,
|
|
3640
|
+
pid: process.pid,
|
|
3641
|
+
ppid: process.ppid,
|
|
3642
|
+
appExePath: env.appExePath || '',
|
|
3643
|
+
})
|
|
3644
|
+
log.warn('running custom restart command:', command)
|
|
3645
|
+
|
|
3646
|
+
// (1) THE SAFETY NET. The restart command usually kills THIS process
|
|
3647
|
+
// (that is what restarting the host means), so the plugin cannot
|
|
3648
|
+
// confirm anything once the command runs. The watcher is started
|
|
3649
|
+
// BEFORE the command: it waits for the new instance outside the host
|
|
3650
|
+
// and puts the snapshot back when the new instance never comes.
|
|
3651
|
+
//
|
|
3652
|
+
// It is NOT armed when the supervisor of this plugin is in charge:
|
|
3653
|
+
// that one restores the snapshot itself, and two of them would write
|
|
3654
|
+
// the same files at the same time.
|
|
3655
|
+
//
|
|
3656
|
+
// It is NOT armed when the watchdog v2 is armed either: v2 does the
|
|
3657
|
+
// same job AND starts the host again. Two watchdogs take the same
|
|
3658
|
+
// lock, and the one that loses it gives up — if v2 lost, the profile
|
|
3659
|
+
// would be restored but nothing would be running any more.
|
|
3660
|
+
const arbiter = planWatchdogArbiter({
|
|
3661
|
+
v2Armed: watchdogArm !== null && watchdogArm.ok === true,
|
|
3662
|
+
supervised: detected.supervised,
|
|
3663
|
+
})
|
|
3664
|
+
if (arbiter.startV1 === true) {
|
|
3665
|
+
armRollbackWatcher({ backup, profileDir })
|
|
3666
|
+
writeArbiterReport(ARBITERS.V1, arbiter.reason, 'none')
|
|
3667
|
+
} else if (arbiter.arbiter === ARBITERS.V2) {
|
|
3668
|
+
writeArbiterReport(ARBITERS.V2, arbiter.reason, watchdogArm.tier)
|
|
3669
|
+
log.warn('the watchdog v2 takes over: the old rollback watcher is NOT started', {
|
|
3670
|
+
tier: watchdogArm.tier,
|
|
3671
|
+
tierReason: watchdogArm.tierReason || '',
|
|
3672
|
+
})
|
|
3673
|
+
}
|
|
3674
|
+
|
|
3675
|
+
try {
|
|
3676
|
+
// (2) THE COMMAND, with a real success judgement. The old code only
|
|
3677
|
+
// looked at "the spawn did not throw", so a command that does not
|
|
3678
|
+
// exist or that exits with an error was reported as a successful
|
|
3679
|
+
// restart.
|
|
3680
|
+
const outcome = await runCommandFn({
|
|
3681
|
+
command,
|
|
3682
|
+
timeoutMs: config.restartCommandTimeoutMs,
|
|
3683
|
+
})
|
|
3684
|
+
if (!outcome || outcome.ok !== true) {
|
|
3685
|
+
const reason = (outcome && outcome.error) || 'the restart command failed'
|
|
3686
|
+
log.error('the restart command failed; the pre-restart snapshot is restored:', reason)
|
|
3687
|
+
rollbackInProcess({ backup, profileDir, reason })
|
|
3688
|
+
writeRestartOutcome({ decided: true, ok: false, reason })
|
|
3689
|
+
recordRestartVerdict({
|
|
3690
|
+
result: RESTART_RESULTS.FAILED,
|
|
3691
|
+
verifiedAt: Date.now(),
|
|
3692
|
+
reason,
|
|
3693
|
+
checks: [{ name: 'command', ok: false, detail: reason }],
|
|
3694
|
+
})
|
|
3695
|
+
return
|
|
3696
|
+
}
|
|
3697
|
+
if (outcome.timedOut === true) {
|
|
3698
|
+
log.warn(
|
|
3699
|
+
`the restart command is still running after ${config.restartCommandTimeoutMs}ms; `
|
|
3700
|
+
+ 'the new instance is confirmed by the rollback watcher',
|
|
3701
|
+
{ command, timeoutMs: config.restartCommandTimeoutMs },
|
|
3702
|
+
)
|
|
3703
|
+
} else {
|
|
3704
|
+
log.warn('the restart command finished', { command, exitCode: outcome.exitCode ?? null })
|
|
3705
|
+
}
|
|
3706
|
+
|
|
3707
|
+
// (3) THE HEALTH CONFIRMATION — only possible when this process
|
|
3708
|
+
// survived the command. The new instance publishes its own port and
|
|
3709
|
+
// its own boot id; the confirmation waits for THAT instance and not
|
|
3710
|
+
// for the one that is being replaced.
|
|
3711
|
+
const confirmed = await probeInstanceFn({
|
|
3712
|
+
accessFile: accessFilePath(stateDir),
|
|
3713
|
+
expectedBootId: bootId,
|
|
3714
|
+
healthPath: resolveHealthPath(config),
|
|
3715
|
+
timeoutMs: config.rollbackTimeoutMs,
|
|
3716
|
+
})
|
|
3717
|
+
if (confirmed && confirmed.ok === true) {
|
|
3718
|
+
log.warn('the new instance answered its health route:', confirmed)
|
|
3719
|
+
writeRestartOutcome({ decided: true, ok: true, reason: 'the new instance answered' })
|
|
3720
|
+
recordRestartVerdict({
|
|
3721
|
+
result: RESTART_RESULTS.OK,
|
|
3722
|
+
verifiedAt: Date.now(),
|
|
3723
|
+
reason: 'the new instance answered',
|
|
3724
|
+
checks: [{ name: 'command', ok: true, detail: 'the new instance answered its health route' }],
|
|
3725
|
+
}, { skipIfFinalized: true })
|
|
3726
|
+
return
|
|
3727
|
+
}
|
|
3728
|
+
const reason = (confirmed && confirmed.reason) || 'the new instance never answered'
|
|
3729
|
+
log.error('the new instance did not come up; the pre-restart snapshot is restored:', reason)
|
|
3730
|
+
rollbackInProcess({ backup, profileDir, reason })
|
|
3731
|
+
writeRestartOutcome({ decided: true, ok: false, reason })
|
|
3732
|
+
recordRestartVerdict({
|
|
3733
|
+
result: RESTART_RESULTS.FAILED,
|
|
3734
|
+
verifiedAt: Date.now(),
|
|
3735
|
+
reason,
|
|
3736
|
+
checks: [{ name: 'command', ok: false, detail: reason }],
|
|
3737
|
+
})
|
|
3738
|
+
} catch (error) {
|
|
3739
|
+
// The command has already run: a retry would restart the host a
|
|
3740
|
+
// SECOND time. The snapshot is restored and the reason is reported.
|
|
3741
|
+
const reason = 'the hosted restart failed: ' + String(error)
|
|
3742
|
+
log.error(reason)
|
|
3743
|
+
rollbackInProcess({ backup, profileDir, reason })
|
|
3744
|
+
writeRestartOutcome({ decided: true, ok: false, reason })
|
|
3745
|
+
recordRestartVerdict({
|
|
3746
|
+
result: RESTART_RESULTS.FAILED,
|
|
3747
|
+
verifiedAt: Date.now(),
|
|
3748
|
+
reason,
|
|
3749
|
+
checks: [{ name: 'command', ok: false, detail: reason }],
|
|
3750
|
+
})
|
|
3751
|
+
}
|
|
3752
|
+
} else {
|
|
3753
|
+
const aborts = Number(state.restart.gateAborts) || 0
|
|
3754
|
+
const url = aborts > 20 ? null : restartGateUrl()
|
|
3755
|
+
if (aborts > 20) {
|
|
3756
|
+
log.error('falling back to an UNGATED restart after repeated idle-gate aborts', { gateAborts: aborts })
|
|
3757
|
+
} else if (url === null) {
|
|
3758
|
+
log.warn('the web server port is unknown; scheduling an UNGATED restart (the helper cannot re-check the idle gate)')
|
|
3759
|
+
}
|
|
3760
|
+
// The snapshot was already taken above (the shared entry).
|
|
3761
|
+
const healthUrl = restartHealthUrl()
|
|
3762
|
+
const gateScriptPath = url === null ? null : path.join(stateDir, 'restart-' + state.generation + '.cmd')
|
|
3763
|
+
// The watcher does NOT need the port of the old instance (it reads the
|
|
3764
|
+
// port of the NEW one from the instance record), so it is armed as
|
|
3765
|
+
// soon as there is a snapshot and a profile directory.
|
|
3766
|
+
//
|
|
3767
|
+
// ONLY when the watchdog v2 could not be armed: v2 is the better
|
|
3768
|
+
// watcher (it starts the host again), and two watchers take the same
|
|
3769
|
+
// lock — the loser gives up.
|
|
3770
|
+
const arbiter = planWatchdogArbiter({
|
|
3771
|
+
v2Armed: watchdogArm !== null && watchdogArm.ok === true,
|
|
3772
|
+
supervised: false,
|
|
3773
|
+
})
|
|
3774
|
+
const watchScriptPath = arbiter.startV1 === true && backup && backup.ok === true && profileDir !== null
|
|
3775
|
+
? path.join(stateDir, 'restart-' + state.generation + '-watch.ps1')
|
|
3776
|
+
: null
|
|
3777
|
+
if (arbiter.startV1 === true) {
|
|
3778
|
+
writeArbiterReport(ARBITERS.V1, arbiter.reason, 'none')
|
|
3779
|
+
} else if (arbiter.arbiter === ARBITERS.V2) {
|
|
3780
|
+
writeArbiterReport(ARBITERS.V2, arbiter.reason, watchdogArm.tier)
|
|
3781
|
+
log.warn('the watchdog v2 takes over: the old rollback watcher is NOT started', {
|
|
3782
|
+
tier: watchdogArm.tier,
|
|
3783
|
+
tierReason: watchdogArm.tierReason || '',
|
|
3784
|
+
})
|
|
3785
|
+
}
|
|
3786
|
+
const watchScript = watchScriptPath === null
|
|
3787
|
+
? null
|
|
3788
|
+
: buildWatchScript({
|
|
3789
|
+
// The fallback URL of the OLD instance. The watcher uses the port
|
|
3790
|
+
// of the NEW instance (from the instance record) — the desktop
|
|
3791
|
+
// app listens on a random port, so the old one is worthless.
|
|
3792
|
+
healthUrl,
|
|
3793
|
+
healthPath: resolveHealthPath(config),
|
|
3794
|
+
portFile: accessFilePath(stateDir),
|
|
3795
|
+
expectedBootId: bootId,
|
|
3796
|
+
outcomeFile: path.join(stateDir, RESTART_OUTCOME_FILE),
|
|
3797
|
+
backupDir: backup.dir,
|
|
3798
|
+
profileDir,
|
|
3799
|
+
reportPath: path.join(stateDir, ROLLBACK_REPORT_NAME),
|
|
3800
|
+
lockFile: path.join(stateDir, ROLLBACK_LOCK_NAME),
|
|
3801
|
+
timeoutMs: config.rollbackTimeoutMs,
|
|
3802
|
+
rollbackTimeoutMs: config.rollbackCopyTimeoutMs,
|
|
3803
|
+
// Only a crash log written after this point belongs to this restart.
|
|
3804
|
+
since: new Date().toISOString(),
|
|
3805
|
+
// The watcher removes the helper scripts when it ends.
|
|
3806
|
+
selfCleanPaths: [watchScriptPath, gateScriptPath].filter(Boolean),
|
|
3807
|
+
})
|
|
3808
|
+
const result = scheduleDetachedFn({
|
|
3809
|
+
appExePath: env.appExePath,
|
|
3810
|
+
relaunchDelaySeconds: config.relaunchDelaySeconds,
|
|
3811
|
+
gateUrl: url,
|
|
3812
|
+
scriptPath: gateScriptPath,
|
|
3813
|
+
watchScriptPath,
|
|
3814
|
+
watchScript,
|
|
3815
|
+
})
|
|
3816
|
+
log.warn('scheduled detached restart:', result)
|
|
3817
|
+
if (!result || result.ok !== true) rollbackRestart((result && result.error) || 'detached restart refused')
|
|
3818
|
+
else armGateWatchdog()
|
|
3819
|
+
}
|
|
3820
|
+
} catch (error) {
|
|
3821
|
+
rollbackRestart(String(error))
|
|
3822
|
+
}
|
|
3823
|
+
}
|
|
3824
|
+
|
|
3825
|
+
/** One decision tick while a restart is pending. */
|
|
3826
|
+
const tick = () => {
|
|
3827
|
+
if (disposed) return
|
|
3828
|
+
if (!state.restart) {
|
|
3829
|
+
stopPolling()
|
|
3830
|
+
return
|
|
3831
|
+
}
|
|
3832
|
+
const now = Date.now()
|
|
3833
|
+
const busy = busyAgents()
|
|
3834
|
+
// A goal-driven session blocks the restart forever: park its goal BEFORE the
|
|
3835
|
+
// gate decides that it is blocked, so the next poll sees an idle session.
|
|
3836
|
+
if (busy.length > 0) {
|
|
3837
|
+
try {
|
|
2302
3838
|
disarmBlockingGoals(busy)
|
|
2303
3839
|
} catch (error) {
|
|
2304
3840
|
log.warn('could not disarm the goals of the blocking sessions:', String(error))
|
|
@@ -2557,10 +4093,122 @@ const titleOfSummary = (item) => {
|
|
|
2557
4093
|
waitedMs: Number.isFinite(report.waitedMs) ? report.waitedMs : null,
|
|
2558
4094
|
rolledBack: report.rolledBack === true,
|
|
2559
4095
|
backupDir: typeof report.backupDir === 'string' ? report.backupDir : null,
|
|
4096
|
+
// WHY the restart failed. Without it the interface can only say "it was
|
|
4097
|
+
// rolled back", which is the half of the story the user needs most.
|
|
4098
|
+
reason: typeof report.reason === 'string'
|
|
4099
|
+
? report.reason
|
|
4100
|
+
: (typeof report.rollbackError === 'string' ? report.rollbackError : ''),
|
|
2560
4101
|
crashLog: typeof report.crashLog === 'string' ? report.crashLog : null,
|
|
2561
4102
|
crashTail: tail.length > 4000 ? tail.slice(-4000) : tail,
|
|
2562
4103
|
}
|
|
2563
4104
|
},
|
|
4105
|
+
/**
|
|
4106
|
+
* The instance record of the running host: the port it listens on and the
|
|
4107
|
+
* access link (with the boot token) that still works.
|
|
4108
|
+
*
|
|
4109
|
+
* After a restart the OLD link answers 401 — the boot token is created per
|
|
4110
|
+
* process — so this is the link the user needs to get back into the
|
|
4111
|
+
* interface.
|
|
4112
|
+
*/
|
|
4113
|
+
get accessRecord() {
|
|
4114
|
+
const read = readAccessRecord(path.dirname(stateFile))
|
|
4115
|
+
return read && read.ok === true ? read.record : null
|
|
4116
|
+
},
|
|
4117
|
+
/**
|
|
4118
|
+
* What the plugin knows about the self-starting supervisor: who starts
|
|
4119
|
+
* this host (the supervisor of this plugin / the user's own way / nobody)
|
|
4120
|
+
* and whether the restart can actually work.
|
|
4121
|
+
*/
|
|
4122
|
+
supervisorStatus: supervisorStatusOf,
|
|
4123
|
+
/**
|
|
4124
|
+
* HOW this host was started (the startup matrix): the mode, the evidence
|
|
4125
|
+
* and what the plugin may do about it.
|
|
4126
|
+
*/
|
|
4127
|
+
startupMode: startupModeOf,
|
|
4128
|
+
/**
|
|
4129
|
+
* WHAT IS ON DISK RIGHT NOW: the backups of this host.
|
|
4130
|
+
*/
|
|
4131
|
+
backupsInfo,
|
|
4132
|
+
// Where the rescue scripts are, and whether the desktop shortcut exists.
|
|
4133
|
+
rescueKitInfo,
|
|
4134
|
+
// What this machine can do: can it create a one-off scheduled task? (The
|
|
4135
|
+
// automatic rollback hangs on that answer.)
|
|
4136
|
+
capabilitiesInfo,
|
|
4137
|
+
acknowledgeSchedulerWarning,
|
|
4138
|
+
// The watchdog: which tier this machine uses, and what the last run did.
|
|
4139
|
+
watchdogInfo,
|
|
4140
|
+
/**
|
|
4141
|
+
* THE DIAGNOSTICS of "report a problem" (the text the user copies).
|
|
4142
|
+
*/
|
|
4143
|
+
diagnostics: diagnosticsText,
|
|
4144
|
+
/**
|
|
4145
|
+
* WHICH MACHINE IS THIS? (Windows / WSL / Linux / macOS) — the same
|
|
4146
|
+
* answer the graceful shutdown uses.
|
|
4147
|
+
*/
|
|
4148
|
+
hostKind: hostKindOf,
|
|
4149
|
+
/**
|
|
4150
|
+
* Install the supervisor. `confirmed` must be true: the confirmation card
|
|
4151
|
+
* is what says "yes".
|
|
4152
|
+
*/
|
|
4153
|
+
installSupervisor: installSupervisorNow,
|
|
4154
|
+
/**
|
|
4155
|
+
* Remove the supervisor and give the machine back to the user.
|
|
4156
|
+
*/
|
|
4157
|
+
uninstallSupervisor: uninstallSupervisorNow,
|
|
4158
|
+
/**
|
|
4159
|
+
* Record the answer of the one-time question about the supervisor.
|
|
4160
|
+
*/
|
|
4161
|
+
recordSupervisorChoice: recordSupervisorChoice,
|
|
4162
|
+
/**
|
|
4163
|
+
* Derive the command that starts this host again (the dry run: nothing is
|
|
4164
|
+
* written).
|
|
4165
|
+
*/
|
|
4166
|
+
deriveStartCommand: deriveStartCommandNow,
|
|
4167
|
+
/**
|
|
4168
|
+
* Write the derived command into the profile configuration (only after
|
|
4169
|
+
* the user confirmed it).
|
|
4170
|
+
*/
|
|
4171
|
+
applyDerivedCommand: applyDerivedCommand,
|
|
4172
|
+
/**
|
|
4173
|
+
* Put the previous start command back (the undo).
|
|
4174
|
+
*/
|
|
4175
|
+
revertDerivedCommand: revertDerivedCommand,
|
|
4176
|
+
/**
|
|
4177
|
+
* The answer of the one-time question about the supervisor (null when it
|
|
4178
|
+
* was never answered).
|
|
4179
|
+
*/
|
|
4180
|
+
get onboardingChoice() {
|
|
4181
|
+
const record = state.onboarding
|
|
4182
|
+
if (!record || typeof record !== 'object') return null
|
|
4183
|
+
if (record.feature !== 'supervisor') return null
|
|
4184
|
+
return {
|
|
4185
|
+
feature: record.feature,
|
|
4186
|
+
version: record.version,
|
|
4187
|
+
choice: record.choice,
|
|
4188
|
+
answeredAt: record.answeredAt,
|
|
4189
|
+
}
|
|
4190
|
+
},
|
|
4191
|
+
/**
|
|
4192
|
+
* Whether the one-time question about the supervisor may be asked.
|
|
4193
|
+
*
|
|
4194
|
+
* ONLY for the HOSTED hosts: the desktop app brings itself back up
|
|
4195
|
+
* (`restartMode: app`), so asking a hosted user "shall I install a
|
|
4196
|
+
* supervisor?" would be noise there.
|
|
4197
|
+
*
|
|
4198
|
+
* And NEVER when somebody else is already in charge: a second supervisor
|
|
4199
|
+
* would fight over the port with the first one — that is the double start
|
|
4200
|
+
* the whole external-supervisor detection exists to prevent.
|
|
4201
|
+
*/
|
|
4202
|
+
get supervisorOffer() {
|
|
4203
|
+
if (env.isDesktopApp) return false
|
|
4204
|
+
const record = state.onboarding
|
|
4205
|
+
if (record && typeof record === 'object' && record.feature === 'supervisor') return false
|
|
4206
|
+
try {
|
|
4207
|
+
const status = supervisorStatusOf()
|
|
4208
|
+
if (status && status.mode === 'external') return false
|
|
4209
|
+
} catch {}
|
|
4210
|
+
return true
|
|
4211
|
+
},
|
|
2564
4212
|
env,
|
|
2565
4213
|
stateFile,
|
|
2566
4214
|
log,
|
|
@@ -2613,6 +4261,24 @@ const titleOfSummary = (item) => {
|
|
|
2613
4261
|
return { applied: result.applied, showSleepButton: showSleepButton === true }
|
|
2614
4262
|
},
|
|
2615
4263
|
toolErrors,
|
|
4264
|
+
/**
|
|
4265
|
+
* The routes of this plugin that the web server accepted, and the ones it
|
|
4266
|
+
* refused. The boot check reports them: a host that came up without its
|
|
4267
|
+
* routes is NOT a healthy host.
|
|
4268
|
+
*/
|
|
4269
|
+
registeredRoutes,
|
|
4270
|
+
routeErrors,
|
|
4271
|
+
/**
|
|
4272
|
+
* The outcome of the boot recovery pass of THIS boot:
|
|
4273
|
+
* `{ ran, ok, detail }`. `ran` is false when the recovery is switched off.
|
|
4274
|
+
*/
|
|
4275
|
+
recovery: null,
|
|
4276
|
+
/**
|
|
4277
|
+
* The durable record of the last restart (the snapshot that was taken and
|
|
4278
|
+
* the verdict of the instance that came up).
|
|
4279
|
+
*/
|
|
4280
|
+
lastRestartVerdict,
|
|
4281
|
+
recordRestartVerdict,
|
|
2616
4282
|
startPolling,
|
|
2617
4283
|
stopPolling,
|
|
2618
4284
|
tick,
|
|
@@ -3225,6 +4891,137 @@ async function recoverAfterRestart(ctx, runtime, config) {
|
|
|
3225
4891
|
return { recovered: results, phase: 'running' }
|
|
3226
4892
|
}
|
|
3227
4893
|
|
|
4894
|
+
/**
|
|
4895
|
+
* How long a pending restart record stays fresh enough to be finished by
|
|
4896
|
+
* THIS boot. A record that is older belongs to a restart of another day — it
|
|
4897
|
+
* stays `pending`, because that restart really was never confirmed.
|
|
4898
|
+
*/
|
|
4899
|
+
export const RESTART_VERDICT_FRESH_MS = 30 * 60 * 1000
|
|
4900
|
+
|
|
4901
|
+
/**
|
|
4902
|
+
* The four checks the instance that replaced the old one runs on itself.
|
|
4903
|
+
*
|
|
4904
|
+
* They answer the question the operator actually has after a restart: "did
|
|
4905
|
+
* the new host really come up?" — and each of them is a fact about THIS boot,
|
|
4906
|
+
* not about the old one:
|
|
4907
|
+
*
|
|
4908
|
+
* - `listener` — the host listens on a known port;
|
|
4909
|
+
* - `routes` — the routes of this plugin were accepted by the web server;
|
|
4910
|
+
* - `plugin` — the plugin itself is alive in this boot;
|
|
4911
|
+
* - `recovery`— the boot recovery pass finished (or was not needed).
|
|
4912
|
+
*
|
|
4913
|
+
* @param {object} runtime
|
|
4914
|
+
* @returns {Array<{ name: string, ok: boolean, detail: string }>}
|
|
4915
|
+
*/
|
|
4916
|
+
export function collectBootChecks(runtime) {
|
|
4917
|
+
const checks = []
|
|
4918
|
+
|
|
4919
|
+
const port = Number(runtime && runtime.webPort)
|
|
4920
|
+
checks.push({
|
|
4921
|
+
name: 'listener',
|
|
4922
|
+
ok: Number.isFinite(port) && port > 0,
|
|
4923
|
+
detail: Number.isFinite(port) && port > 0
|
|
4924
|
+
? 'the host listens on port ' + port
|
|
4925
|
+
: 'the host does not listen on a known port',
|
|
4926
|
+
})
|
|
4927
|
+
|
|
4928
|
+
const registered = Array.isArray(runtime && runtime.registeredRoutes) ? runtime.registeredRoutes : []
|
|
4929
|
+
const routeErrors = Array.isArray(runtime && runtime.routeErrors) ? runtime.routeErrors : []
|
|
4930
|
+
checks.push({
|
|
4931
|
+
name: 'routes',
|
|
4932
|
+
ok: registered.length > 0 && routeErrors.length === 0,
|
|
4933
|
+
detail: registered.length === 0
|
|
4934
|
+
? 'no route of this plugin was registered'
|
|
4935
|
+
: registered.length + ' route(s) registered'
|
|
4936
|
+
+ (routeErrors.length > 0 ? ', ' + routeErrors.length + ' refused' : ''),
|
|
4937
|
+
})
|
|
4938
|
+
|
|
4939
|
+
const bootId = runtime && typeof runtime.bootId === 'string' ? runtime.bootId : ''
|
|
4940
|
+
const toolErrors = Array.isArray(runtime && runtime.toolErrors) ? runtime.toolErrors : []
|
|
4941
|
+
checks.push({
|
|
4942
|
+
name: 'plugin',
|
|
4943
|
+
ok: bootId !== '' && toolErrors.length === 0,
|
|
4944
|
+
detail: bootId === ''
|
|
4945
|
+
? 'the plugin did not register a boot id'
|
|
4946
|
+
: 'the plugin is alive (boot ' + bootId + ')'
|
|
4947
|
+
+ (toolErrors.length > 0 ? ', ' + toolErrors.length + ' tool(s) failed' : ''),
|
|
4948
|
+
})
|
|
4949
|
+
|
|
4950
|
+
const recovery = runtime && runtime.recovery && typeof runtime.recovery === 'object'
|
|
4951
|
+
? runtime.recovery
|
|
4952
|
+
: null
|
|
4953
|
+
checks.push({
|
|
4954
|
+
name: 'recovery',
|
|
4955
|
+
ok: recovery === null ? false : recovery.ok === true,
|
|
4956
|
+
detail: recovery === null
|
|
4957
|
+
? 'the boot recovery did not report anything'
|
|
4958
|
+
: (typeof recovery.detail === 'string' && recovery.detail !== '' ? recovery.detail : 'the boot recovery finished'),
|
|
4959
|
+
})
|
|
4960
|
+
|
|
4961
|
+
return checks
|
|
4962
|
+
}
|
|
4963
|
+
|
|
4964
|
+
/**
|
|
4965
|
+
* Finish the durable restart record.
|
|
4966
|
+
*
|
|
4967
|
+
* The instance that replaced the old one reads the `pending` record, runs the
|
|
4968
|
+
* four boot checks and writes the final verdict (`ok` / `failed`) with the
|
|
4969
|
+
* time it verified itself and how long the whole restart took.
|
|
4970
|
+
*
|
|
4971
|
+
* Nothing is written when there is no pending record (this boot was not the
|
|
4972
|
+
* result of a restart) or when the pending record belongs to another day.
|
|
4973
|
+
*
|
|
4974
|
+
* @param {object} runtime
|
|
4975
|
+
* @param {{ timeoutMs?: number, now?: number, sleep?: Function }} [options]
|
|
4976
|
+
* @returns {Promise<object|null>} the record that is on disk now
|
|
4977
|
+
*/
|
|
4978
|
+
export async function verifyRestartBoot(runtime, options = {}) {
|
|
4979
|
+
const log = runtime.log
|
|
4980
|
+
const stateDir = path.dirname(runtime.stateFile)
|
|
4981
|
+
const pending = readRestartVerdict(stateDir)
|
|
4982
|
+
if (pending === null) return null
|
|
4983
|
+
if (pending.result !== RESTART_RESULTS.PENDING) return pending
|
|
4984
|
+
const now = typeof options.now === 'function' ? options.now : () => Date.now()
|
|
4985
|
+
const timeoutMs = Number.isFinite(Number(options.timeoutMs)) && Number(options.timeoutMs) > 0
|
|
4986
|
+
? Math.trunc(Number(options.timeoutMs))
|
|
4987
|
+
: 15000
|
|
4988
|
+
const sleep = typeof options.sleep === 'function' ? options.sleep : recoveryDelay
|
|
4989
|
+
const started = now()
|
|
4990
|
+
// The web server may still be coming up: the checks need its port and its
|
|
4991
|
+
// routes, so they wait — but only for a bounded while.
|
|
4992
|
+
while (now() - started < timeoutMs) {
|
|
4993
|
+
const port = Number(runtime.webPort)
|
|
4994
|
+
const registered = Array.isArray(runtime.registeredRoutes) ? runtime.registeredRoutes.length : 0
|
|
4995
|
+
if (Number.isFinite(port) && port > 0 && registered > 0) break
|
|
4996
|
+
await sleep(250)
|
|
4997
|
+
}
|
|
4998
|
+
const checks = collectBootChecks(runtime)
|
|
4999
|
+
const evaluated = evaluateRestartChecks(checks)
|
|
5000
|
+
const verifiedAt = now()
|
|
5001
|
+
const record = runtime.recordRestartVerdict({
|
|
5002
|
+
result: evaluated.ok ? RESTART_RESULTS.OK : RESTART_RESULTS.FAILED,
|
|
5003
|
+
verifiedAt,
|
|
5004
|
+
durationMs: pending.startedAt !== null ? Math.max(0, verifiedAt - pending.startedAt) : null,
|
|
5005
|
+
bootId: runtime.bootId,
|
|
5006
|
+
checks,
|
|
5007
|
+
reason: evaluated.ok
|
|
5008
|
+
? 'the new instance came up and every check passed'
|
|
5009
|
+
: 'the new instance came up, but ' + evaluated.failed.join(', ') + ' failed',
|
|
5010
|
+
})
|
|
5011
|
+
if (record !== null) {
|
|
5012
|
+
if (evaluated.ok) {
|
|
5013
|
+
log.warn('the new instance verified itself:', { durationMs: record.durationMs, checks })
|
|
5014
|
+
} else {
|
|
5015
|
+
log.error('the new instance did NOT come up cleanly:', {
|
|
5016
|
+
durationMs: record.durationMs,
|
|
5017
|
+
failed: evaluated.failed,
|
|
5018
|
+
checks,
|
|
5019
|
+
})
|
|
5020
|
+
}
|
|
5021
|
+
}
|
|
5022
|
+
return record
|
|
5023
|
+
}
|
|
5024
|
+
|
|
3228
5025
|
/** Promise-based pause used by boot recovery (batching + resolveAgent retries). */
|
|
3229
5026
|
function recoveryDelay(ms) {
|
|
3230
5027
|
const wait = Number.isFinite(ms) && ms > 0 ? ms : 0
|
|
@@ -3436,6 +5233,11 @@ function registerTools(ctx, runtime, config) {
|
|
|
3436
5233
|
profileDirSource: { type: 'string' },
|
|
3437
5234
|
backupIncludePlugins: { type: 'boolean' },
|
|
3438
5235
|
lastRollback: { type: 'object' },
|
|
5236
|
+
// The last restart: the snapshot that was taken before it and the
|
|
5237
|
+
// verdict of the instance that came up.
|
|
5238
|
+
lastRestart: { type: 'object' },
|
|
5239
|
+
// The access link of the running instance (with the boot token).
|
|
5240
|
+
accessUrl: { type: 'string' },
|
|
3439
5241
|
environment: { type: 'object' },
|
|
3440
5242
|
sessions: { type: 'array', items: { type: 'object' } },
|
|
3441
5243
|
},
|
|
@@ -3472,6 +5274,15 @@ function registerTools(ctx, runtime, config) {
|
|
|
3472
5274
|
backupIncludePlugins: config.backupIncludePlugins !== false,
|
|
3473
5275
|
// Present only when the previous restart had to be rolled back.
|
|
3474
5276
|
...(runtime.lastRollback ? { lastRollback: runtime.lastRollback } : {}),
|
|
5277
|
+
// THE LAST RESTART: the snapshot that was taken before it (directory,
|
|
5278
|
+
// LOCAL time, duration, pruned) and the verdict of the instance that
|
|
5279
|
+
// came up (the four boot checks). Present only when a restart
|
|
5280
|
+
// happened — otherwise there is nothing to report.
|
|
5281
|
+
...(runtime.lastRestartVerdict() ? { lastRestart: runtime.lastRestartVerdict() } : {}),
|
|
5282
|
+
// The access link of THIS instance (with the boot token). The boot
|
|
5283
|
+
// token is created per process, so the link the browser holds answers
|
|
5284
|
+
// 401 after a restart — this is the link that still works.
|
|
5285
|
+
...(accessUrlOf(runtime) ? { accessUrl: accessUrlOf(runtime) } : {}),
|
|
3475
5286
|
environment: {
|
|
3476
5287
|
platform: runtime.env.platform,
|
|
3477
5288
|
isDesktopApp: runtime.env.isDesktopApp,
|
|
@@ -3647,11 +5458,19 @@ function registerTools(ctx, runtime, config) {
|
|
|
3647
5458
|
|
|
3648
5459
|
tool({
|
|
3649
5460
|
name: 'continuum_cancel_restart',
|
|
3650
|
-
description:
|
|
5461
|
+
description:
|
|
5462
|
+
'Cancel a restart requested through continuum_restart. By default ONLY the calling session\'s own request is '
|
|
5463
|
+
+ 'withdrawn (scope "mine"): the other sessions that asked for the same restart keep it. Use scope "all" to '
|
|
5464
|
+
+ 'cancel the request for EVERYONE ("nobody restarts any more"). Does nothing when no restart is pending.',
|
|
3651
5465
|
parameters: {
|
|
3652
5466
|
type: 'object',
|
|
3653
5467
|
additionalProperties: false,
|
|
3654
5468
|
properties: {
|
|
5469
|
+
scope: {
|
|
5470
|
+
type: 'string',
|
|
5471
|
+
enum: ['mine', 'all'],
|
|
5472
|
+
description: 'Whose request to cancel: "mine" (the default) withdraws only the calling session\'s own request and leaves the requests of the other sessions alone; "all" cancels the request for every session.',
|
|
5473
|
+
},
|
|
3655
5474
|
reason: { type: 'string', description: 'Optional note recorded in the log.' },
|
|
3656
5475
|
},
|
|
3657
5476
|
},
|
|
@@ -3663,20 +5482,248 @@ function registerTools(ctx, runtime, config) {
|
|
|
3663
5482
|
properties: {
|
|
3664
5483
|
ok: { type: 'boolean' },
|
|
3665
5484
|
cancelled: { type: 'boolean' },
|
|
5485
|
+
scope: { type: 'string' },
|
|
5486
|
+
remainingRequesters: { type: 'integer' },
|
|
5487
|
+
sessionId: { type: 'string' },
|
|
3666
5488
|
phase: { type: 'string' },
|
|
5489
|
+
error: { type: 'string' },
|
|
3667
5490
|
},
|
|
3668
5491
|
},
|
|
3669
5492
|
(_args, value) => [{ type: 'text', text: JSON.stringify(value) }],
|
|
3670
5493
|
),
|
|
3671
|
-
execute(args = {}) {
|
|
5494
|
+
execute(args = {}, exec) {
|
|
3672
5495
|
const state = runtime.state
|
|
3673
|
-
const
|
|
3674
|
-
|
|
3675
|
-
|
|
3676
|
-
|
|
5496
|
+
const now = Date.now()
|
|
5497
|
+
const scope = args.scope === 'all' ? 'all' : 'mine'
|
|
5498
|
+
const agent = exec && exec.agent
|
|
5499
|
+
const sessionId = agent && agent.id ? String(agent.id) : ''
|
|
5500
|
+
const reason = typeof args.reason === 'string' && args.reason.trim() !== '' ? args.reason.trim() : ''
|
|
5501
|
+
|
|
5502
|
+
// How many sessions are waiting for the pending restart.
|
|
5503
|
+
const requesterCount = () => {
|
|
5504
|
+
const restart = state.restart
|
|
5505
|
+
if (!restart) return 0
|
|
5506
|
+
const ids = new Set()
|
|
5507
|
+
if (Array.isArray(restart.requesters)) {
|
|
5508
|
+
for (const requester of restart.requesters) {
|
|
5509
|
+
if (requester && typeof requester.sessionId === 'string' && requester.sessionId) ids.add(requester.sessionId)
|
|
5510
|
+
}
|
|
5511
|
+
}
|
|
5512
|
+
if (typeof restart.bySessionId === 'string' && restart.bySessionId) ids.add(restart.bySessionId)
|
|
5513
|
+
return ids.size
|
|
5514
|
+
}
|
|
5515
|
+
|
|
5516
|
+
if (scope === 'all') {
|
|
5517
|
+
// The explicit "nobody restarts any more": the whole request goes.
|
|
5518
|
+
const hadPending = state.restart !== null
|
|
5519
|
+
runtime.releaseRequesters()
|
|
5520
|
+
cancelRestart(state, { now })
|
|
5521
|
+
runtime.stopPolling()
|
|
5522
|
+
runtime.persist()
|
|
5523
|
+
if (hadPending) log.info('pending restart cancelled for EVERYONE:', reason || '(no reason)')
|
|
5524
|
+
return { ok: true, cancelled: hadPending, scope: 'all', remainingRequesters: 0, phase: state.phase }
|
|
5525
|
+
}
|
|
5526
|
+
|
|
5527
|
+
// scope "mine": withdraw ONLY the calling session's request.
|
|
5528
|
+
if (state.restart === null) {
|
|
5529
|
+
// Nothing is pending: the tool does nothing, as before.
|
|
5530
|
+
return { ok: true, cancelled: false, scope: 'mine', remainingRequesters: 0, phase: state.phase }
|
|
5531
|
+
}
|
|
5532
|
+
if (sessionId === '') {
|
|
5533
|
+
// The calling session is unknown: refuse instead of guessing. A
|
|
5534
|
+
// silent "cancel everything" is exactly the accident this scope
|
|
5535
|
+
// exists to prevent.
|
|
5536
|
+
log.warn('the restart request of the calling session could not be withdrawn: the calling session is unknown; nothing was cancelled')
|
|
5537
|
+
return {
|
|
5538
|
+
ok: false,
|
|
5539
|
+
cancelled: false,
|
|
5540
|
+
scope: 'mine',
|
|
5541
|
+
remainingRequesters: requesterCount(),
|
|
5542
|
+
phase: state.phase,
|
|
5543
|
+
error: 'The calling session is unknown, so its own request cannot be identified. Nothing was cancelled. Call this tool from the session that requested the restart, or pass scope "all" to cancel the request for everyone.',
|
|
5544
|
+
}
|
|
5545
|
+
}
|
|
5546
|
+
const outcome = cancelOwnRestart(state, { sessionId, now })
|
|
5547
|
+
if (!outcome.removed) {
|
|
5548
|
+
log.warn(`session ${sessionId} did not request the pending restart; nothing was cancelled`)
|
|
5549
|
+
return {
|
|
5550
|
+
ok: false,
|
|
5551
|
+
cancelled: false,
|
|
5552
|
+
scope: 'mine',
|
|
5553
|
+
remainingRequesters: outcome.remaining,
|
|
5554
|
+
sessionId,
|
|
5555
|
+
phase: state.phase,
|
|
5556
|
+
error: 'This session did not request the pending restart, so nothing was cancelled. Pass scope "all" to cancel the request for every session.',
|
|
5557
|
+
}
|
|
5558
|
+
}
|
|
5559
|
+
// Release ONLY this session's continuation request: the other sessions
|
|
5560
|
+
// stay pinned for the restart they asked for.
|
|
5561
|
+
runtime.releaseRequesters([sessionId])
|
|
5562
|
+
if (outcome.cancelled) {
|
|
5563
|
+
// The last requester left: the restart is really over.
|
|
5564
|
+
runtime.stopPolling()
|
|
5565
|
+
log.info(`session ${sessionId} withdrew the last restart request; the pending restart is cancelled`, reason || '')
|
|
5566
|
+
} else {
|
|
5567
|
+
log.info(`session ${sessionId} withdrew its restart request; ${outcome.remaining} requester(s) remain`)
|
|
5568
|
+
}
|
|
3677
5569
|
runtime.persist()
|
|
3678
|
-
|
|
3679
|
-
|
|
5570
|
+
return {
|
|
5571
|
+
ok: true,
|
|
5572
|
+
cancelled: outcome.cancelled,
|
|
5573
|
+
scope: 'mine',
|
|
5574
|
+
remainingRequesters: outcome.remaining,
|
|
5575
|
+
sessionId,
|
|
5576
|
+
phase: state.phase,
|
|
5577
|
+
}
|
|
5578
|
+
},
|
|
5579
|
+
})
|
|
5580
|
+
|
|
5581
|
+
tool({
|
|
5582
|
+
name: 'continuum_supervisor',
|
|
5583
|
+
description:
|
|
5584
|
+
'Inspect and manage the self-starting supervisor of this plugin (the process that brings the host back when it '
|
|
5585
|
+
+ 'dies). action "status" reports who starts this host (the supervisor of this plugin / your own way / nobody) '
|
|
5586
|
+
+ 'and whether the restart can work at all; action "plan" reports what an installation WOULD change without '
|
|
5587
|
+
+ 'touching anything; action "install" and action "uninstall" need confirmed: true, because they really change '
|
|
5588
|
+
+ 'the machine (a scheduled task or a systemd user unit that starts the supervisor at login).',
|
|
5589
|
+
parameters: {
|
|
5590
|
+
type: 'object',
|
|
5591
|
+
additionalProperties: false,
|
|
5592
|
+
properties: {
|
|
5593
|
+
action: {
|
|
5594
|
+
type: 'string',
|
|
5595
|
+
enum: ['status', 'plan', 'dry-run', 'install', 'uninstall', 'apply', 'revert'],
|
|
5596
|
+
description: 'What to do: "status" (read-only), "plan" (what the installation would change), "dry-run" (the command this host would be restarted with, and why it may not be derivable), "install"/"uninstall" (the real thing, they need confirmed: true), "apply" (write the derived restart command after the user confirmed the dry run) and "revert" (put the previous command back).',
|
|
5597
|
+
},
|
|
5598
|
+
confirmed: {
|
|
5599
|
+
type: 'boolean',
|
|
5600
|
+
description: 'Required for "install" and "uninstall": the user confirmed the card that says what would change.',
|
|
5601
|
+
},
|
|
5602
|
+
choice: {
|
|
5603
|
+
type: 'string',
|
|
5604
|
+
enum: ['install', 'external', 'dismissed'],
|
|
5605
|
+
description: 'The answer of the one-time question about the supervisor (recorded so the question is never asked twice).',
|
|
5606
|
+
},
|
|
5607
|
+
},
|
|
5608
|
+
},
|
|
5609
|
+
output: textOutput(
|
|
5610
|
+
{
|
|
5611
|
+
type: 'object',
|
|
5612
|
+
additionalProperties: false,
|
|
5613
|
+
required: ['ok', 'action'],
|
|
5614
|
+
properties: {
|
|
5615
|
+
ok: { type: 'boolean' },
|
|
5616
|
+
action: { type: 'string' },
|
|
5617
|
+
kind: { type: 'string' },
|
|
5618
|
+
degraded: { type: 'boolean' },
|
|
5619
|
+
reason: { type: 'string' },
|
|
5620
|
+
error: { type: 'string' },
|
|
5621
|
+
supervisor: { type: 'object' },
|
|
5622
|
+
steps: { type: 'array', items: { type: 'object' } },
|
|
5623
|
+
previous: { type: 'object' },
|
|
5624
|
+
pending: { type: 'object' },
|
|
5625
|
+
},
|
|
5626
|
+
},
|
|
5627
|
+
(_args, value) => [{ type: 'text', text: JSON.stringify(value, null, 2) }],
|
|
5628
|
+
),
|
|
5629
|
+
async execute(args = {}) {
|
|
5630
|
+
const action = typeof args.action === 'string' ? args.action.trim() : 'status'
|
|
5631
|
+
const supervisor = runtime.supervisorStatus()
|
|
5632
|
+
if (action === 'status') {
|
|
5633
|
+
return { ok: true, action, supervisor }
|
|
5634
|
+
}
|
|
5635
|
+
if (action === 'dry-run' || action === 'derive') {
|
|
5636
|
+
// B: the dry run. It reports what WOULD be written; it writes nothing.
|
|
5637
|
+
const plan = await runtime.deriveStartCommand()
|
|
5638
|
+
return {
|
|
5639
|
+
ok: plan.ok === true,
|
|
5640
|
+
action,
|
|
5641
|
+
level: plan.level,
|
|
5642
|
+
mode: plan.mode,
|
|
5643
|
+
reason: plan.reason,
|
|
5644
|
+
command: plan.command,
|
|
5645
|
+
plan: plan.plan,
|
|
5646
|
+
needsConfirmation: plan.needsConfirmation !== false,
|
|
5647
|
+
writes: plan.writes,
|
|
5648
|
+
compose: plan.compose,
|
|
5649
|
+
supervisor,
|
|
5650
|
+
}
|
|
5651
|
+
}
|
|
5652
|
+
if (action === 'plan') {
|
|
5653
|
+
const plan = planSupervisorInstall({
|
|
5654
|
+
platform: runtime.env.platform,
|
|
5655
|
+
// The desktop app manages itself: the plan is refused there, and no
|
|
5656
|
+
// step is generated that could be run by accident.
|
|
5657
|
+
isDesktopApp: runtime.env.isDesktopApp === true,
|
|
5658
|
+
// The startup matrix: only a `bare` host may get a supervisor.
|
|
5659
|
+
startup: runtime.startupMode(),
|
|
5660
|
+
// WHICH MACHINE IS THIS? (WSL is not a Linux.)
|
|
5661
|
+
host: runtime.hostKind(),
|
|
5662
|
+
nodePath: process.execPath,
|
|
5663
|
+
execPath: process.execPath,
|
|
5664
|
+
supervisorPath: supervisorPaths(path.dirname(runtime.stateFile)).script,
|
|
5665
|
+
configPath: supervisorPaths(path.dirname(runtime.stateFile)).config,
|
|
5666
|
+
kind: config.supervisorKind,
|
|
5667
|
+
})
|
|
5668
|
+
return {
|
|
5669
|
+
ok: plan.error ? false : true,
|
|
5670
|
+
action,
|
|
5671
|
+
kind: plan.kind,
|
|
5672
|
+
reason: plan.reason,
|
|
5673
|
+
error: plan.error ?? undefined,
|
|
5674
|
+
steps: plan.steps,
|
|
5675
|
+
supervisor,
|
|
5676
|
+
}
|
|
5677
|
+
}
|
|
5678
|
+
if (action === 'install' && runtime.env.isDesktopApp === true) {
|
|
5679
|
+
// The desktop app manages itself: a login record would open a SECOND
|
|
5680
|
+
// client at every login. The refusal comes BEFORE the confirmation, so
|
|
5681
|
+
// the user is never asked to confirm something that cannot happen.
|
|
5682
|
+
return {
|
|
5683
|
+
ok: false,
|
|
5684
|
+
action,
|
|
5685
|
+
error: 'desktop-manages-itself',
|
|
5686
|
+
reason: 'the desktop app relaunches itself; a supervisor would start a second copy',
|
|
5687
|
+
steps: [],
|
|
5688
|
+
supervisor,
|
|
5689
|
+
}
|
|
5690
|
+
}
|
|
5691
|
+
if (args.confirmed !== true) {
|
|
5692
|
+
// The card is what says "yes": without it the machine is not touched.
|
|
5693
|
+
return {
|
|
5694
|
+
ok: false,
|
|
5695
|
+
action,
|
|
5696
|
+
error: 'confirmation-required',
|
|
5697
|
+
reason: 'the installation and the uninstallation change the machine, so the user has to confirm the card first',
|
|
5698
|
+
supervisor,
|
|
5699
|
+
}
|
|
5700
|
+
}
|
|
5701
|
+
if (action === 'install') {
|
|
5702
|
+
const choice = typeof args.choice === 'string' ? args.choice : ''
|
|
5703
|
+
if (choice !== '') runtime.recordSupervisorChoice(choice)
|
|
5704
|
+
const result = runtime.installSupervisor({ confirmed: true })
|
|
5705
|
+
return {
|
|
5706
|
+
ok: result.ok === true,
|
|
5707
|
+
action,
|
|
5708
|
+
kind: result.kind,
|
|
5709
|
+
degraded: result.degraded,
|
|
5710
|
+
reason: result.reason,
|
|
5711
|
+
error: result.error ?? undefined,
|
|
5712
|
+
steps: result.steps,
|
|
5713
|
+
supervisor: runtime.supervisorStatus(),
|
|
5714
|
+
}
|
|
5715
|
+
}
|
|
5716
|
+
const result = await runtime.uninstallSupervisor()
|
|
5717
|
+
return {
|
|
5718
|
+
ok: result.ok === true,
|
|
5719
|
+
action,
|
|
5720
|
+
reason: result.ok === true ? 'the supervisor was removed' : undefined,
|
|
5721
|
+
error: result.error ?? undefined,
|
|
5722
|
+
steps: result.steps,
|
|
5723
|
+
previous: result.previous ?? undefined,
|
|
5724
|
+
pending: result.pending ?? undefined,
|
|
5725
|
+
supervisor: runtime.supervisorStatus(),
|
|
5726
|
+
}
|
|
3680
5727
|
},
|
|
3681
5728
|
})
|
|
3682
5729
|
}
|
|
@@ -3719,7 +5766,7 @@ const AUTORESTART_PROMPT = [
|
|
|
3719
5766
|
'- Call the `continuum_restart` tool instead (optional arguments: `reason`, `resume`, `defer_seconds`, `parkGoals`). It waits until every conversation is idle (no running turn — a running turn ALWAYS blocks; an inbox message nobody claimed blocks only until `queuedGraceMs`, default 60000 ms, after which it is treated as stuck and no longer blocks) AND the host has been quiet for `settleMs` (default 2000 ms); the detached helper re-checks that gate immediately before killing, so other conversations are never killed mid-turn. It then relaunches the app, and after boot this plugin resumes the session and continues the work automatically.',
|
|
3720
5767
|
'- A goal-driven session is DISARMED automatically before the restart (its current round still finishes, nothing is lost), because such a session would otherwise keep the restart waiting forever: disarming only removes the process-local continuation authority, so no new round starts and the session goes idle. After the restart the goal is re-armed and continues. A goal whose round budget is spent cannot be continued and is reported instead. You can disarm your own goal on demand with `continuum_park_goal`, and retry a stuck recovery with `continuum_recover_now`.',
|
|
3721
5768
|
'- Persist anything the next turn needs before restarting (workspace files, `todo_write`, goal state); do not rely on in-memory state.',
|
|
3722
|
-
'- Inspect the lifecycle with `continuum_status`; cancel a pending restart with `continuum_cancel_restart`.',
|
|
5769
|
+
'- Inspect the lifecycle with `continuum_status`; cancel a pending restart with `continuum_cancel_restart`. Its default scope is `mine`: it withdraws ONLY the calling session\'s own request and leaves the requests of the other sessions alone (a restart request is coalesced, so several sessions can wait for the same restart). Use `scope: "all"` to cancel the request for EVERYONE.',
|
|
3723
5770
|
'- A session whose context exceeds the model window can be handed off to a fresh session with `continuum_handoff` (it carries the goal objective, the recorded reasons and a bounded tail of the conversation); this plugin refuses to auto-resume such a session, so hand it off instead of retrying it.',
|
|
3724
5771
|
'- For multi-turn objectives prefer a durable goal (`create_goal`): after a restart the plugin re-arms it and the goal round driver keeps producing rounds. No goal is required for restart continuation — without one the plugin injects a single continuation message that continues the work in a fresh turn.',
|
|
3725
5772
|
].join('\n')
|
|
@@ -4021,28 +6068,146 @@ function buildParkRoute(ctx, runtime, config) {
|
|
|
4021
6068
|
reported.push({ sessionId: target, disarmed: false, reason: 'disarm-failed', error: describeError(error) })
|
|
4022
6069
|
}
|
|
4023
6070
|
}
|
|
4024
|
-
if (restart && sessionId !== '') {
|
|
4025
|
-
// The session becomes a requester of the pending restart: the restart
|
|
4026
|
-
// proceeds and the session continues afterwards.
|
|
4027
|
-
requestRestart(runtime.state, {
|
|
4028
|
-
reason: 'disarm route requested a restart',
|
|
4029
|
-
bySessionId: sessionId,
|
|
4030
|
-
resume: true,
|
|
4031
|
-
now: Date.now(),
|
|
6071
|
+
if (restart && sessionId !== '') {
|
|
6072
|
+
// The session becomes a requester of the pending restart: the restart
|
|
6073
|
+
// proceeds and the session continues afterwards.
|
|
6074
|
+
requestRestart(runtime.state, {
|
|
6075
|
+
reason: 'disarm route requested a restart',
|
|
6076
|
+
bySessionId: sessionId,
|
|
6077
|
+
resume: true,
|
|
6078
|
+
now: Date.now(),
|
|
6079
|
+
})
|
|
6080
|
+
runtime.persist()
|
|
6081
|
+
runtime.startPolling()
|
|
6082
|
+
}
|
|
6083
|
+
res.statusCode = 200
|
|
6084
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6085
|
+
res.setHeader('cache-control', 'no-store')
|
|
6086
|
+
res.end(JSON.stringify({
|
|
6087
|
+
ok: true,
|
|
6088
|
+
disarmed,
|
|
6089
|
+
reported,
|
|
6090
|
+
pendingRestart: runtime.state.restart !== null,
|
|
6091
|
+
restartRequested: restart === true && sessionId !== '',
|
|
6092
|
+
}))
|
|
6093
|
+
}).catch((error) => {
|
|
6094
|
+
res.statusCode = 400
|
|
6095
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6096
|
+
res.end(JSON.stringify({ ok: false, error: describeError(error) }))
|
|
6097
|
+
})
|
|
6098
|
+
},
|
|
6099
|
+
}
|
|
6100
|
+
}
|
|
6101
|
+
|
|
6102
|
+
/**
|
|
6103
|
+
* Build the POST `/cancel` route: withdraw ONE session's own restart request.
|
|
6104
|
+
*
|
|
6105
|
+
* It is the sibling of the `/park` route next to the health route and uses the
|
|
6106
|
+
* same authentication rule (loopback OR a request the connection service
|
|
6107
|
+
* admits). Body: `{ "sessionId": string, "scope"?: "mine" }`.
|
|
6108
|
+
*
|
|
6109
|
+
* The route NEVER cancels the request of everybody: `scope: "all"` is
|
|
6110
|
+
* refused, that operation belongs to the tool (where the caller says it out
|
|
6111
|
+
* loud). The answer reports what happened: `{ ok, cancelled, reason,
|
|
6112
|
+
* remainingRequesters, sessionId }`.
|
|
6113
|
+
*
|
|
6114
|
+
* @param {object} ctx
|
|
6115
|
+
* @param {object} runtime
|
|
6116
|
+
* @param {object} config
|
|
6117
|
+
*/
|
|
6118
|
+
function buildCancelRoute(ctx, runtime, config) {
|
|
6119
|
+
const routePath = resolveCancelPath(config)
|
|
6120
|
+
return {
|
|
6121
|
+
kind: 'exact',
|
|
6122
|
+
path: routePath,
|
|
6123
|
+
handler: (req, res) => {
|
|
6124
|
+
const remote = req && req.socket ? req.socket.remoteAddress : undefined
|
|
6125
|
+
const loopback = remote === undefined
|
|
6126
|
+
|| remote === '127.0.0.1'
|
|
6127
|
+
|| remote === '::1'
|
|
6128
|
+
|| remote === '::ffff:127.0.0.1'
|
|
6129
|
+
if (!loopback && !admittedByConnection(ctx, req)) {
|
|
6130
|
+
res.statusCode = 403
|
|
6131
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6132
|
+
res.end('{"ok":false,"error":"loopback-only"}')
|
|
6133
|
+
return
|
|
6134
|
+
}
|
|
6135
|
+
if (!req || req.method !== 'POST') {
|
|
6136
|
+
res.statusCode = 405
|
|
6137
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6138
|
+
res.setHeader('allow', 'POST')
|
|
6139
|
+
res.end('{"ok":false,"error":"method-not-allowed"}')
|
|
6140
|
+
return
|
|
6141
|
+
}
|
|
6142
|
+
void readJsonBody(req).then((payload) => {
|
|
6143
|
+
const body = payload && typeof payload === 'object' ? payload : {}
|
|
6144
|
+
const sessionId = typeof body.sessionId === 'string' ? body.sessionId.trim() : ''
|
|
6145
|
+
const scope = typeof body.scope === 'string' && body.scope.trim() !== '' ? body.scope.trim() : 'mine'
|
|
6146
|
+
const answer = (statusCode, value) => {
|
|
6147
|
+
res.statusCode = statusCode
|
|
6148
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6149
|
+
res.setHeader('cache-control', 'no-store')
|
|
6150
|
+
res.end(JSON.stringify(value))
|
|
6151
|
+
}
|
|
6152
|
+
if (scope !== 'mine') {
|
|
6153
|
+
// "Cancel everybody" is a tool operation: the route says no.
|
|
6154
|
+
answer(400, {
|
|
6155
|
+
ok: false,
|
|
6156
|
+
cancelled: false,
|
|
6157
|
+
reason: 'scope-not-allowed',
|
|
6158
|
+
error: 'This route can only withdraw one session\'s own request (scope "mine").',
|
|
6159
|
+
remainingRequesters: 0,
|
|
6160
|
+
sessionId,
|
|
4032
6161
|
})
|
|
4033
|
-
|
|
4034
|
-
runtime.startPolling()
|
|
6162
|
+
return
|
|
4035
6163
|
}
|
|
4036
|
-
|
|
4037
|
-
|
|
4038
|
-
|
|
4039
|
-
|
|
6164
|
+
if (sessionId === '') {
|
|
6165
|
+
answer(400, {
|
|
6166
|
+
ok: false,
|
|
6167
|
+
cancelled: false,
|
|
6168
|
+
reason: 'session-id-required',
|
|
6169
|
+
error: 'The session id is required: without it the own request cannot be identified.',
|
|
6170
|
+
remainingRequesters: 0,
|
|
6171
|
+
sessionId: '',
|
|
6172
|
+
})
|
|
6173
|
+
return
|
|
6174
|
+
}
|
|
6175
|
+
const outcome = cancelOwnRestart(runtime.state, { sessionId, now: Date.now() })
|
|
6176
|
+
if (!outcome.removed) {
|
|
6177
|
+
// Nothing was changed: the caller is not a requester (or the
|
|
6178
|
+
// request is already gone). The reason says which.
|
|
6179
|
+
const error = outcome.reason === 'not-a-requester'
|
|
6180
|
+
? 'This session did not request the pending restart, so nothing was cancelled.'
|
|
6181
|
+
: outcome.reason === 'no-restart-pending'
|
|
6182
|
+
? 'There is no pending restart to cancel.'
|
|
6183
|
+
: 'The own request could not be identified, so nothing was cancelled.'
|
|
6184
|
+
runtime.log.warn(`the cancel route refused the request of session ${sessionId}: ${outcome.reason}`)
|
|
6185
|
+
answer(200, {
|
|
6186
|
+
ok: false,
|
|
6187
|
+
cancelled: false,
|
|
6188
|
+
reason: outcome.reason,
|
|
6189
|
+
error,
|
|
6190
|
+
remainingRequesters: outcome.remaining,
|
|
6191
|
+
sessionId,
|
|
6192
|
+
})
|
|
6193
|
+
return
|
|
6194
|
+
}
|
|
6195
|
+
// Release ONLY this session's continuation request.
|
|
6196
|
+
runtime.releaseRequesters([sessionId])
|
|
6197
|
+
if (outcome.cancelled) runtime.stopPolling()
|
|
6198
|
+
runtime.persist()
|
|
6199
|
+
runtime.log.info(
|
|
6200
|
+
outcome.cancelled
|
|
6201
|
+
? `session ${sessionId} withdrew the last restart request through the cancel route; the pending restart is cancelled`
|
|
6202
|
+
: `session ${sessionId} withdrew its restart request through the cancel route; ${outcome.remaining} requester(s) remain`,
|
|
6203
|
+
)
|
|
6204
|
+
answer(200, {
|
|
4040
6205
|
ok: true,
|
|
4041
|
-
|
|
4042
|
-
|
|
4043
|
-
|
|
4044
|
-
|
|
4045
|
-
})
|
|
6206
|
+
cancelled: outcome.cancelled,
|
|
6207
|
+
reason: outcome.reason,
|
|
6208
|
+
remainingRequesters: outcome.remaining,
|
|
6209
|
+
sessionId,
|
|
6210
|
+
})
|
|
4046
6211
|
}).catch((error) => {
|
|
4047
6212
|
res.statusCode = 400
|
|
4048
6213
|
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
@@ -4433,6 +6598,393 @@ function buildChoiceRoute(ctx, runtime, config) {
|
|
|
4433
6598
|
}
|
|
4434
6599
|
}
|
|
4435
6600
|
|
|
6601
|
+
/**
|
|
6602
|
+
* The route of the self-starting supervisor.
|
|
6603
|
+
*
|
|
6604
|
+
* Three actions, and NOTHING happens without the confirmation:
|
|
6605
|
+
*
|
|
6606
|
+
* - `status` — who starts this host and whether the restart can work;
|
|
6607
|
+
* - `plan` — what the installation WOULD change (the dry run: it writes
|
|
6608
|
+
* nothing and runs nothing);
|
|
6609
|
+
* - `install` / `uninstall` — the real thing, and they need `confirmed: true`
|
|
6610
|
+
* (the confirmation card is what says "yes").
|
|
6611
|
+
*
|
|
6612
|
+
* @param {object} ctx the plugin context
|
|
6613
|
+
* @param {any} runtime the runtime of this plugin
|
|
6614
|
+
* @param {object} config the normalized config
|
|
6615
|
+
* @returns {object} the route
|
|
6616
|
+
*/
|
|
6617
|
+
/**
|
|
6618
|
+
* Read the capabilities record of this machine.
|
|
6619
|
+
*
|
|
6620
|
+
* It lives in a file of its own next to the state: the checkpoint has a
|
|
6621
|
+
* versioned shape, and a capability that is checked once does not belong in
|
|
6622
|
+
* the record of the running sessions.
|
|
6623
|
+
*
|
|
6624
|
+
* @param {string} file
|
|
6625
|
+
* @returns {object|null}
|
|
6626
|
+
*/
|
|
6627
|
+
function readCapabilitiesRecord(file) {
|
|
6628
|
+
try {
|
|
6629
|
+
const raw = fs.readFileSync(file, 'utf8')
|
|
6630
|
+
const parsed = JSON.parse(raw)
|
|
6631
|
+
if (!isCapabilities(parsed)) return null
|
|
6632
|
+
return {
|
|
6633
|
+
scheduler: parsed.scheduler,
|
|
6634
|
+
checkedAt: typeof parsed.checkedAt === 'number' ? parsed.checkedAt : null,
|
|
6635
|
+
evidence: parsed.evidence && typeof parsed.evidence === 'object' ? parsed.evidence : null,
|
|
6636
|
+
warnedAt: typeof parsed.warnedAt === 'number' ? parsed.warnedAt : null,
|
|
6637
|
+
tier: typeof parsed.tier === 'string' ? parsed.tier : '',
|
|
6638
|
+
tierReason: typeof parsed.tierReason === 'string' ? parsed.tierReason : '',
|
|
6639
|
+
}
|
|
6640
|
+
} catch {
|
|
6641
|
+
// A record that cannot be read is a record that does not exist yet.
|
|
6642
|
+
return null
|
|
6643
|
+
}
|
|
6644
|
+
}
|
|
6645
|
+
|
|
6646
|
+
/**
|
|
6647
|
+
* Write the capabilities record of this machine.
|
|
6648
|
+
*
|
|
6649
|
+
* @param {string} file
|
|
6650
|
+
* @param {object} record
|
|
6651
|
+
*/
|
|
6652
|
+
function writeCapabilitiesRecord(file, record) {
|
|
6653
|
+
try {
|
|
6654
|
+
fs.mkdirSync(path.dirname(file), { recursive: true })
|
|
6655
|
+
fs.writeFileSync(file, JSON.stringify(record, null, 2), 'utf8')
|
|
6656
|
+
} catch {
|
|
6657
|
+
// A capability that cannot be written is a capability that is checked
|
|
6658
|
+
// again next time — not a reason to break the plugin.
|
|
6659
|
+
}
|
|
6660
|
+
}
|
|
6661
|
+
|
|
6662
|
+
/**
|
|
6663
|
+
* Resolve the rescue-route path: a sibling of the health route, under the same
|
|
6664
|
+
* prefix.
|
|
6665
|
+
*
|
|
6666
|
+
* @param {object} config
|
|
6667
|
+
* @returns {string}
|
|
6668
|
+
*/
|
|
6669
|
+
function resolveRescuePath(config) {
|
|
6670
|
+
const healthPath = resolveHealthPath(config)
|
|
6671
|
+
const index = healthPath.lastIndexOf('/')
|
|
6672
|
+
const prefix = index > 0 ? healthPath.slice(0, index) : '/__dsh_autorestart'
|
|
6673
|
+
return `${prefix}/rescue`
|
|
6674
|
+
}
|
|
6675
|
+
|
|
6676
|
+
/**
|
|
6677
|
+
* The rescue route: the two scripts, and the desktop shortcut.
|
|
6678
|
+
*
|
|
6679
|
+
* It does three things, all of them on behalf of the USER:
|
|
6680
|
+
*
|
|
6681
|
+
* - `status` : where the scripts are (the paths and the state of the
|
|
6682
|
+
* desktop shortcut) — nothing is written;
|
|
6683
|
+
* - `shortcut` : write the desktop shortcut. This happens ONLY because the
|
|
6684
|
+
* user pressed the button: loading the settings page never
|
|
6685
|
+
* writes anything to the desktop;
|
|
6686
|
+
* - `reveal` : open the folder of a script in the file manager.
|
|
6687
|
+
*
|
|
6688
|
+
* Like every other route of this plugin it only answers the loopback.
|
|
6689
|
+
*
|
|
6690
|
+
* @param {object} ctx
|
|
6691
|
+
* @param {object} runtime
|
|
6692
|
+
* @param {object} config
|
|
6693
|
+
* @returns {{ kind: string, path: string, handler: Function }}
|
|
6694
|
+
*/
|
|
6695
|
+
function buildRescueRoute(ctx, runtime, config) {
|
|
6696
|
+
const routePath = resolveRescuePath(config)
|
|
6697
|
+
return {
|
|
6698
|
+
kind: 'exact',
|
|
6699
|
+
path: routePath,
|
|
6700
|
+
handler: (req, res) => {
|
|
6701
|
+
const remote = req && req.socket ? req.socket.remoteAddress : undefined
|
|
6702
|
+
const loopback = remote === undefined
|
|
6703
|
+
|| remote === '127.0.0.1'
|
|
6704
|
+
|| remote === '::1'
|
|
6705
|
+
|| remote === '::ffff:127.0.0.1'
|
|
6706
|
+
if (!loopback && !admittedByConnection(ctx, req)) {
|
|
6707
|
+
res.statusCode = 403
|
|
6708
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6709
|
+
res.end('{"ok":false,"error":"loopback-only"}')
|
|
6710
|
+
return
|
|
6711
|
+
}
|
|
6712
|
+
if (!req || req.method !== 'POST') {
|
|
6713
|
+
res.statusCode = 405
|
|
6714
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6715
|
+
res.setHeader('allow', 'POST')
|
|
6716
|
+
res.end('{"ok":false,"error":"method-not-allowed"}')
|
|
6717
|
+
return
|
|
6718
|
+
}
|
|
6719
|
+
void readJsonBody(req).then((payload) => {
|
|
6720
|
+
const body = payload && typeof payload === 'object' ? payload : {}
|
|
6721
|
+
const action = typeof body.action === 'string' ? body.action.trim() : 'status'
|
|
6722
|
+
const info = runtime.rescueKitInfo()
|
|
6723
|
+
if (action === 'shortcut') {
|
|
6724
|
+
// The user pressed the button: NOW the shortcut may be written.
|
|
6725
|
+
const written = createDesktopShortcut({
|
|
6726
|
+
desktop: info.desktopDir,
|
|
6727
|
+
restoreProfilePath: info.restoreProfile,
|
|
6728
|
+
home: os.homedir(),
|
|
6729
|
+
writeFile: (target, content) => fs.writeFileSync(target, content, 'ascii'),
|
|
6730
|
+
exists: (target) => fs.existsSync(target),
|
|
6731
|
+
})
|
|
6732
|
+
res.statusCode = written.ok ? 200 : 400
|
|
6733
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6734
|
+
res.setHeader('cache-control', 'no-store')
|
|
6735
|
+
res.end(JSON.stringify({
|
|
6736
|
+
ok: written.ok,
|
|
6737
|
+
action,
|
|
6738
|
+
path: written.path,
|
|
6739
|
+
error: written.error ?? null,
|
|
6740
|
+
rescue: runtime.rescueKitInfo(),
|
|
6741
|
+
}))
|
|
6742
|
+
return
|
|
6743
|
+
}
|
|
6744
|
+
if (action === 'ack-scheduler') {
|
|
6745
|
+
// The user has read the warning about the missing automatic
|
|
6746
|
+
// rollback: it is not shown again.
|
|
6747
|
+
const marked = runtime.acknowledgeSchedulerWarning()
|
|
6748
|
+
res.statusCode = 200
|
|
6749
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6750
|
+
res.setHeader('cache-control', 'no-store')
|
|
6751
|
+
res.end(JSON.stringify({ ok: true, action, capabilities: marked }))
|
|
6752
|
+
return
|
|
6753
|
+
}
|
|
6754
|
+
if (action === 'reveal') {
|
|
6755
|
+
const wanted = typeof body.path === 'string' ? body.path.trim() : ''
|
|
6756
|
+
// Only a directory of the plugin may be opened — never anything the
|
|
6757
|
+
// caller names.
|
|
6758
|
+
const allowed = [info.scriptsDir].filter((entry) => entry !== '')
|
|
6759
|
+
const folder = allowed.includes(wanted) ? wanted : info.scriptsDir
|
|
6760
|
+
let opened = false
|
|
6761
|
+
try {
|
|
6762
|
+
const child = spawn(process.platform === 'win32' ? 'explorer.exe' : 'xdg-open', [folder], {
|
|
6763
|
+
detached: true,
|
|
6764
|
+
stdio: 'ignore',
|
|
6765
|
+
windowsHide: true,
|
|
6766
|
+
})
|
|
6767
|
+
child.unref()
|
|
6768
|
+
opened = true
|
|
6769
|
+
} catch (error) {
|
|
6770
|
+
opened = false
|
|
6771
|
+
}
|
|
6772
|
+
res.statusCode = opened ? 200 : 500
|
|
6773
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6774
|
+
res.setHeader('cache-control', 'no-store')
|
|
6775
|
+
res.end(JSON.stringify({ ok: opened, action, path: folder, error: opened ? null : 'the file manager could not be opened' }))
|
|
6776
|
+
return
|
|
6777
|
+
}
|
|
6778
|
+
res.statusCode = 200
|
|
6779
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6780
|
+
res.setHeader('cache-control', 'no-store')
|
|
6781
|
+
res.end(JSON.stringify({ ok: true, action, rescue: info }))
|
|
6782
|
+
})
|
|
6783
|
+
},
|
|
6784
|
+
}
|
|
6785
|
+
}
|
|
6786
|
+
|
|
6787
|
+
function buildSupervisorRoute(ctx, runtime, config) {
|
|
6788
|
+
const routePath = resolveSupervisorPath(config)
|
|
6789
|
+
return {
|
|
6790
|
+
kind: 'exact',
|
|
6791
|
+
path: routePath,
|
|
6792
|
+
handler: (req, res) => {
|
|
6793
|
+
const remote = req && req.socket ? req.socket.remoteAddress : undefined
|
|
6794
|
+
const loopback = remote === undefined
|
|
6795
|
+
|| remote === '127.0.0.1'
|
|
6796
|
+
|| remote === '::1'
|
|
6797
|
+
|| remote === '::ffff:127.0.0.1'
|
|
6798
|
+
if (!loopback && !admittedByConnection(ctx, req)) {
|
|
6799
|
+
res.statusCode = 403
|
|
6800
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6801
|
+
res.end('{"ok":false,"error":"loopback-only"}')
|
|
6802
|
+
return
|
|
6803
|
+
}
|
|
6804
|
+
if (!req || req.method !== 'POST') {
|
|
6805
|
+
res.statusCode = 405
|
|
6806
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6807
|
+
res.setHeader('allow', 'POST')
|
|
6808
|
+
res.end('{"ok":false,"error":"method-not-allowed"}')
|
|
6809
|
+
return
|
|
6810
|
+
}
|
|
6811
|
+
void readJsonBody(req).then(async (payload) => {
|
|
6812
|
+
const body = payload && typeof payload === 'object' ? payload : {}
|
|
6813
|
+
const action = typeof body.action === 'string' ? body.action.trim() : ''
|
|
6814
|
+
if (action === 'status') {
|
|
6815
|
+
res.statusCode = 200
|
|
6816
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6817
|
+
res.setHeader('cache-control', 'no-store')
|
|
6818
|
+
res.end(JSON.stringify({ ok: true, action, supervisor: runtime.supervisorStatus() }))
|
|
6819
|
+
return
|
|
6820
|
+
}
|
|
6821
|
+
if (action === 'plan' || action === 'install-plan') {
|
|
6822
|
+
// The installation plan: what the registration WOULD do. Nothing is
|
|
6823
|
+
// written and nothing is run.
|
|
6824
|
+
const plan = planSupervisorInstall({
|
|
6825
|
+
platform: runtime.env.platform,
|
|
6826
|
+
// The desktop app manages itself: there is no plan there.
|
|
6827
|
+
isDesktopApp: runtime.env.isDesktopApp === true,
|
|
6828
|
+
// The startup matrix: only a `bare` host may get a supervisor.
|
|
6829
|
+
startup: runtime.startupMode(),
|
|
6830
|
+
// WHICH MACHINE IS THIS? (WSL is not a Linux.)
|
|
6831
|
+
host: runtime.hostKind(),
|
|
6832
|
+
nodePath: process.execPath,
|
|
6833
|
+
execPath: process.execPath,
|
|
6834
|
+
supervisorPath: supervisorPaths(path.dirname(runtime.stateFile)).script,
|
|
6835
|
+
configPath: supervisorPaths(path.dirname(runtime.stateFile)).config,
|
|
6836
|
+
kind: config.supervisorKind,
|
|
6837
|
+
})
|
|
6838
|
+
res.statusCode = 200
|
|
6839
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6840
|
+
res.setHeader('cache-control', 'no-store')
|
|
6841
|
+
res.end(JSON.stringify({
|
|
6842
|
+
ok: plan.error ? false : true,
|
|
6843
|
+
action,
|
|
6844
|
+
plan,
|
|
6845
|
+
error: plan.error ?? null,
|
|
6846
|
+
reason: plan.reason,
|
|
6847
|
+
supervisor: runtime.supervisorStatus(),
|
|
6848
|
+
}))
|
|
6849
|
+
return
|
|
6850
|
+
}
|
|
6851
|
+
if (action === 'dry-run' || action === 'derive') {
|
|
6852
|
+
// B: the dry run. It shows what WOULD be written and runs nothing.
|
|
6853
|
+
const plan = await runtime.deriveStartCommand()
|
|
6854
|
+
res.statusCode = 200
|
|
6855
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6856
|
+
res.setHeader('cache-control', 'no-store')
|
|
6857
|
+
res.end(JSON.stringify({ ok: true, action, plan, supervisor: runtime.supervisorStatus() }))
|
|
6858
|
+
return
|
|
6859
|
+
}
|
|
6860
|
+
if (action === 'apply' || action === 'revert') {
|
|
6861
|
+
if (body.confirmed !== true) {
|
|
6862
|
+
// The dry run is the confirmation: without it nothing is written.
|
|
6863
|
+
res.statusCode = 400
|
|
6864
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6865
|
+
res.setHeader('cache-control', 'no-store')
|
|
6866
|
+
res.end(JSON.stringify({
|
|
6867
|
+
ok: false,
|
|
6868
|
+
error: 'confirmation-required',
|
|
6869
|
+
instruction: 'Send { "action": "' + action + '", "confirmed": true } after the user confirmed the dry run.',
|
|
6870
|
+
}))
|
|
6871
|
+
return
|
|
6872
|
+
}
|
|
6873
|
+
const result = action === 'apply'
|
|
6874
|
+
? await runtime.applyDerivedCommand()
|
|
6875
|
+
: await runtime.revertDerivedCommand()
|
|
6876
|
+
res.statusCode = result.ok === true ? 200 : 500
|
|
6877
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6878
|
+
res.setHeader('cache-control', 'no-store')
|
|
6879
|
+
res.end(JSON.stringify({
|
|
6880
|
+
ok: result.ok === true,
|
|
6881
|
+
action,
|
|
6882
|
+
error: result.error ?? null,
|
|
6883
|
+
writes: result.writes ?? null,
|
|
6884
|
+
previous: result.previous ?? null,
|
|
6885
|
+
reason: result.reason ?? null,
|
|
6886
|
+
supervisor: runtime.supervisorStatus(),
|
|
6887
|
+
}))
|
|
6888
|
+
return
|
|
6889
|
+
}
|
|
6890
|
+
if (action === 'external') {
|
|
6891
|
+
// "I keep my own way": nothing is installed, but the answer is
|
|
6892
|
+
// recorded — so the plugin knows that a supervisor is already there
|
|
6893
|
+
// and never installs a second one.
|
|
6894
|
+
runtime.recordSupervisorChoice('external')
|
|
6895
|
+
try {
|
|
6896
|
+
await runtime.applyConfigChoice({ supervisorMode: 'external' })
|
|
6897
|
+
} catch (error) {
|
|
6898
|
+
runtime.log.warn('the supervisor mode could not be written:', String(error))
|
|
6899
|
+
}
|
|
6900
|
+
res.statusCode = 200
|
|
6901
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6902
|
+
res.setHeader('cache-control', 'no-store')
|
|
6903
|
+
res.end(JSON.stringify({ ok: true, action, supervisor: runtime.supervisorStatus() }))
|
|
6904
|
+
return
|
|
6905
|
+
}
|
|
6906
|
+
if (action === 'install' || action === 'uninstall') {
|
|
6907
|
+
if (action === 'install' && runtime.env.isDesktopApp === true) {
|
|
6908
|
+
// The desktop app manages itself: a login record would open a
|
|
6909
|
+
// SECOND client at every login. The refusal comes BEFORE the
|
|
6910
|
+
// confirmation, so nothing is asked of the user.
|
|
6911
|
+
res.statusCode = 400
|
|
6912
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6913
|
+
res.setHeader('cache-control', 'no-store')
|
|
6914
|
+
res.end(JSON.stringify({
|
|
6915
|
+
ok: false,
|
|
6916
|
+
action,
|
|
6917
|
+
error: 'desktop-manages-itself',
|
|
6918
|
+
reason: 'the desktop app relaunches itself; a supervisor would start a second copy',
|
|
6919
|
+
steps: [],
|
|
6920
|
+
supervisor: runtime.supervisorStatus(),
|
|
6921
|
+
}))
|
|
6922
|
+
return
|
|
6923
|
+
}
|
|
6924
|
+
if (body.confirmed !== true) {
|
|
6925
|
+
// The confirmation card is what says "yes": without it nothing is
|
|
6926
|
+
// changed on the machine.
|
|
6927
|
+
res.statusCode = 400
|
|
6928
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6929
|
+
res.setHeader('cache-control', 'no-store')
|
|
6930
|
+
res.end(JSON.stringify({
|
|
6931
|
+
ok: false,
|
|
6932
|
+
error: 'confirmation-required',
|
|
6933
|
+
instruction: 'Send { "action": "' + action + '", "confirmed": true } after the user confirmed the card.',
|
|
6934
|
+
}))
|
|
6935
|
+
return
|
|
6936
|
+
}
|
|
6937
|
+
if (action === 'install') {
|
|
6938
|
+
const choice = typeof body.choice === 'string' ? body.choice : ''
|
|
6939
|
+
if (choice !== '') runtime.recordSupervisorChoice(choice)
|
|
6940
|
+
const result = runtime.installSupervisor({ confirmed: true })
|
|
6941
|
+
res.statusCode = result.ok ? 200 : 500
|
|
6942
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6943
|
+
res.setHeader('cache-control', 'no-store')
|
|
6944
|
+
res.end(JSON.stringify({
|
|
6945
|
+
ok: result.ok === true,
|
|
6946
|
+
action,
|
|
6947
|
+
kind: result.kind,
|
|
6948
|
+
degraded: result.degraded,
|
|
6949
|
+
steps: result.steps,
|
|
6950
|
+
error: result.error,
|
|
6951
|
+
reason: result.reason,
|
|
6952
|
+
supervisor: runtime.supervisorStatus(),
|
|
6953
|
+
}))
|
|
6954
|
+
return
|
|
6955
|
+
}
|
|
6956
|
+
const result = await runtime.uninstallSupervisor()
|
|
6957
|
+
res.statusCode = result.ok ? 200 : 500
|
|
6958
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6959
|
+
res.setHeader('cache-control', 'no-store')
|
|
6960
|
+
res.end(JSON.stringify({
|
|
6961
|
+
ok: result.ok === true,
|
|
6962
|
+
action,
|
|
6963
|
+
steps: result.steps,
|
|
6964
|
+
error: result.error,
|
|
6965
|
+
pending: result.pending,
|
|
6966
|
+
previous: result.previous,
|
|
6967
|
+
supervisor: runtime.supervisorStatus(),
|
|
6968
|
+
}))
|
|
6969
|
+
return
|
|
6970
|
+
}
|
|
6971
|
+
res.statusCode = 400
|
|
6972
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6973
|
+
res.setHeader('cache-control', 'no-store')
|
|
6974
|
+
res.end(JSON.stringify({
|
|
6975
|
+
ok: false,
|
|
6976
|
+
error: 'unknown-action',
|
|
6977
|
+
instruction: 'Send { "action": "status" | "plan" | "install" | "uninstall" }.',
|
|
6978
|
+
}))
|
|
6979
|
+
}).catch((error) => {
|
|
6980
|
+
res.statusCode = 500
|
|
6981
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6982
|
+
res.end(JSON.stringify({ ok: false, error: describeError(error) }))
|
|
6983
|
+
})
|
|
6984
|
+
},
|
|
6985
|
+
}
|
|
6986
|
+
}
|
|
6987
|
+
|
|
4436
6988
|
/**
|
|
4437
6989
|
* Is the graceful sleep a dry run?
|
|
4438
6990
|
*
|
|
@@ -4874,7 +7426,9 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
4874
7426
|
// window it stops blocking the restart.
|
|
4875
7427
|
queuedGraceMs: config.queuedGraceMs,
|
|
4876
7428
|
quietForMs: typeof verdict.quietForMs === 'number' ? verdict.quietForMs : 0,
|
|
4877
|
-
|
|
7429
|
+
// The number of TASKS the restart waits for: a session and its own
|
|
7430
|
+
// subagents are ONE task, exactly as the left sidebar counts them.
|
|
7431
|
+
busyCount: countBlockerTasks(verdict.blockedBy),
|
|
4878
7432
|
agentCount,
|
|
4879
7433
|
})
|
|
4880
7434
|
res.statusCode = 200
|
|
@@ -4958,6 +7512,9 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
4958
7512
|
// the restart).
|
|
4959
7513
|
parkUrl: resolveParkPath(config),
|
|
4960
7514
|
recoverUrl: resolveRecoverPath(config),
|
|
7515
|
+
// The route that withdraws ONE session's own restart request (the
|
|
7516
|
+
// manual "cancel my restart" of the pill).
|
|
7517
|
+
cancelUrl: resolveCancelPath(config),
|
|
4961
7518
|
// The route that persists the one-time install choice, plus the
|
|
4962
7519
|
// value that is effective right now.
|
|
4963
7520
|
choiceUrl: resolveChoicePath(config),
|
|
@@ -4966,8 +7523,79 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
4966
7523
|
announcementsUrl: resolveAnnouncementsPath(config),
|
|
4967
7524
|
backupOnRestart: config.backupOnRestart !== false,
|
|
4968
7525
|
backupIncludePlugins: config.backupIncludePlugins !== false,
|
|
7526
|
+
// WHAT IS ON DISK RIGHT NOW: the backups of this host (the count,
|
|
7527
|
+
// the local time, the size, whether the plugins are in it and
|
|
7528
|
+
// where it lives). The browser cannot read the disk, so the Host
|
|
7529
|
+
// reads it — only the directory, never the contents.
|
|
7530
|
+
backups: runtime.backupsInfo(),
|
|
7531
|
+
// THE RESCUE KIT: where the two scripts are, so a user whose DSH
|
|
7532
|
+
// does not come up can find them (and copy the path) even though
|
|
7533
|
+
// they cannot open a README.
|
|
7534
|
+
rescue: runtime.rescueKitInfo(),
|
|
7535
|
+
rescueUrl: resolveRescuePath(config),
|
|
7536
|
+
// What this machine can do (the automatic rollback needs a
|
|
7537
|
+
// one-off scheduled task; when that is refused the user is told
|
|
7538
|
+
// what they lose).
|
|
7539
|
+
capabilities: runtime.capabilitiesInfo(),
|
|
7540
|
+
// The watchdog: which tier this machine uses to trigger it, and
|
|
7541
|
+
// what the last run did.
|
|
7542
|
+
watchdog: runtime.watchdogInfo(),
|
|
7543
|
+
// THE RESTART COMMAND that is in effect: what is written, and what
|
|
7544
|
+
// an undo would restore. The settings page shows it in a folded
|
|
7545
|
+
// "for a technical colleague" section.
|
|
7546
|
+
restart: {
|
|
7547
|
+
mode: config.restartMode,
|
|
7548
|
+
command: typeof config.restartCommand === 'string' ? config.restartCommand : '',
|
|
7549
|
+
written: typeof config.restartCommand === 'string' && config.restartCommand !== '',
|
|
7550
|
+
canRevert: Boolean(
|
|
7551
|
+
live.onboarding
|
|
7552
|
+
&& typeof live.onboarding === 'object'
|
|
7553
|
+
&& live.onboarding.previous
|
|
7554
|
+
&& typeof live.onboarding.previous.restartMode === 'string',
|
|
7555
|
+
),
|
|
7556
|
+
previous: live.onboarding && typeof live.onboarding === 'object' && live.onboarding.previous
|
|
7557
|
+
? live.onboarding.previous
|
|
7558
|
+
: null,
|
|
7559
|
+
},
|
|
4969
7560
|
parkedGoals: parkedGoalList(runtime),
|
|
4970
7561
|
recovery: recoveryList(live),
|
|
7562
|
+
// THE ACCESS LINK of this instance (with the boot token). The boot
|
|
7563
|
+
// token is created per process, so the link the browser holds
|
|
7564
|
+
// answers 401 after a restart. The client needs the new link to
|
|
7565
|
+
// get the user back into the interface.
|
|
7566
|
+
accessUrl: accessUrlOf(runtime),
|
|
7567
|
+
// The supervisor: who starts this host, and whether the restart
|
|
7568
|
+
// can work at all.
|
|
7569
|
+
supervisor: runtime.supervisorStatus(),
|
|
7570
|
+
// WHICH END this is. The browser side serves the desktop app AND
|
|
7571
|
+
// the hosted web host, so it may not assume one of them: the
|
|
7572
|
+
// desktop app brings itself back up (there is nothing to install
|
|
7573
|
+
// there), while the hosted host may ask for a supervisor.
|
|
7574
|
+
instance: {
|
|
7575
|
+
isDesktopApp: runtime.env.isDesktopApp === true,
|
|
7576
|
+
platform: runtime.env.platform,
|
|
7577
|
+
},
|
|
7578
|
+
// THE LAST RESTART: the snapshot that was taken before it (the
|
|
7579
|
+
// directory, its LOCAL time, how long it took, which older
|
|
7580
|
+
// snapshots were pruned) and the verdict of the instance that came
|
|
7581
|
+
// up (the four checks). Without it a restart has no visible
|
|
7582
|
+
// result at all.
|
|
7583
|
+
lastRestart: runtime.lastRestartVerdict(),
|
|
7584
|
+
// Whether the one-time question about the supervisor may be asked.
|
|
7585
|
+
supervisorOffer: runtime.supervisorOffer,
|
|
7586
|
+
// The answer of the one-time question about the supervisor (null
|
|
7587
|
+
// while it was never answered).
|
|
7588
|
+
onboarding: runtime.onboardingChoice,
|
|
7589
|
+
// The route that installs, uninstalls and reports the supervisor.
|
|
7590
|
+
supervisorUrl: resolveSupervisorPath(config),
|
|
7591
|
+
// "REPORT A PROBLEM": the diagnostic text (built by ONE generator on
|
|
7592
|
+
// the host, so the browser and the host can never drift) and the
|
|
7593
|
+
// two places a report can go.
|
|
7594
|
+
diagnostics: runtime.diagnostics(),
|
|
7595
|
+
report: {
|
|
7596
|
+
email: SUPPORT_EMAIL,
|
|
7597
|
+
issueUrlBase: ISSUE_URL_BASE,
|
|
7598
|
+
},
|
|
4971
7599
|
})
|
|
4972
7600
|
} catch {
|
|
4973
7601
|
// The banner must never break the health route: fall back to "no restart".
|
|
@@ -4983,6 +7611,7 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
4983
7611
|
longWait: null,
|
|
4984
7612
|
parkUrl: resolveParkPath(config),
|
|
4985
7613
|
recoverUrl: resolveRecoverPath(config),
|
|
7614
|
+
cancelUrl: resolveCancelPath(config),
|
|
4986
7615
|
choiceUrl: resolveChoicePath(config),
|
|
4987
7616
|
announcementsUrl: resolveAnnouncementsPath(config),
|
|
4988
7617
|
backupOnRestart: config.backupOnRestart !== false,
|
|
@@ -4998,6 +7627,27 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
4998
7627
|
return
|
|
4999
7628
|
}
|
|
5000
7629
|
const probeIndex = typeof req.url === 'string' && req.url.includes('probeIndex=1')
|
|
7630
|
+
// THE READY MARKER: this host answers its own health route, so it is
|
|
7631
|
+
// up. The watchdog (which waits for the relaunched host) sees the
|
|
7632
|
+
// marker and stands down — no rollback of a host that is fine.
|
|
7633
|
+
try {
|
|
7634
|
+
const marker = readReady(stateDirOf())
|
|
7635
|
+
if (!marker || marker.bootId !== runtime.bootId) {
|
|
7636
|
+
writeReady(stateDirOf(), { bootId: runtime.bootId, port: runtime.port ?? null })
|
|
7637
|
+
// The host is up: the watchdog has nothing to do. Its scheduling
|
|
7638
|
+
// item and its pending marker go NOW — nothing may be left behind.
|
|
7639
|
+
if (readPending(stateDirOf()) !== null) {
|
|
7640
|
+
clearPending(stateDirOf())
|
|
7641
|
+
try {
|
|
7642
|
+
removeScheduledTask({ taskName: 'dsh-autorestart-watchdog' })
|
|
7643
|
+
} catch {
|
|
7644
|
+
// The watchdog removes it itself when it runs.
|
|
7645
|
+
}
|
|
7646
|
+
}
|
|
7647
|
+
}
|
|
7648
|
+
} catch {
|
|
7649
|
+
// The marker is a convenience: the watchdog also asks the health route.
|
|
7650
|
+
}
|
|
5001
7651
|
const body = JSON.stringify({
|
|
5002
7652
|
ok: true,
|
|
5003
7653
|
plugin: name,
|
|
@@ -5044,49 +7694,46 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
5044
7694
|
const recoverRoute = buildRecoverRoute(ctx, runtime, config)
|
|
5045
7695
|
const handoffRoute = buildHandoffRoute(ctx, runtime, config)
|
|
5046
7696
|
const choiceRoute = buildChoiceRoute(ctx, runtime, config)
|
|
7697
|
+
const cancelRoute = buildCancelRoute(ctx, runtime, config)
|
|
5047
7698
|
const announcementsRoute = buildAnnouncementsRoute(ctx, runtime, config)
|
|
7699
|
+
const supervisorRoute = buildSupervisorRoute(ctx, runtime, config)
|
|
7700
|
+
const rescueRoute = buildRescueRoute(ctx, runtime, config)
|
|
5048
7701
|
const sleepRoute = sleep ? buildSleepRoute(ctx, runtime, config, sleep) : null
|
|
5049
7702
|
try {
|
|
5050
7703
|
const origin = typeof webServer.getOrigin === 'function' ? String(webServer.getOrigin()) : ''
|
|
5051
7704
|
const portMatch = /:(\d+)\/?$/.exec(origin)
|
|
5052
7705
|
if (portMatch) runtime.webPort = Number(portMatch[1])
|
|
7706
|
+
// THE INSTANCE RECORD. The new instance publishes its own port and its
|
|
7707
|
+
// own access link (with the boot token), so:
|
|
7708
|
+
// - the rollback watcher learns the port of the NEW instance instead
|
|
7709
|
+
// of guessing the (random) port of the old one;
|
|
7710
|
+
// - the user finds the access link of the new instance after a
|
|
7711
|
+
// restart, instead of being locked out by a 401.
|
|
7712
|
+
publishInstanceRecord(ctx, runtime, config, origin)
|
|
5053
7713
|
} catch {}
|
|
5054
|
-
|
|
5055
|
-
|
|
5056
|
-
|
|
5057
|
-
|
|
5058
|
-
|
|
5059
|
-
|
|
5060
|
-
|
|
5061
|
-
|
|
5062
|
-
|
|
5063
|
-
|
|
5064
|
-
|
|
5065
|
-
|
|
5066
|
-
|
|
5067
|
-
|
|
5068
|
-
|
|
5069
|
-
|
|
5070
|
-
try {
|
|
5071
|
-
target.effect(() => webServer.register(handoffRoute), 'dsh-autorestart: handoff route')
|
|
5072
|
-
} catch (error) {
|
|
5073
|
-
runtime.log.warn('could not register the handoff route:', String(error))
|
|
5074
|
-
}
|
|
5075
|
-
try {
|
|
5076
|
-
target.effect(() => webServer.register(choiceRoute), 'dsh-autorestart: choice route')
|
|
5077
|
-
} catch (error) {
|
|
5078
|
-
runtime.log.warn('could not register the choice route:', String(error))
|
|
5079
|
-
}
|
|
5080
|
-
try {
|
|
5081
|
-
target.effect(() => webServer.register(announcementsRoute), 'dsh-autorestart: announcements route')
|
|
5082
|
-
} catch (error) {
|
|
5083
|
-
runtime.log.warn('could not register the announcements route:', String(error))
|
|
5084
|
-
}
|
|
5085
|
-
if (sleepRoute !== null) {
|
|
7714
|
+
// The routes of this plugin. Every one of them is remembered: the boot
|
|
7715
|
+
// check of the restart record reports "the routes are there" — a host
|
|
7716
|
+
// that came up without them is not a healthy host.
|
|
7717
|
+
const routes = [
|
|
7718
|
+
{ name: 'health', route },
|
|
7719
|
+
{ name: 'park', route: parkRoute },
|
|
7720
|
+
{ name: 'recover', route: recoverRoute },
|
|
7721
|
+
{ name: 'handoff', route: handoffRoute },
|
|
7722
|
+
{ name: 'choice', route: choiceRoute },
|
|
7723
|
+
{ name: 'cancel', route: cancelRoute },
|
|
7724
|
+
{ name: 'announcements', route: announcementsRoute },
|
|
7725
|
+
{ name: 'supervisor', route: supervisorRoute },
|
|
7726
|
+
{ name: 'rescue', route: rescueRoute },
|
|
7727
|
+
]
|
|
7728
|
+
if (sleepRoute !== null) routes.push({ name: 'sleep', route: sleepRoute })
|
|
7729
|
+
for (const entry of routes) {
|
|
5086
7730
|
try {
|
|
5087
|
-
target.effect(() => webServer.register(
|
|
7731
|
+
target.effect(() => webServer.register(entry.route), 'dsh-autorestart: ' + entry.name + ' route')
|
|
7732
|
+
runtime.registeredRoutes.push(entry.name)
|
|
5088
7733
|
} catch (error) {
|
|
5089
|
-
|
|
7734
|
+
// A duplicate registration means another load of this plugin already owns it.
|
|
7735
|
+
runtime.log.warn('could not register the ' + entry.name + ' route:', String(error))
|
|
7736
|
+
runtime.routeErrors.push({ route: entry.name, error: String(error) })
|
|
5090
7737
|
}
|
|
5091
7738
|
}
|
|
5092
7739
|
if (config.autoReloadPage !== false && typeof webServer.tapIndex === 'function') {
|
|
@@ -5311,15 +7958,35 @@ export function apply(ctx, rawConfig, options = {}) {
|
|
|
5311
7958
|
if (runtime.isNewBoot && config.resumeOnBoot) {
|
|
5312
7959
|
runtime.recoverTimer = ctx.timeout(() => {
|
|
5313
7960
|
recoverAfterRestart(ctx, runtime, config)
|
|
5314
|
-
.then(() => {
|
|
7961
|
+
.then(() => {
|
|
7962
|
+
runtime.recovery = { ran: true, ok: true, detail: 'the boot recovery finished' }
|
|
7963
|
+
restoreSleepIntent()
|
|
7964
|
+
})
|
|
5315
7965
|
.catch((error) => {
|
|
5316
7966
|
log.warn('boot recovery failed:', String(error))
|
|
7967
|
+
runtime.recovery = { ran: true, ok: false, detail: 'the boot recovery failed: ' + String(error) }
|
|
5317
7968
|
restoreSleepIntent()
|
|
5318
7969
|
})
|
|
7970
|
+
// THE VERDICT OF THE RESTART. The instance that replaced the old one
|
|
7971
|
+
// checks itself (listener / routes / plugin / recovery) and writes the
|
|
7972
|
+
// result into the durable restart record. Without this the four
|
|
7973
|
+
// checks would live nowhere and a failed restart would look like a
|
|
7974
|
+
// successful one.
|
|
7975
|
+
.then(() => verifyRestartBoot(runtime))
|
|
7976
|
+
.catch((error) => {
|
|
7977
|
+
log.warn('the boot verification failed:', String(error))
|
|
7978
|
+
})
|
|
5319
7979
|
}, Math.max(0, config.recoveryDelayMs))
|
|
5320
7980
|
} else {
|
|
5321
7981
|
// No recovery pass: the intent is restored right away (a hot reload must
|
|
5322
7982
|
// not lose it either).
|
|
7983
|
+
runtime.recovery = {
|
|
7984
|
+
ran: false,
|
|
7985
|
+
ok: true,
|
|
7986
|
+
detail: runtime.isNewBoot
|
|
7987
|
+
? 'the boot recovery is switched off'
|
|
7988
|
+
: 'this is not a new boot',
|
|
7989
|
+
}
|
|
5323
7990
|
restoreSleepIntent()
|
|
5324
7991
|
}
|
|
5325
7992
|
|
|
@@ -5349,4 +8016,5 @@ export {
|
|
|
5349
8016
|
pluginVersion,
|
|
5350
8017
|
PLUGIN_VERSION_FALLBACK,
|
|
5351
8018
|
SLEEP_FEATURE_VERSION,
|
|
8019
|
+
SUPERVISOR_FEATURE_VERSION,
|
|
5352
8020
|
}
|