dsh-autorestart 0.6.1 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,884 @@
1
+ /**
2
+ * dsh-autorestart — the supervisor.
3
+ *
4
+ * WHY A SEPARATE PROCESS. The plugin lives INSIDE the host. When the host
5
+ * dies, the plugin dies with it — so the plugin can never be the thing that
6
+ * brings the host back. The supervisor is therefore a standalone Node process
7
+ * (zero dependencies, no plugin imports) that the plugin COPIES into the
8
+ * state directory and registers with the platform's "start at login"
9
+ * mechanism.
10
+ *
11
+ * WHAT IT DOES (the whole job, in order):
12
+ *
13
+ * 1. take the exclusive lock — a second supervisor must never race the
14
+ * first one (two supervisors = two hosts = two web servers);
15
+ * 2. write its pid and open the control channel (a loopback TCP socket);
16
+ * 3. start the host with the command from `supervisor.json`;
17
+ * 4. watch it. When it exits, wait a backoff and start it again;
18
+ * 5. when the host does NOT come back, restore the snapshot the plugin
19
+ * asked for (`rollback-request.json`) and try once more;
20
+ * 6. answer the control channel: `restart` (kill + start again) and `stop`
21
+ * (kill and exit).
22
+ *
23
+ * THE BACKOFF. The delay doubles after every failed start (1s → 2s → 4s …)
24
+ * and is capped (30s by default). It is reset to the base delay ONLY after a
25
+ * start that actually worked — a host that starts and dies again keeps the
26
+ * delay, so a crash loop does not become a spawn loop.
27
+ *
28
+ * THE ENVIRONMENT. The host is NOT started with `env: process.env`: that
29
+ * would carry the host's own internal variables (`DSH_HOME`, `DSH_PROFILE`,
30
+ * …) into the next start and the profile would drift. Only a WHITELIST is
31
+ * inherited, plus the two profile variables the plugin knows and the three
32
+ * markers that tell the plugin "I was started by the supervisor".
33
+ *
34
+ * STANDALONE. This file is copied to `<stateDir>/supervisor/supervisor.js`
35
+ * and runs from there: it imports NOTHING from the plugin (a rollback of the
36
+ * plugin tree would take the supervisor with it otherwise).
37
+ *
38
+ * Usage: `node supervisor.js --config <supervisor.json>`
39
+ */
40
+
41
+ import fs from 'node:fs'
42
+ import http from 'node:http'
43
+ import net from 'node:net'
44
+ import path from 'node:path'
45
+ import { spawn } from 'node:child_process'
46
+
47
+ /** The version of the supervisor configuration this file understands. */
48
+ export const SUPERVISOR_CONFIG_VERSION = 1
49
+
50
+ /** The name of the autostart record (a scheduled task / a systemd unit). */
51
+ export const SUPERVISOR_TASK_NAME = 'dsh-autorestart-web'
52
+
53
+ /** The environment marker that says "a supervisor started me". */
54
+ export const SUPERVISED_ENV = 'DSH_AUTORESTART_SUPERVISED'
55
+
56
+ /** The environment marker that carries the supervisor's pid. */
57
+ export const SUPERVISOR_PID_ENV = 'DSH_AUTORESTART_SUPERVISOR_PID'
58
+
59
+ /** The environment marker that carries the restart generation. */
60
+ export const GENERATION_ENV = 'DSH_AUTORESTART_GENERATION'
61
+
62
+ /**
63
+ * The variables the host inherits from the supervisor.
64
+ *
65
+ * Everything else is dropped on purpose: the host's own internal variables
66
+ * (`DSH_HOME`, `DSH_PROFILE`, …) must NOT leak into the next start.
67
+ */
68
+ export const INHERITED_ENV_KEYS = Object.freeze([
69
+ 'PATH',
70
+ 'HOME',
71
+ 'USER',
72
+ 'LOGNAME',
73
+ 'SHELL',
74
+ 'LANG',
75
+ 'LC_ALL',
76
+ 'SystemRoot',
77
+ 'TEMP',
78
+ 'TMP',
79
+ 'APPDATA',
80
+ 'LOCALAPPDATA',
81
+ 'ProgramData',
82
+ 'NUMBER_OF_PROCESSORS',
83
+ 'WSLENV',
84
+ 'WSL_DISTRO_NAME',
85
+ 'WSL_INTEROP',
86
+ ])
87
+
88
+ /**
89
+ * The profile files a rollback puts back.
90
+ *
91
+ * The SAME list lives in `lib/backup.js` of the plugin. It is written out
92
+ * here on purpose: the supervisor is a standalone process and may not import
93
+ * the plugin (a rollback of the plugin tree would take it down with it).
94
+ * `test/supervisor.test.mjs` asserts that the two lists agree.
95
+ */
96
+ export const CONFIG_FILES = Object.freeze(['cordis.patch.yml', 'package.json', 'pnpm-lock.yaml'])
97
+
98
+ /** The directory (inside the profile) that holds the installed plugins. */
99
+ export const PLUGINS_DIR = 'node_modules'
100
+
101
+ /** The default backoff: the first retry waits this long. */
102
+ export const DEFAULT_RESTART_DELAY_MS = 1000
103
+
104
+ /** The default cap of the backoff. */
105
+ export const DEFAULT_RESTART_DELAY_MAX_MS = 30000
106
+
107
+ /** The default time the supervisor waits for the host to answer. */
108
+ export const DEFAULT_START_TIMEOUT_MS = 90000
109
+
110
+ /** The default grace period between SIGTERM and SIGKILL. */
111
+ export const DEFAULT_KILL_GRACE_MS = 15000
112
+
113
+ /** The default time the supervisor waits for the port to become free. */
114
+ export const DEFAULT_PORT_FREE_TIMEOUT_MS = 30000
115
+
116
+ /** The log file grows to this size before it is truncated. */
117
+ export const MAX_LOG_BYTES = 2 * 1024 * 1024
118
+
119
+ /** @param {unknown} value */
120
+ function errorText(value) {
121
+ if (value && typeof value.message === 'string' && value.message) return value.message
122
+ return String(value)
123
+ }
124
+
125
+ /**
126
+ * Parse the command line of the supervisor.
127
+ *
128
+ * @param {string[]} argv
129
+ * @returns {{ configPath: string|null, help: boolean }}
130
+ */
131
+ export function parseArgs(argv) {
132
+ const list = Array.isArray(argv) ? argv : []
133
+ let configPath = null
134
+ for (let index = 0; index < list.length; index += 1) {
135
+ const arg = String(list[index] ?? '')
136
+ if (arg === '--config' || arg === '-c') {
137
+ configPath = typeof list[index + 1] === 'string' ? list[index + 1] : null
138
+ index += 1
139
+ continue
140
+ }
141
+ if (arg.startsWith('--config=')) {
142
+ configPath = arg.slice('--config='.length)
143
+ continue
144
+ }
145
+ if (arg === '--help' || arg === '-h') return { configPath: null, help: true }
146
+ }
147
+ return { configPath, help: false }
148
+ }
149
+
150
+ /**
151
+ * The environment the host is started with.
152
+ *
153
+ * @param {object} baseEnv the environment of the supervisor
154
+ * @param {{ supervisorPid?: number, generation?: number, profileEnv?: object }} [options]
155
+ * @returns {Record<string, string>}
156
+ */
157
+ export function buildChildEnv(baseEnv, options = {}) {
158
+ const source = baseEnv && typeof baseEnv === 'object' ? baseEnv : {}
159
+ const env = {}
160
+ for (const key of INHERITED_ENV_KEYS) {
161
+ const value = source[key]
162
+ if (typeof value === 'string' && value !== '') env[key] = value
163
+ }
164
+ const profileEnv = options.profileEnv && typeof options.profileEnv === 'object' ? options.profileEnv : {}
165
+ for (const [key, value] of Object.entries(profileEnv)) {
166
+ if (typeof value === 'string' && value !== '') env[key] = value
167
+ }
168
+ env[SUPERVISED_ENV] = '1'
169
+ if (Number.isFinite(Number(options.supervisorPid))) env[SUPERVISOR_PID_ENV] = String(options.supervisorPid)
170
+ if (Number.isFinite(Number(options.generation))) env[GENERATION_ENV] = String(options.generation)
171
+ return env
172
+ }
173
+
174
+ /**
175
+ * The next backoff delay.
176
+ *
177
+ * The delay DOUBLES (and is capped). It is reset to the base delay only
178
+ * after a start that worked — see {@link shouldResetBackoff}.
179
+ *
180
+ * @param {number} current the delay of the last attempt
181
+ * @param {{ baseMs?: number, maxMs?: number }} [options]
182
+ * @returns {number} the delay for the next attempt, in milliseconds
183
+ */
184
+ export function nextRestartDelay(current, options = {}) {
185
+ const base = Number.isFinite(Number(options.baseMs)) && Number(options.baseMs) > 0
186
+ ? Math.trunc(Number(options.baseMs))
187
+ : DEFAULT_RESTART_DELAY_MS
188
+ const max = Number.isFinite(Number(options.maxMs)) && Number(options.maxMs) > 0
189
+ ? Math.trunc(Number(options.maxMs))
190
+ : DEFAULT_RESTART_DELAY_MAX_MS
191
+ const previous = Number.isFinite(Number(current)) && Number(current) > 0 ? Math.trunc(Number(current)) : base
192
+ if (previous >= max) return max
193
+ return Math.min(max, previous * 2)
194
+ }
195
+
196
+ /**
197
+ * Whether the backoff is reset after this attempt.
198
+ *
199
+ * ONLY a start that worked resets it: a host that starts and dies again
200
+ * keeps the delay, so a crash loop does not become a spawn loop.
201
+ *
202
+ * @param {boolean} healthy whether the host answered its health route
203
+ * @returns {boolean}
204
+ */
205
+ export function shouldResetBackoff(healthy) {
206
+ return healthy === true
207
+ }
208
+
209
+ /**
210
+ * Whether a pid is alive.
211
+ *
212
+ * @param {number} pid
213
+ * @returns {boolean}
214
+ */
215
+ export function isProcessAlive(pid) {
216
+ const number = Number(pid)
217
+ if (!Number.isFinite(number) || number <= 0) return false
218
+ try {
219
+ process.kill(number, 0)
220
+ return true
221
+ } catch (error) {
222
+ // EPERM means the process exists but belongs to somebody else.
223
+ return error && error.code === 'EPERM'
224
+ }
225
+ }
226
+
227
+ /**
228
+ * Take the exclusive lock of the supervisor.
229
+ *
230
+ * A lock that belongs to a DEAD process is stale and is taken over — a
231
+ * supervisor that was killed must not block its own successor for ever.
232
+ *
233
+ * @param {string} lockPath
234
+ * @param {{ pid?: number, isAlive?: (pid: number) => boolean }} [options]
235
+ * @returns {{ ok: boolean, stale: boolean, error: string|null }}
236
+ */
237
+ export function acquireLock(lockPath, options = {}) {
238
+ const file = typeof lockPath === 'string' ? lockPath.trim() : ''
239
+ if (file === '') return { ok: false, stale: false, error: 'acquireLock requires a path' }
240
+ const pid = Number.isFinite(Number(options.pid)) ? Number(options.pid) : process.pid
241
+ const alive = typeof options.isAlive === 'function' ? options.isAlive : isProcessAlive
242
+ try {
243
+ fs.mkdirSync(path.dirname(file), { recursive: true })
244
+ } catch {
245
+ /* the directory may already be there */
246
+ }
247
+ try {
248
+ const handle = fs.openSync(file, 'wx')
249
+ try {
250
+ fs.writeFileSync(handle, JSON.stringify({ pid, at: new Date().toISOString() }), 'utf8')
251
+ } catch {
252
+ /* the owner is what matters */
253
+ }
254
+ try {
255
+ fs.closeSync(handle)
256
+ } catch {
257
+ /* nothing else to do */
258
+ }
259
+ return { ok: true, stale: false, error: null }
260
+ } catch {
261
+ // The lock exists: is its owner still alive?
262
+ let owner = 0
263
+ try {
264
+ const raw = fs.readFileSync(file, 'utf8')
265
+ const parsed = JSON.parse(raw)
266
+ owner = Number(parsed && parsed.pid)
267
+ } catch {
268
+ owner = 0
269
+ }
270
+ if (owner > 0 && alive(owner)) {
271
+ return { ok: false, stale: false, error: 'another supervisor is already running (pid ' + owner + ')' }
272
+ }
273
+ try {
274
+ fs.rmSync(file, { force: true })
275
+ } catch {
276
+ /* the takeover fails below */
277
+ }
278
+ try {
279
+ const handle = fs.openSync(file, 'wx')
280
+ try {
281
+ fs.writeFileSync(handle, JSON.stringify({ pid, at: new Date().toISOString() }), 'utf8')
282
+ } catch {
283
+ /* the owner is what matters */
284
+ }
285
+ try {
286
+ fs.closeSync(handle)
287
+ } catch {
288
+ /* nothing else to do */
289
+ }
290
+ return { ok: true, stale: true, error: null }
291
+ } catch (error) {
292
+ return { ok: false, stale: false, error: errorText(error) }
293
+ }
294
+ }
295
+ }
296
+
297
+ /**
298
+ * Release the lock.
299
+ *
300
+ * @param {string} lockPath
301
+ * @returns {boolean}
302
+ */
303
+ export function releaseLock(lockPath) {
304
+ try {
305
+ fs.rmSync(String(lockPath ?? ''), { force: true })
306
+ return true
307
+ } catch {
308
+ return false
309
+ }
310
+ }
311
+
312
+ /**
313
+ * Whether a port is free (nothing is listening on it).
314
+ *
315
+ * @param {number} port
316
+ * @param {string} [host]
317
+ * @returns {Promise<boolean>}
318
+ */
319
+ export function isPortFree(port, host = '127.0.0.1') {
320
+ const number = Number(port)
321
+ if (!Number.isFinite(number) || number <= 0) return Promise.resolve(true)
322
+ return new Promise((resolve) => {
323
+ const socket = net.connect({ port: number, host })
324
+ const done = (free) => {
325
+ try {
326
+ socket.destroy()
327
+ } catch {
328
+ /* nothing else to do */
329
+ }
330
+ resolve(free)
331
+ }
332
+ socket.once('connect', () => done(false))
333
+ socket.once('error', () => done(true))
334
+ socket.setTimeout(1000, () => done(true))
335
+ })
336
+ }
337
+
338
+ /**
339
+ * Wait until a port is free.
340
+ *
341
+ * @param {number} port
342
+ * @param {{ timeoutMs?: number, intervalMs?: number, host?: string, isFree?: Function }} [options]
343
+ * @returns {Promise<boolean>} whether the port became free in time
344
+ */
345
+ export async function waitForPortFree(port, options = {}) {
346
+ const timeout = Number.isFinite(Number(options.timeoutMs)) && Number(options.timeoutMs) > 0
347
+ ? Math.trunc(Number(options.timeoutMs))
348
+ : DEFAULT_PORT_FREE_TIMEOUT_MS
349
+ const interval = Number.isFinite(Number(options.intervalMs)) && Number(options.intervalMs) > 0
350
+ ? Math.trunc(Number(options.intervalMs))
351
+ : 500
352
+ const isFree = typeof options.isFree === 'function' ? options.isFree : isPortFree
353
+ const deadline = Date.now() + timeout
354
+ while (Date.now() < deadline) {
355
+ if (await isFree(port, options.host || '127.0.0.1')) return true
356
+ await new Promise((resolve) => setTimeout(resolve, interval))
357
+ }
358
+ return false
359
+ }
360
+
361
+ /**
362
+ * Read the configuration of the supervisor.
363
+ *
364
+ * @param {string} configPath
365
+ * @returns {{ ok: boolean, config: object|null, error: string|null }}
366
+ */
367
+ export function readConfig(configPath) {
368
+ const file = typeof configPath === 'string' ? configPath.trim() : ''
369
+ if (file === '') return { ok: false, config: null, error: 'readConfig requires a path' }
370
+ try {
371
+ const parsed = JSON.parse(fs.readFileSync(file, 'utf8'))
372
+ if (!parsed || typeof parsed !== 'object') return { ok: false, config: null, error: 'the configuration is not an object' }
373
+ return { ok: true, config: parsed, error: null }
374
+ } catch (error) {
375
+ return { ok: false, config: null, error: errorText(error) }
376
+ }
377
+ }
378
+
379
+ /**
380
+ * Append a line to the log file (and truncate it when it grew too big).
381
+ *
382
+ * @param {string} logPath
383
+ * @param {string} message
384
+ * @returns {boolean} whether the line was written
385
+ */
386
+ export function appendLog(logPath, message) {
387
+ const file = typeof logPath === 'string' ? logPath.trim() : ''
388
+ if (file === '') return false
389
+ const line = new Date().toISOString() + ' ' + String(message ?? '') + '\n'
390
+ try {
391
+ fs.mkdirSync(path.dirname(file), { recursive: true })
392
+ try {
393
+ const stat = fs.statSync(file)
394
+ if (stat.size > MAX_LOG_BYTES) fs.rmSync(file, { force: true })
395
+ } catch {
396
+ /* no log yet */
397
+ }
398
+ fs.appendFileSync(file, line, 'utf8')
399
+ return true
400
+ } catch {
401
+ return false
402
+ }
403
+ }
404
+
405
+ /**
406
+ * The answer of the control channel to one request.
407
+ *
408
+ * @param {unknown} request
409
+ * @returns {{ ok: boolean, action: string, error: string|null }}
410
+ */
411
+ export function buildControlResponse(request) {
412
+ const body = request && typeof request === 'object' ? request : {}
413
+ const action = typeof body.action === 'string' ? body.action.trim() : ''
414
+ if (action === 'ping') return { ok: true, action: 'ping', error: null }
415
+ if (action === 'restart') return { ok: true, action: 'restart', error: null }
416
+ if (action === 'stop') return { ok: true, action: 'stop', error: null }
417
+ return { ok: false, action: '', error: 'unknown action' }
418
+ }
419
+
420
+ /**
421
+ * Restore a snapshot over the profile.
422
+ *
423
+ * The same semantics as `restoreBackup` in the plugin, implemented here
424
+ * because the supervisor may not import the plugin.
425
+ *
426
+ * @param {{ backupDir?: string, profileDir?: string }} options
427
+ * @returns {{ ok: boolean, restored: string[], error: string|null }}
428
+ */
429
+ export function restoreSnapshot({ backupDir = '', profileDir = '' } = {}) {
430
+ const backup = typeof backupDir === 'string' ? backupDir.trim() : ''
431
+ const profile = typeof profileDir === 'string' ? profileDir.trim() : ''
432
+ const restored = []
433
+ if (backup === '') return { ok: false, restored, error: 'restoreSnapshot requires backupDir' }
434
+ if (profile === '') return { ok: false, restored, error: 'restoreSnapshot requires profileDir' }
435
+ try {
436
+ const configDir = path.join(backup, 'config')
437
+ for (const name of CONFIG_FILES) {
438
+ const source = path.join(configDir, name)
439
+ if (!fs.existsSync(source)) continue
440
+ fs.copyFileSync(source, path.join(profile, name))
441
+ restored.push(name)
442
+ }
443
+ const plugins = path.join(backup, PLUGINS_DIR)
444
+ if (fs.existsSync(plugins)) {
445
+ fs.cpSync(plugins, path.join(profile, PLUGINS_DIR), { recursive: true, force: true })
446
+ restored.push(PLUGINS_DIR)
447
+ }
448
+ } catch (error) {
449
+ return { ok: false, restored, error: errorText(error) }
450
+ }
451
+ if (restored.length === 0) return { ok: false, restored, error: 'the snapshot holds nothing that could be restored' }
452
+ return { ok: true, restored, error: null }
453
+ }
454
+
455
+ /**
456
+ * Read the rollback request the plugin left behind.
457
+ *
458
+ * @param {string} file
459
+ * @returns {object|null}
460
+ */
461
+ export function readRollbackRequest(file) {
462
+ try {
463
+ const parsed = JSON.parse(fs.readFileSync(String(file ?? ''), 'utf8'))
464
+ return parsed && typeof parsed === 'object' ? parsed : null
465
+ } catch {
466
+ return null
467
+ }
468
+ }
469
+
470
+ /**
471
+ * Ask the health route whether the host is alive.
472
+ *
473
+ * The URL is the LOOPBACK one WITHOUT a token: the plugin's health route
474
+ * answers loopback requests without authentication, so the supervisor never
475
+ * needs (and never sees) the boot token.
476
+ *
477
+ * @param {string} url
478
+ * @param {{ timeoutMs?: number }} [options]
479
+ * @returns {Promise<boolean>}
480
+ */
481
+ export function probeHealth(url, options = {}) {
482
+ const target = typeof url === 'string' ? url.trim() : ''
483
+ if (target === '') return Promise.resolve(false)
484
+ const timeout = Number.isFinite(Number(options.timeoutMs)) && Number(options.timeoutMs) > 0
485
+ ? Math.trunc(Number(options.timeoutMs))
486
+ : 5000
487
+ return new Promise((resolve) => {
488
+ let settled = false
489
+ const done = (healthy) => {
490
+ if (settled) return
491
+ settled = true
492
+ resolve(healthy)
493
+ }
494
+ let request
495
+ try {
496
+ request = http.get(target, { timeout }, (response) => {
497
+ const chunks = []
498
+ response.on('data', (chunk) => chunks.push(chunk))
499
+ response.on('end', () => {
500
+ try {
501
+ const body = JSON.parse(Buffer.concat(chunks).toString('utf8'))
502
+ done(body && body.ok === true)
503
+ } catch {
504
+ done(false)
505
+ }
506
+ })
507
+ response.on('error', () => done(false))
508
+ })
509
+ } catch {
510
+ done(false)
511
+ return
512
+ }
513
+ request.on('timeout', () => {
514
+ try {
515
+ request.destroy()
516
+ } catch {
517
+ /* nothing else to do */
518
+ }
519
+ done(false)
520
+ })
521
+ request.on('error', () => done(false))
522
+ })
523
+ }
524
+
525
+ /**
526
+ * Start the host and wait until it exits.
527
+ *
528
+ * @param {{
529
+ * config: object,
530
+ * generation: number,
531
+ * log: (message: string) => void,
532
+ * spawnFn?: Function,
533
+ * onControl?: (action: string) => void,
534
+ * }} options
535
+ * @returns {Promise<{ code: number|null, signal: string|null, stopped: boolean }>}
536
+ */
537
+ export async function runChild({ config, generation = 1, log = () => {}, spawnFn = spawn, onControl = null } = {}) {
538
+ const execPath = typeof config.execPath === 'string' ? config.execPath : ''
539
+ const execArgv = Array.isArray(config.execArgv) ? config.execArgv.map(String) : []
540
+ const argv = Array.isArray(config.argv) ? config.argv.map(String) : []
541
+ if (execPath === '' || argv.length === 0) {
542
+ log('the configuration has no start command')
543
+ return { code: null, signal: null, stopped: false }
544
+ }
545
+ const env = buildChildEnv(process.env, {
546
+ supervisorPid: process.pid,
547
+ generation,
548
+ profileEnv: config.profileEnv && typeof config.profileEnv === 'object' ? config.profileEnv : {},
549
+ })
550
+ log('starting the host: ' + [execPath, ...execArgv, ...argv].join(' '))
551
+ const child = spawnFn(execPath, [...execArgv, ...argv], {
552
+ cwd: typeof config.cwd === 'string' && config.cwd !== '' ? config.cwd : process.cwd(),
553
+ env,
554
+ stdio: ['ignore', 'pipe', 'pipe'],
555
+ })
556
+ if (child.stdout && typeof child.stdout.on === 'function') {
557
+ child.stdout.on('data', (chunk) => log('host: ' + String(chunk).trimEnd()))
558
+ }
559
+ if (child.stderr && typeof child.stderr.on === 'function') {
560
+ child.stderr.on('data', (chunk) => log('host: ' + String(chunk).trimEnd()))
561
+ }
562
+ return await new Promise((resolve) => {
563
+ let stopped = false
564
+ let settled = false
565
+ const finish = (code, signal) => {
566
+ if (settled) return
567
+ settled = true
568
+ resolve({ code, signal, stopped })
569
+ }
570
+ if (typeof onControl === 'function') {
571
+ onControl((action) => {
572
+ if (action === 'stop') {
573
+ stopped = true
574
+ try {
575
+ child.kill('SIGTERM')
576
+ } catch {
577
+ /* nothing else to do */
578
+ }
579
+ }
580
+ if (action === 'restart') {
581
+ try {
582
+ child.kill('SIGTERM')
583
+ } catch {
584
+ /* nothing else to do */
585
+ }
586
+ }
587
+ })
588
+ }
589
+ child.on('error', (error) => {
590
+ log('the host could not be started: ' + errorText(error))
591
+ finish(null, null)
592
+ })
593
+ child.on('exit', (code, signal) => {
594
+ log('the host exited (code ' + String(code) + ', signal ' + String(signal) + ')')
595
+ finish(code === undefined ? null : code, signal ?? null)
596
+ })
597
+ })
598
+ }
599
+
600
+ /**
601
+ * The control channel: a loopback TCP socket the plugin can talk to.
602
+ *
603
+ * @param {{ host?: string, port?: number, onAction?: (action: string) => void, log?: (message: string) => void }} options
604
+ * @returns {Promise<{ ok: boolean, port: number, close: () => void, error: string|null }>}
605
+ */
606
+ export function openControlChannel(options = {}) {
607
+ const host = typeof options.host === 'string' && options.host !== '' ? options.host : '127.0.0.1'
608
+ const log = typeof options.log === 'function' ? options.log : () => {}
609
+ return new Promise((resolve) => {
610
+ const server = net.createServer((socket) => {
611
+ let buffer = ''
612
+ socket.on('data', (chunk) => {
613
+ buffer += String(chunk)
614
+ let index = buffer.indexOf('\n')
615
+ while (index >= 0) {
616
+ const line = buffer.slice(0, index)
617
+ buffer = buffer.slice(index + 1)
618
+ let request = null
619
+ try {
620
+ request = JSON.parse(line)
621
+ } catch {
622
+ request = null
623
+ }
624
+ const response = buildControlResponse(request)
625
+ try {
626
+ socket.write(JSON.stringify(response) + '\n')
627
+ } catch {
628
+ /* the socket is gone */
629
+ }
630
+ if (response.ok && typeof options.onAction === 'function') options.onAction(response.action)
631
+ index = buffer.indexOf('\n')
632
+ }
633
+ })
634
+ socket.on('error', () => {
635
+ /* a broken client must not take the supervisor down */
636
+ })
637
+ })
638
+ server.on('error', (error) => {
639
+ log('the control channel could not be opened: ' + errorText(error))
640
+ resolve({ ok: false, port: 0, close: () => {}, error: errorText(error) })
641
+ })
642
+ server.listen(options.port && Number(options.port) > 0 ? Number(options.port) : 0, host, () => {
643
+ const address = server.address()
644
+ const port = address && typeof address === 'object' ? Number(address.port) : 0
645
+ resolve({
646
+ ok: true,
647
+ port,
648
+ close: () => {
649
+ try {
650
+ server.close()
651
+ } catch {
652
+ /* nothing else to do */
653
+ }
654
+ },
655
+ error: null,
656
+ })
657
+ })
658
+ })
659
+ }
660
+
661
+ /**
662
+ * The main loop of the supervisor.
663
+ *
664
+ * @param {{
665
+ * config: object,
666
+ * stateDir?: string,
667
+ * log?: (message: string) => void,
668
+ * spawnFn?: Function,
669
+ * probe?: (url: string, options: object) => Promise<boolean>,
670
+ * sleep?: (ms: number) => Promise<void>,
671
+ * now?: () => number,
672
+ * }} options
673
+ * @returns {Promise<{ ok: boolean, reason: string }>}
674
+ */
675
+ export async function runSupervisor(options = {}) {
676
+ const config = options.config && typeof options.config === 'object' ? options.config : {}
677
+ const log = typeof options.log === 'function' ? options.log : () => {}
678
+ const probe = typeof options.probe === 'function' ? options.probe : probeHealth
679
+ const sleep = typeof options.sleep === 'function'
680
+ ? options.sleep
681
+ : (ms) => new Promise((resolve) => setTimeout(resolve, ms))
682
+ const stateDir = typeof options.stateDir === 'string' && options.stateDir !== ''
683
+ ? options.stateDir
684
+ : (typeof config.stateDir === 'string' ? config.stateDir : '')
685
+ const supervisorDir = path.join(stateDir, 'supervisor')
686
+ const lockPath = path.join(supervisorDir, 'supervisor.lock')
687
+ const pidPath = path.join(supervisorDir, 'supervisor.pid')
688
+ const controlPath = path.join(supervisorDir, 'control.json')
689
+ const rollbackRequestPath = path.join(supervisorDir, 'rollback-request.json')
690
+ const reportPath = path.join(stateDir, 'last-rollback.json')
691
+
692
+ const lock = acquireLock(lockPath)
693
+ if (!lock.ok) {
694
+ log('another supervisor is already running: ' + String(lock.error))
695
+ return { ok: false, reason: String(lock.error) }
696
+ }
697
+ if (lock.stale) log('the lock of a dead supervisor was taken over')
698
+
699
+ const timings = config.timings && typeof config.timings === 'object' ? config.timings : {}
700
+ const baseDelay = Number(timings.restartDelayMs) > 0 ? Number(timings.restartDelayMs) : DEFAULT_RESTART_DELAY_MS
701
+ const maxDelay = Number(timings.restartDelayMaxMs) > 0 ? Number(timings.restartDelayMaxMs) : DEFAULT_RESTART_DELAY_MAX_MS
702
+ const startTimeout = Number(timings.startTimeoutMs) > 0 ? Number(timings.startTimeoutMs) : DEFAULT_START_TIMEOUT_MS
703
+ const killGrace = Number(timings.killGraceMs) > 0 ? Number(timings.killGraceMs) : DEFAULT_KILL_GRACE_MS
704
+ const portFreeTimeout = Number(timings.portFreeTimeoutMs) > 0
705
+ ? Number(timings.portFreeTimeoutMs)
706
+ : DEFAULT_PORT_FREE_TIMEOUT_MS
707
+
708
+ try {
709
+ fs.mkdirSync(supervisorDir, { recursive: true })
710
+ fs.writeFileSync(pidPath, String(process.pid), 'utf8')
711
+ } catch {
712
+ /* the lock is the real guard */
713
+ }
714
+
715
+ let pendingAction = null
716
+ const channel = await openControlChannel({
717
+ host: config.control && typeof config.control.host === 'string' ? config.control.host : '127.0.0.1',
718
+ port: config.control && Number(config.control.port) > 0 ? Number(config.control.port) : 0,
719
+ log,
720
+ onAction: (action) => {
721
+ pendingAction = action
722
+ },
723
+ })
724
+ if (!channel.ok) {
725
+ releaseLock(lockPath)
726
+ return { ok: false, reason: String(channel.error) }
727
+ }
728
+ try {
729
+ fs.writeFileSync(controlPath, JSON.stringify({
730
+ type: 'tcp',
731
+ host: config.control && typeof config.control.host === 'string' ? config.control.host : '127.0.0.1',
732
+ port: channel.port,
733
+ pid: process.pid,
734
+ at: new Date().toISOString(),
735
+ }), 'utf8')
736
+ } catch {
737
+ /* the control port is written into the config as well */
738
+ }
739
+ log('the supervisor is up (pid ' + process.pid + ', control port ' + channel.port + ')')
740
+
741
+ let delay = baseDelay
742
+ let generation = Number(config.generation) > 0 ? Number(config.generation) : 1
743
+ let stopping = false
744
+ try {
745
+ while (!stopping) {
746
+ const port = Number(config.port) > 0 ? Number(config.port) : 0
747
+ if (port > 0) {
748
+ const free = await waitForPortFree(port, { timeoutMs: portFreeTimeout })
749
+ if (!free) log('the port ' + port + ' is still taken; starting anyway')
750
+ }
751
+ const result = await runChild({
752
+ config,
753
+ generation,
754
+ log,
755
+ spawnFn: options.spawnFn,
756
+ onControl: (action) => {
757
+ pendingAction = action
758
+ },
759
+ })
760
+ if (pendingAction === 'stop') {
761
+ stopping = true
762
+ log('the supervisor was asked to stop')
763
+ break
764
+ }
765
+ if (pendingAction === 'restart') {
766
+ pendingAction = null
767
+ log('the supervisor was asked to restart the host')
768
+ delay = baseDelay
769
+ continue
770
+ }
771
+ pendingAction = null
772
+
773
+ // The host exited. Did it come back on its own (a self-healing host)?
774
+ const deadline = Date.now() + startTimeout
775
+ let healthy = false
776
+ while (Date.now() < deadline) {
777
+ healthy = await probe(config.healthUrl, { timeoutMs: 5000 })
778
+ if (healthy) break
779
+ await sleep(2000)
780
+ }
781
+ if (healthy) {
782
+ log('the host is answering its health route again; the supervisor waits')
783
+ delay = shouldResetBackoff(true) ? baseDelay : delay
784
+ await sleep(delay)
785
+ continue
786
+ }
787
+
788
+ // It never came back. A snapshot may be waiting to be put back.
789
+ const request = readRollbackRequest(rollbackRequestPath)
790
+ if (request && typeof request.backupDir === 'string') {
791
+ log('the host never came back; restoring the snapshot')
792
+ const restored = restoreSnapshot({
793
+ backupDir: request.backupDir,
794
+ profileDir: typeof request.profileDir === 'string' ? request.profileDir : '',
795
+ })
796
+ try {
797
+ fs.writeFileSync(reportPath, JSON.stringify({
798
+ ok: false,
799
+ at: new Date().toISOString(),
800
+ waitedMs: startTimeout,
801
+ rolledBack: restored.ok === true,
802
+ rollbackError: restored.ok === true ? null : String(restored.error),
803
+ reason: 'the host never came back',
804
+ backupDir: request.backupDir,
805
+ appStarted: false,
806
+ }), 'utf8')
807
+ } catch {
808
+ /* the report is advisory */
809
+ }
810
+ try {
811
+ fs.rmSync(rollbackRequestPath, { force: true })
812
+ } catch {
813
+ /* nothing else to do */
814
+ }
815
+ generation += 1
816
+ continue
817
+ }
818
+
819
+ delay = nextRestartDelay(delay, { baseMs: baseDelay, maxMs: maxDelay })
820
+ log('restarting the host in ' + delay + 'ms')
821
+ await sleep(delay)
822
+ }
823
+ } finally {
824
+ channel.close()
825
+ try {
826
+ fs.rmSync(pidPath, { force: true })
827
+ } catch {
828
+ /* nothing else to do */
829
+ }
830
+ releaseLock(lockPath)
831
+ }
832
+ return { ok: true, reason: 'the supervisor stopped' }
833
+ }
834
+
835
+ /**
836
+ * The entry point: `node supervisor.js --config <supervisor.json>`.
837
+ *
838
+ * @param {string[]} [argv]
839
+ * @returns {Promise<{ ok: boolean, reason: string }>}
840
+ */
841
+ export async function main(argv = process.argv.slice(2)) {
842
+ const args = parseArgs(argv)
843
+ if (args.help) {
844
+ // eslint-disable-next-line no-console
845
+ console.log('usage: node supervisor.js --config <supervisor.json>')
846
+ return { ok: true, reason: 'help' }
847
+ }
848
+ if (!args.configPath) {
849
+ // eslint-disable-next-line no-console
850
+ console.error('the supervisor needs --config <supervisor.json>')
851
+ return { ok: false, reason: 'no --config' }
852
+ }
853
+ const read = readConfig(args.configPath)
854
+ if (!read.ok) {
855
+ // eslint-disable-next-line no-console
856
+ console.error('the supervisor configuration could not be read: ' + String(read.error))
857
+ return { ok: false, reason: String(read.error) }
858
+ }
859
+ const config = read.config
860
+ const stateDir = typeof config.stateDir === 'string' ? config.stateDir : path.dirname(path.dirname(args.configPath))
861
+ const logPath = path.join(stateDir, 'supervisor', 'supervisor.log')
862
+ const log = (message) => {
863
+ appendLog(logPath, message)
864
+ // eslint-disable-next-line no-console
865
+ console.log(message)
866
+ }
867
+ return await runSupervisor({ config: { ...config, stateDir }, stateDir, log })
868
+ }
869
+
870
+ // The file is a script AND a module: the tests import the pure pieces, the
871
+ // scheduler runs it.
872
+ const invokedDirectly = (() => {
873
+ try {
874
+ const entry = process.argv[1]
875
+ if (!entry) return false
876
+ return path.resolve(entry) === path.resolve(new URL(import.meta.url).pathname.replace(/^\/([A-Za-z]:)/, '$1'))
877
+ } catch {
878
+ return false
879
+ }
880
+ })()
881
+
882
+ if (invokedDirectly) {
883
+ void main()
884
+ }