dsh-autorestart 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +97 -0
- package/README.md +55 -6
- package/client.js +102 -9
- package/docs/README.zh.md +4 -2
- package/index.js +514 -39
- package/lib/backup.js +279 -9
- package/lib/checkpoint.js +134 -0
- package/lib/diagnostics.js +18 -0
- package/lib/restart-phase.js +263 -0
- package/lib/watchdog-disarm.js +506 -0
- package/lib/watchdog.js +86 -0
- package/locale/en.json +1 -0
- package/locale/zh.json +1 -0
- package/package.json +6 -1
- package/scripts/drill-rescue.sh +427 -0
- package/scripts/drill-watchdog.mjs +474 -0
- package/scripts/emergency-disable.sh +314 -0
- package/scripts/rescue-strings.zh.json +168 -0
- package/scripts/restore-profile.sh +593 -0
- package/scripts/watchdog.mjs +310 -0
package/index.js
CHANGED
|
@@ -49,9 +49,17 @@ import {
|
|
|
49
49
|
setHandoff,
|
|
50
50
|
setRecoveryOutcome,
|
|
51
51
|
setPhase,
|
|
52
|
+
setRestartPhase,
|
|
53
|
+
setRestartProgress,
|
|
54
|
+
clearRestartPhase,
|
|
52
55
|
stampResumedGoal,
|
|
53
56
|
writeState,
|
|
54
57
|
} from './lib/checkpoint.js'
|
|
58
|
+
import {
|
|
59
|
+
cancelRefusalReason,
|
|
60
|
+
describeRestartPhase,
|
|
61
|
+
RESTART_PHASES,
|
|
62
|
+
} from './lib/restart-phase.js'
|
|
55
63
|
import {
|
|
56
64
|
bootPlan,
|
|
57
65
|
composeHandoff,
|
|
@@ -117,13 +125,18 @@ import {
|
|
|
117
125
|
} from './lib/watchdog-tier.js'
|
|
118
126
|
import {
|
|
119
127
|
buildPending,
|
|
128
|
+
clearPending,
|
|
120
129
|
clearReady,
|
|
121
|
-
|
|
130
|
+
readPending,
|
|
122
131
|
readReport,
|
|
123
132
|
writePending,
|
|
124
|
-
writeReady,
|
|
125
133
|
writeReport,
|
|
126
134
|
} from './lib/watchdog.js'
|
|
135
|
+
import {
|
|
136
|
+
disarmWatchdog as runDisarmWatchdog,
|
|
137
|
+
markReady,
|
|
138
|
+
readDisarmRecord,
|
|
139
|
+
} from './lib/watchdog-disarm.js'
|
|
127
140
|
import {
|
|
128
141
|
ARBITERS,
|
|
129
142
|
planWatchdogArbiter,
|
|
@@ -374,7 +387,7 @@ function snapGraceChoice(value, fallback = DEFAULT_RESTART_GRACE_MS) {
|
|
|
374
387
|
* `pluginVersion()`), but a bundled installation may not ship that file — so
|
|
375
388
|
* the constant below is the last resort. A test keeps the two in sync.
|
|
376
389
|
*/
|
|
377
|
-
const PLUGIN_VERSION_FALLBACK = '0.8.
|
|
390
|
+
const PLUGIN_VERSION_FALLBACK = '0.8.2'
|
|
378
391
|
|
|
379
392
|
/**
|
|
380
393
|
* The version of the graceful-shutdown FEATURE, not the version of the package.
|
|
@@ -1398,12 +1411,16 @@ function backupsDirFor(dir) {
|
|
|
1398
1411
|
* error: string|null,
|
|
1399
1412
|
* }}
|
|
1400
1413
|
*/
|
|
1401
|
-
export function takePreRestartSnapshot({
|
|
1414
|
+
export async function takePreRestartSnapshot({
|
|
1402
1415
|
config = {},
|
|
1403
1416
|
stateDir = '',
|
|
1404
1417
|
log = console,
|
|
1405
1418
|
profile = null,
|
|
1406
1419
|
createBackup: createBackupFn = createBackup,
|
|
1420
|
+
// The REAL progress of the copy, and the way out of it: both are handed to
|
|
1421
|
+
// the backup itself, so the numbers on screen are the numbers on disk.
|
|
1422
|
+
onProgress = null,
|
|
1423
|
+
shouldCancel = null,
|
|
1407
1424
|
} = {}) {
|
|
1408
1425
|
const startedAt = Date.now()
|
|
1409
1426
|
const info = profile && typeof profile === 'object'
|
|
@@ -1427,17 +1444,33 @@ export function takePreRestartSnapshot({
|
|
|
1427
1444
|
}
|
|
1428
1445
|
let backup = null
|
|
1429
1446
|
try {
|
|
1430
|
-
backup = createBackupFn({
|
|
1447
|
+
backup = await createBackupFn({
|
|
1431
1448
|
profileDir,
|
|
1432
1449
|
backupsDir: backupsDirFor(stateDir),
|
|
1433
1450
|
includePlugins: config.backupIncludePlugins !== false,
|
|
1434
1451
|
maxBackups: config.maxBackups,
|
|
1452
|
+
onProgress: typeof onProgress === 'function' ? onProgress : null,
|
|
1453
|
+
shouldCancel: typeof shouldCancel === 'function' ? shouldCancel : null,
|
|
1435
1454
|
})
|
|
1436
1455
|
} catch (error) {
|
|
1437
1456
|
// A backup that throws must not take the restart down with it.
|
|
1438
1457
|
backup = { ok: false, error: String(error) }
|
|
1439
1458
|
}
|
|
1440
1459
|
const durationMs = Math.max(0, Date.now() - startedAt)
|
|
1460
|
+
if (backup && backup.cancelled === true) {
|
|
1461
|
+
// The user called it off: the half-written snapshot was already removed by
|
|
1462
|
+
// the copy loop, and there is nothing to report as a rollback point.
|
|
1463
|
+
log.warn('the pre-restart backup was cancelled by the user', { durationMs })
|
|
1464
|
+
return {
|
|
1465
|
+
backup,
|
|
1466
|
+
profileDir,
|
|
1467
|
+
profile: info,
|
|
1468
|
+
requested: true,
|
|
1469
|
+
durationMs,
|
|
1470
|
+
cancelled: true,
|
|
1471
|
+
error: null,
|
|
1472
|
+
}
|
|
1473
|
+
}
|
|
1441
1474
|
if (backup && backup.ok === true) {
|
|
1442
1475
|
log.warn('pre-restart backup written', {
|
|
1443
1476
|
dir: backup.dir,
|
|
@@ -1645,6 +1678,17 @@ export function createRuntime(ctx, config, options = {}) {
|
|
|
1645
1678
|
let recoverTimer
|
|
1646
1679
|
let disposed = false
|
|
1647
1680
|
let lastPersistError
|
|
1681
|
+
/**
|
|
1682
|
+
* THE LAST DISARM of the watchdog: the record of "the host is up, so nothing
|
|
1683
|
+
* may roll it back" (see lib/watchdog-disarm.js). It is kept on `runtime` so
|
|
1684
|
+
* the settings page and the diagnostics can show a FAILED disarm instead of
|
|
1685
|
+
* the silence that caused the 0.8.2 incident.
|
|
1686
|
+
*/
|
|
1687
|
+
let watchdogDisarm = readDisarmRecord(path.dirname(stateFile))
|
|
1688
|
+
/** The disarm that is currently running (one at a time; null when idle). */
|
|
1689
|
+
let disarmInFlight = null
|
|
1690
|
+
/** A disarm was asked for while one was running: run once more afterwards. */
|
|
1691
|
+
let disarmPendingAgain = false
|
|
1648
1692
|
/**
|
|
1649
1693
|
* The tool registrations that failed. A failed registration is recorded and
|
|
1650
1694
|
* logged, never thrown: the plugin still loads and the other tools still work.
|
|
@@ -2605,6 +2649,93 @@ const titleOfSummary = (item) => {
|
|
|
2605
2649
|
*/
|
|
2606
2650
|
const stateDirOf = () => path.dirname(stateFile)
|
|
2607
2651
|
|
|
2652
|
+
/**
|
|
2653
|
+
* THE STAGE LINE of a restart ("waiting / backing up / restarting / back").
|
|
2654
|
+
*
|
|
2655
|
+
* It is written ONCE per stage change and persisted IMMEDIATELY, because the
|
|
2656
|
+
* whole point of it is the first second after the click: a stage that reaches
|
|
2657
|
+
* the interface only after the 156 MB snapshot would tell the user nothing
|
|
2658
|
+
* about the ten seconds they spent wondering whether the button worked.
|
|
2659
|
+
*
|
|
2660
|
+
* @param {string} phase one of RESTART_PHASES
|
|
2661
|
+
* @param {{ blocked?: number, now?: number }} [options]
|
|
2662
|
+
* @returns {object|null} the recorded stage
|
|
2663
|
+
*/
|
|
2664
|
+
const markRestartStage = (phase, { blocked = 0, now = Date.now() } = {}) => {
|
|
2665
|
+
if (!RESTART_PHASES.includes(phase)) return null
|
|
2666
|
+
const record = setRestartPhase(state, { phase, blocked, now })
|
|
2667
|
+
persist()
|
|
2668
|
+
return record
|
|
2669
|
+
}
|
|
2670
|
+
|
|
2671
|
+
/**
|
|
2672
|
+
* Record the REAL progress of the running snapshot (files and bytes).
|
|
2673
|
+
*
|
|
2674
|
+
* The copy loop calls this between files; the checkpoint is written with a
|
|
2675
|
+
* small throttle, because a state write per copied file would slow the copy
|
|
2676
|
+
* down for no gain. The in-memory record is always current, so a `state=1`
|
|
2677
|
+
* poll that arrives between two writes still sees the newest numbers.
|
|
2678
|
+
*
|
|
2679
|
+
* @param {{ files?: number, bytes?: number, total?: number, filesTotal?: number, now?: number }} progress
|
|
2680
|
+
*/
|
|
2681
|
+
let lastProgressPersist = 0
|
|
2682
|
+
const recordSnapshotProgress = (progress) => {
|
|
2683
|
+
const now = Date.now()
|
|
2684
|
+
const record = setRestartProgress(state, { ...progress, now })
|
|
2685
|
+
if (now - lastProgressPersist >= 250) {
|
|
2686
|
+
lastProgressPersist = now
|
|
2687
|
+
persist()
|
|
2688
|
+
}
|
|
2689
|
+
return record
|
|
2690
|
+
}
|
|
2691
|
+
|
|
2692
|
+
/** Forget the stage (cancel, or a restart that could not be started). */
|
|
2693
|
+
const clearRestartStage = () => {
|
|
2694
|
+
if (clearRestartPhase(state)) persist()
|
|
2695
|
+
}
|
|
2696
|
+
|
|
2697
|
+
/**
|
|
2698
|
+
* The stage as every surface shows it (state route, health route, client).
|
|
2699
|
+
*
|
|
2700
|
+
* @returns {object} the described stage (always an object, never null)
|
|
2701
|
+
*/
|
|
2702
|
+
const restartStageInfo = () => {
|
|
2703
|
+
const record = state.restartPhase
|
|
2704
|
+
const blocked = (() => {
|
|
2705
|
+
try {
|
|
2706
|
+
return countBlockerTasks(gate().blockedBy)
|
|
2707
|
+
} catch {
|
|
2708
|
+
return 0
|
|
2709
|
+
}
|
|
2710
|
+
})()
|
|
2711
|
+
return describeRestartPhase({
|
|
2712
|
+
phase: record && typeof record === 'object' ? record.phase : 'idle',
|
|
2713
|
+
startedAt: record && typeof record === 'object' ? record.startedAt : null,
|
|
2714
|
+
at: record && typeof record === 'object' ? record.at : null,
|
|
2715
|
+
progress: record && typeof record === 'object' ? record.progress : null,
|
|
2716
|
+
blocking: record && typeof record === 'object' && Number.isFinite(record.blocked) ? record.blocked : blocked,
|
|
2717
|
+
now: Date.now(),
|
|
2718
|
+
})
|
|
2719
|
+
}
|
|
2720
|
+
|
|
2721
|
+
/**
|
|
2722
|
+
* A cancel request that arrived while the snapshot was being written.
|
|
2723
|
+
*
|
|
2724
|
+
* The backup loop asks this between files: the copy stops, its half-written
|
|
2725
|
+
* directory is removed, and the host never restarts. A flag (not a thrown
|
|
2726
|
+
* error) because the loop is synchronous and must finish its bookkeeping.
|
|
2727
|
+
*/
|
|
2728
|
+
let snapshotCancelRequested = false
|
|
2729
|
+
const requestSnapshotCancel = () => {
|
|
2730
|
+
snapshotCancelRequested = true
|
|
2731
|
+
return true
|
|
2732
|
+
}
|
|
2733
|
+
const snapshotCancelled = () => snapshotCancelRequested === true
|
|
2734
|
+
const resetSnapshotCancel = () => {
|
|
2735
|
+
snapshotCancelRequested = false
|
|
2736
|
+
}
|
|
2737
|
+
|
|
2738
|
+
|
|
2608
2739
|
/**
|
|
2609
2740
|
* The durable record of the last restart: the snapshot that was taken, the
|
|
2610
2741
|
* verdict of the new instance and its four boot checks.
|
|
@@ -2616,17 +2747,14 @@ const titleOfSummary = (item) => {
|
|
|
2616
2747
|
/**
|
|
2617
2748
|
* The version of THIS plugin (the one the user runs).
|
|
2618
2749
|
*
|
|
2750
|
+
* It delegates to the module-level reader ON PURPOSE: a local copy used to
|
|
2751
|
+
* look for `../package.json`, which is one directory ABOVE the plugin, so
|
|
2752
|
+
* every answer was `unknown` — and the health route reported that instead of
|
|
2753
|
+
* the version that is really loaded.
|
|
2754
|
+
*
|
|
2619
2755
|
* @returns {string}
|
|
2620
2756
|
*/
|
|
2621
|
-
const
|
|
2622
|
-
try {
|
|
2623
|
-
const raw = fs.readFileSync(new URL('../package.json', import.meta.url), 'utf8')
|
|
2624
|
-
const parsed = JSON.parse(raw)
|
|
2625
|
-
return typeof parsed.version === 'string' && parsed.version !== '' ? parsed.version : 'unknown'
|
|
2626
|
-
} catch {
|
|
2627
|
-
return 'unknown'
|
|
2628
|
-
}
|
|
2629
|
-
}
|
|
2757
|
+
const ownVersion = () => pluginVersion()
|
|
2630
2758
|
|
|
2631
2759
|
/**
|
|
2632
2760
|
* Write (or update) the durable record of the restart.
|
|
@@ -2960,7 +3088,7 @@ const titleOfSummary = (item) => {
|
|
|
2960
3088
|
* @returns {string}
|
|
2961
3089
|
*/
|
|
2962
3090
|
const diagnosticsText = () => buildDiagnostics({
|
|
2963
|
-
version:
|
|
3091
|
+
version: ownVersion(),
|
|
2964
3092
|
platform: env.platform,
|
|
2965
3093
|
hostKind: hostKindOf().kind,
|
|
2966
3094
|
isDesktopApp: env.isDesktopApp === true,
|
|
@@ -2970,6 +3098,9 @@ const titleOfSummary = (item) => {
|
|
|
2970
3098
|
lastRestart: lastRestartVerdict(),
|
|
2971
3099
|
stateDir: stateDirOf(),
|
|
2972
3100
|
watchdog: watchdogInfo(),
|
|
3101
|
+
// DID THE DISARM WORK? A failed disarm leaves an armed watchdog behind,
|
|
3102
|
+
// and that is the first thing a bug report has to show.
|
|
3103
|
+
watchdogDisarm: summarizeDisarm(watchdogDisarm),
|
|
2973
3104
|
})
|
|
2974
3105
|
|
|
2975
3106
|
/**
|
|
@@ -3540,13 +3671,50 @@ const titleOfSummary = (item) => {
|
|
|
3540
3671
|
// app relaunch and the custom restart command share this single entry, so
|
|
3541
3672
|
// a hosted host that is restarted by a command is just as recoverable as
|
|
3542
3673
|
// the desktop app.
|
|
3674
|
+
//
|
|
3675
|
+
// THE STAGE IS ON SCREEN BEFORE THE COPY STARTS, and the copy reports the
|
|
3676
|
+
// files and bytes it really writes. That is the fix of "I clicked restart
|
|
3677
|
+
// and nothing happened for ten seconds": the ten seconds ARE the snapshot,
|
|
3678
|
+
// and now they say so.
|
|
3543
3679
|
const stateDir = path.dirname(stateFile)
|
|
3544
|
-
|
|
3545
|
-
|
|
3546
|
-
|
|
3547
|
-
|
|
3548
|
-
|
|
3549
|
-
|
|
3680
|
+
resetSnapshotCancel()
|
|
3681
|
+
markRestartStage('snapshotting')
|
|
3682
|
+
let cancelled = false
|
|
3683
|
+
let snapshot
|
|
3684
|
+
try {
|
|
3685
|
+
snapshot = await snapshotFn({
|
|
3686
|
+
config,
|
|
3687
|
+
stateDir,
|
|
3688
|
+
log,
|
|
3689
|
+
profile: resolveProfileDirDetailed(),
|
|
3690
|
+
onProgress: (progress) => {
|
|
3691
|
+
recordSnapshotProgress(progress)
|
|
3692
|
+
},
|
|
3693
|
+
shouldCancel: () => snapshotCancelled(),
|
|
3694
|
+
})
|
|
3695
|
+
} finally {
|
|
3696
|
+
// The flag belongs to ONE snapshot; a stale `true` would cancel the
|
|
3697
|
+
// next restart before it began.
|
|
3698
|
+
const wasCancelled = snapshotCancelled()
|
|
3699
|
+
resetSnapshotCancel()
|
|
3700
|
+
// Two ways a copy can be called off: the flag this process set (the
|
|
3701
|
+
// cancel route) or the snapshot reporting it itself. Both mean the same
|
|
3702
|
+
// thing, and both must stop the restart.
|
|
3703
|
+
cancelled = wasCancelled || (snapshot !== null && typeof snapshot === 'object' && snapshot.cancelled === true)
|
|
3704
|
+
}
|
|
3705
|
+
if (cancelled) {
|
|
3706
|
+
// The user called it off while the files were being copied: the
|
|
3707
|
+
// half-written snapshot is already gone (createBackup removes it), the
|
|
3708
|
+
// host does not restart, and the request is WITHDRAWN — a restart the
|
|
3709
|
+
// user cancelled must not be retried five seconds later.
|
|
3710
|
+
log.warn('the restart was cancelled while the snapshot was being written')
|
|
3711
|
+
clearRestartStage()
|
|
3712
|
+
releaseRequesters()
|
|
3713
|
+
cancelRestart(state, { now: Date.now() })
|
|
3714
|
+
persist()
|
|
3715
|
+
stopPolling()
|
|
3716
|
+
return
|
|
3717
|
+
}
|
|
3550
3718
|
// The snapshot is the rollback point of BOTH branches.
|
|
3551
3719
|
const backup = snapshot && snapshot.backup ? snapshot.backup : null
|
|
3552
3720
|
const profileDir = snapshot && snapshot.profileDir ? snapshot.profileDir : null
|
|
@@ -3677,6 +3845,10 @@ const titleOfSummary = (item) => {
|
|
|
3677
3845
|
// looked at "the spawn did not throw", so a command that does not
|
|
3678
3846
|
// exist or that exits with an error was reported as a successful
|
|
3679
3847
|
// restart.
|
|
3848
|
+
//
|
|
3849
|
+
// FROM HERE ON THE RESTART CANNOT BE CANCELLED, and the interface says
|
|
3850
|
+
// so: the host is on its way out.
|
|
3851
|
+
markRestartStage('restarting')
|
|
3680
3852
|
const outcome = await runCommandFn({
|
|
3681
3853
|
command,
|
|
3682
3854
|
timeoutMs: config.restartCommandTimeoutMs,
|
|
@@ -3814,8 +3986,15 @@ const titleOfSummary = (item) => {
|
|
|
3814
3986
|
watchScript,
|
|
3815
3987
|
})
|
|
3816
3988
|
log.warn('scheduled detached restart:', result)
|
|
3817
|
-
if (!result || result.ok !== true)
|
|
3818
|
-
|
|
3989
|
+
if (!result || result.ok !== true) {
|
|
3990
|
+
markRestartStage('waiting', { blocked: 0 })
|
|
3991
|
+
rollbackRestart((result && result.error) || 'detached restart refused')
|
|
3992
|
+
} else {
|
|
3993
|
+
// The helper is scheduled: the host is going down, and from here on a
|
|
3994
|
+
// cancel would be a promise this plugin cannot keep.
|
|
3995
|
+
markRestartStage('restarting')
|
|
3996
|
+
armGateWatchdog()
|
|
3997
|
+
}
|
|
3819
3998
|
}
|
|
3820
3999
|
} catch (error) {
|
|
3821
4000
|
rollbackRestart(String(error))
|
|
@@ -4059,6 +4238,157 @@ const titleOfSummary = (item) => {
|
|
|
4059
4238
|
get bootId() {
|
|
4060
4239
|
return bootId
|
|
4061
4240
|
},
|
|
4241
|
+
/**
|
|
4242
|
+
* THE STATE DIRECTORY of this boot — the checkpoint, the watchdog markers
|
|
4243
|
+
* and the scheduling-item leftovers all live in it.
|
|
4244
|
+
*
|
|
4245
|
+
* It is exposed ON PURPOSE: the health route is a module-level function
|
|
4246
|
+
* and used to read the directory from a variable that only existed inside
|
|
4247
|
+
* this factory. The lookup threw `ReferenceError: stateDirOf is not
|
|
4248
|
+
* defined` on every request, an empty `catch {}` ate it, and a healthy
|
|
4249
|
+
* host left an armed watchdog behind (the 0.8.2 incident). Anything a
|
|
4250
|
+
* route needs must be reachable through `runtime`, never through a
|
|
4251
|
+
* closure it cannot see.
|
|
4252
|
+
*/
|
|
4253
|
+
get stateDir() {
|
|
4254
|
+
return path.dirname(stateFile)
|
|
4255
|
+
},
|
|
4256
|
+
/**
|
|
4257
|
+
* The version of the plugin that is actually LOADED in this process.
|
|
4258
|
+
*
|
|
4259
|
+
* The health route reports it so a real-machine check can tell "the new
|
|
4260
|
+
* code is running" from "the old code is still loaded".
|
|
4261
|
+
*/
|
|
4262
|
+
get pluginVersion() {
|
|
4263
|
+
return ownVersion()
|
|
4264
|
+
},
|
|
4265
|
+
/**
|
|
4266
|
+
* DISARM THE WATCHDOG: "this host is up, so nothing may roll it back".
|
|
4267
|
+
*
|
|
4268
|
+
* The ready marker is written on the FIRST tick of the first attempt (so a
|
|
4269
|
+
* caller inside an HTTP route has it on disk before it answers), the
|
|
4270
|
+
* pending marker and the scheduling item are cleaned with retries, and the
|
|
4271
|
+
* record is kept on `runtime` and written next to the markers.
|
|
4272
|
+
*
|
|
4273
|
+
* One run at a time: a second call while one is running returns the same
|
|
4274
|
+
* promise instead of starting a second `schtasks` dance.
|
|
4275
|
+
*
|
|
4276
|
+
* @param {string} [source] where the call came from ('boot', 'health-route', …)
|
|
4277
|
+
* @returns {Promise<object>} the record
|
|
4278
|
+
*/
|
|
4279
|
+
disarmWatchdog(source = 'boot') {
|
|
4280
|
+
if (disarmInFlight !== null) {
|
|
4281
|
+
// One run at a time (two `schtasks /Delete` calls fight each other),
|
|
4282
|
+
// but a request that arrives DURING a run is remembered: a pending
|
|
4283
|
+
// marker written in the meantime must still be cleaned, not lost.
|
|
4284
|
+
disarmPendingAgain = true
|
|
4285
|
+
return disarmInFlight
|
|
4286
|
+
}
|
|
4287
|
+
const one = (where) => {
|
|
4288
|
+
const port = Number(runtime.webPort)
|
|
4289
|
+
return runDisarmWatchdog({
|
|
4290
|
+
stateDir: stateDirOf(),
|
|
4291
|
+
bootId,
|
|
4292
|
+
port: Number.isFinite(port) && port > 0 ? port : null,
|
|
4293
|
+
source: where,
|
|
4294
|
+
log,
|
|
4295
|
+
})
|
|
4296
|
+
.then((record) => {
|
|
4297
|
+
watchdogDisarm = record
|
|
4298
|
+
if (record.ok === true) {
|
|
4299
|
+
log.info('the watchdog is disarmed (' + where + ')', {
|
|
4300
|
+
attempts: record.attempts,
|
|
4301
|
+
tier: record.tier,
|
|
4302
|
+
})
|
|
4303
|
+
}
|
|
4304
|
+
return record
|
|
4305
|
+
})
|
|
4306
|
+
.catch((error) => {
|
|
4307
|
+
// The disarm must never reject: the health route and the boot path
|
|
4308
|
+
// both call it without a `catch`.
|
|
4309
|
+
watchdogDisarm = {
|
|
4310
|
+
at: new Date().toISOString(),
|
|
4311
|
+
source: String(where),
|
|
4312
|
+
bootId,
|
|
4313
|
+
attempts: 0,
|
|
4314
|
+
ok: false,
|
|
4315
|
+
ready: false,
|
|
4316
|
+
pending: null,
|
|
4317
|
+
tier: null,
|
|
4318
|
+
scheduleRemoved: null,
|
|
4319
|
+
scheduleError: null,
|
|
4320
|
+
errors: [{
|
|
4321
|
+
attempt: 0,
|
|
4322
|
+
step: 'disarm',
|
|
4323
|
+
name: error && error.name ? String(error.name) : 'Error',
|
|
4324
|
+
message: error && error.message ? String(error.message) : String(error),
|
|
4325
|
+
}],
|
|
4326
|
+
}
|
|
4327
|
+
log.warn('the watchdog could not be disarmed:', String(error))
|
|
4328
|
+
return watchdogDisarm
|
|
4329
|
+
})
|
|
4330
|
+
}
|
|
4331
|
+
const drive = async () => {
|
|
4332
|
+
let record = await one(source)
|
|
4333
|
+
while (disarmPendingAgain === true) {
|
|
4334
|
+
disarmPendingAgain = false
|
|
4335
|
+
record = await one(String(source) + '+again')
|
|
4336
|
+
}
|
|
4337
|
+
return record
|
|
4338
|
+
}
|
|
4339
|
+
const running = drive().then(
|
|
4340
|
+
(record) => {
|
|
4341
|
+
disarmInFlight = null
|
|
4342
|
+
disarmPendingAgain = false
|
|
4343
|
+
return record
|
|
4344
|
+
},
|
|
4345
|
+
(error) => {
|
|
4346
|
+
disarmInFlight = null
|
|
4347
|
+
disarmPendingAgain = false
|
|
4348
|
+
throw error
|
|
4349
|
+
},
|
|
4350
|
+
)
|
|
4351
|
+
disarmInFlight = running
|
|
4352
|
+
return running
|
|
4353
|
+
},
|
|
4354
|
+
/**
|
|
4355
|
+
* The record of the last disarm (null when this host never disarmed one).
|
|
4356
|
+
*/
|
|
4357
|
+
get watchdogDisarm() {
|
|
4358
|
+
return watchdogDisarm
|
|
4359
|
+
},
|
|
4360
|
+
// THE STAGE OF A RESTART, as every surface reads it. `markRestartStage`
|
|
4361
|
+
// writes it and persists immediately (that is what makes the first stage
|
|
4362
|
+
// appear within the first second after the click), `recordSnapshotProgress`
|
|
4363
|
+
// carries the REAL bytes of the running copy, and `restartStageInfo` is the
|
|
4364
|
+
// single description (text included) that the state route, the health route
|
|
4365
|
+
// and the settings page share.
|
|
4366
|
+
markRestartStage,
|
|
4367
|
+
recordSnapshotProgress,
|
|
4368
|
+
clearRestartStage,
|
|
4369
|
+
restartStageInfo,
|
|
4370
|
+
requestSnapshotCancel,
|
|
4371
|
+
snapshotCancelled,
|
|
4372
|
+
resetSnapshotCancel,
|
|
4373
|
+
/**
|
|
4374
|
+
* The stage of a restart, cut down for the health route (no long texts).
|
|
4375
|
+
*
|
|
4376
|
+
* @returns {object} `{ phase, at, progress, slow, cancellable }`
|
|
4377
|
+
*/
|
|
4378
|
+
restartStageBrief() {
|
|
4379
|
+
const info = restartStageInfo()
|
|
4380
|
+
return {
|
|
4381
|
+
phase: info.phase,
|
|
4382
|
+
at: info.at,
|
|
4383
|
+
elapsedMs: info.elapsedMs,
|
|
4384
|
+
progress: info.progress === null
|
|
4385
|
+
? null
|
|
4386
|
+
: { files: info.progress.files, bytes: info.progress.bytes, total: info.progress.total, percent: info.progress.percent, ageMs: info.progress.ageMs },
|
|
4387
|
+
slow: info.slow,
|
|
4388
|
+
cancellable: info.cancellable,
|
|
4389
|
+
cancelReason: info.cancelReason,
|
|
4390
|
+
}
|
|
4391
|
+
},
|
|
4062
4392
|
/**
|
|
4063
4393
|
* Where the effective `backupIncludePlugins` value came from:
|
|
4064
4394
|
* `config` (the profile row), `fallback-file` (the degraded path) or
|
|
@@ -5149,6 +5479,10 @@ function registerTools(ctx, runtime, config) {
|
|
|
5149
5479
|
if (typeof args.defer_seconds === 'number' && Number.isFinite(args.defer_seconds) && args.defer_seconds > 0) {
|
|
5150
5480
|
state.restart.requestedAt = now + Math.min(args.defer_seconds, 3600) * 1000
|
|
5151
5481
|
}
|
|
5482
|
+
// THE FIRST STAGE, RIGHT NOW — not when the backup starts. The interface
|
|
5483
|
+
// polls this out of the state route, so the click produces a visible
|
|
5484
|
+
// answer in well under a second.
|
|
5485
|
+
runtime.markRestartStage('waiting', { blocked: countBlockerTasks(runtime.gate().blockedBy) })
|
|
5152
5486
|
runtime.persist()
|
|
5153
5487
|
runtime.startPolling()
|
|
5154
5488
|
log.info('restart requested', { generation: state.generation, sessionId, resume, parkGoals })
|
|
@@ -6077,6 +6411,9 @@ function buildParkRoute(ctx, runtime, config) {
|
|
|
6077
6411
|
resume: true,
|
|
6078
6412
|
now: Date.now(),
|
|
6079
6413
|
})
|
|
6414
|
+
// The first stage, in the same breath as the request: the answer to
|
|
6415
|
+
// this POST is already "waiting for the sessions", never silence.
|
|
6416
|
+
runtime.markRestartStage('waiting', { blocked: blockingSessionIds(runtime).length })
|
|
6080
6417
|
runtime.persist()
|
|
6081
6418
|
runtime.startPolling()
|
|
6082
6419
|
}
|
|
@@ -6172,6 +6509,28 @@ function buildCancelRoute(ctx, runtime, config) {
|
|
|
6172
6509
|
})
|
|
6173
6510
|
return
|
|
6174
6511
|
}
|
|
6512
|
+
// THE STAGE DECIDES WHETHER A CANCEL IS HONEST.
|
|
6513
|
+
//
|
|
6514
|
+
// While the snapshot is being written the copy is stopped between two
|
|
6515
|
+
// files and its half-written directory is removed — the host never
|
|
6516
|
+
// restarts. Once the host is on its way out (`restarting`) a cancel
|
|
6517
|
+
// would be a promise this plugin cannot keep, so it is REFUSED with the
|
|
6518
|
+
// reason, and the button is greyed out on the same answer.
|
|
6519
|
+
const stage = runtime.restartStageInfo()
|
|
6520
|
+
if (stage.cancellable !== true) {
|
|
6521
|
+
const reason = cancelRefusalReason(stage.phase) || 'the host is already restarting'
|
|
6522
|
+
runtime.log.warn(`the cancel route refused the request of session ${sessionId}: ${reason}`)
|
|
6523
|
+
answer(409, {
|
|
6524
|
+
ok: false,
|
|
6525
|
+
cancelled: false,
|
|
6526
|
+
reason: 'not-cancellable',
|
|
6527
|
+
stage: stage.phase,
|
|
6528
|
+
error: reason,
|
|
6529
|
+
remainingRequesters: 0,
|
|
6530
|
+
sessionId,
|
|
6531
|
+
})
|
|
6532
|
+
return
|
|
6533
|
+
}
|
|
6175
6534
|
const outcome = cancelOwnRestart(runtime.state, { sessionId, now: Date.now() })
|
|
6176
6535
|
if (!outcome.removed) {
|
|
6177
6536
|
// Nothing was changed: the caller is not a requester (or the
|
|
@@ -6194,7 +6553,14 @@ function buildCancelRoute(ctx, runtime, config) {
|
|
|
6194
6553
|
}
|
|
6195
6554
|
// Release ONLY this session's continuation request.
|
|
6196
6555
|
runtime.releaseRequesters([sessionId])
|
|
6197
|
-
if (outcome.cancelled)
|
|
6556
|
+
if (outcome.cancelled) {
|
|
6557
|
+
// The whole restart is off. If the snapshot is being written right
|
|
6558
|
+
// now, the copy stops at the next file and removes its half-written
|
|
6559
|
+
// directory (the host never restarts); otherwise the stage just goes.
|
|
6560
|
+
if (stage.phase === 'snapshotting') runtime.requestSnapshotCancel()
|
|
6561
|
+
runtime.clearRestartStage()
|
|
6562
|
+
runtime.stopPolling()
|
|
6563
|
+
}
|
|
6198
6564
|
runtime.persist()
|
|
6199
6565
|
runtime.log.info(
|
|
6200
6566
|
outcome.cancelled
|
|
@@ -7377,6 +7743,39 @@ function buildSleepRoute(ctx, runtime, config, sleep) {
|
|
|
7377
7743
|
}
|
|
7378
7744
|
|
|
7379
7745
|
/** Expose a tiny read-only health route for humans and verification scripts. */
|
|
7746
|
+
/**
|
|
7747
|
+
* The disarm record, cut down to what a reader outside the process needs.
|
|
7748
|
+
*
|
|
7749
|
+
* The health route answers with it, so a real-machine check can tell "the
|
|
7750
|
+
* watchdog was stood down" from "it is still armed" without reading a log.
|
|
7751
|
+
*
|
|
7752
|
+
* @param {object|null} record
|
|
7753
|
+
* @returns {object|null}
|
|
7754
|
+
*/
|
|
7755
|
+
function summarizeDisarm(record) {
|
|
7756
|
+
if (!record || typeof record !== 'object') return null
|
|
7757
|
+
const errors = Array.isArray(record.errors) ? record.errors : []
|
|
7758
|
+
return {
|
|
7759
|
+
ok: record.ok === true,
|
|
7760
|
+
at: typeof record.at === 'string' ? record.at : null,
|
|
7761
|
+
source: typeof record.source === 'string' ? record.source : null,
|
|
7762
|
+
attempts: Number.isFinite(Number(record.attempts)) ? Number(record.attempts) : 0,
|
|
7763
|
+
ready: record.ready === true,
|
|
7764
|
+
pending: record.pending === true,
|
|
7765
|
+
tier: typeof record.tier === 'string' ? record.tier : null,
|
|
7766
|
+
scheduleRemoved: record.scheduleRemoved === true ? true : (record.scheduleRemoved === false ? false : null),
|
|
7767
|
+
// Only the first failure: the route answer must stay small, and one named
|
|
7768
|
+
// error is what makes the failure diagnosable.
|
|
7769
|
+
error: errors.length > 0
|
|
7770
|
+
? {
|
|
7771
|
+
step: typeof errors[0].step === 'string' ? errors[0].step : null,
|
|
7772
|
+
name: typeof errors[0].name === 'string' ? errors[0].name : null,
|
|
7773
|
+
message: typeof errors[0].message === 'string' ? errors[0].message : null,
|
|
7774
|
+
}
|
|
7775
|
+
: null,
|
|
7776
|
+
}
|
|
7777
|
+
}
|
|
7778
|
+
|
|
7380
7779
|
function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
7381
7780
|
const routePath = typeof config.healthPath === 'string' && config.healthPath.startsWith('/')
|
|
7382
7781
|
? config.healthPath
|
|
@@ -7471,6 +7870,10 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
7471
7870
|
}
|
|
7472
7871
|
}
|
|
7473
7872
|
const verdict = runtime.gate()
|
|
7873
|
+
// The restart stage (waiting / snapshotting / restarting / back), the
|
|
7874
|
+
// real snapshot progress and the "is a cancel still honest" answer —
|
|
7875
|
+
// computed ONCE per answer so every field of the payload agrees.
|
|
7876
|
+
const stage = runtime.restartStageInfo()
|
|
7474
7877
|
// Best effort: label the blocking sessions with their titles when the
|
|
7475
7878
|
// session controller knows them. The title is optional, so an entry
|
|
7476
7879
|
// without one is sent exactly as the gate produced it. The goal
|
|
@@ -7506,6 +7909,14 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
7506
7909
|
// long the restart has been waiting, why, and who holds it — the
|
|
7507
7910
|
// request itself is never dropped.
|
|
7508
7911
|
longWait: runtime.longWaitInfo(),
|
|
7912
|
+
// THE STAGE OF THE RESTART, for the pill and the settings page:
|
|
7913
|
+
// which of the four stages is on screen, the real copy progress of
|
|
7914
|
+
// the snapshot, whether it is going slower than expected, and
|
|
7915
|
+
// whether a cancel would still be honest (`cancellable`).
|
|
7916
|
+
restartPhase: stage,
|
|
7917
|
+
// The copy progress on its own as well, so a client that only wants
|
|
7918
|
+
// the numbers does not have to walk the stage object.
|
|
7919
|
+
progress: stage.progress,
|
|
7509
7920
|
updatedAt: live.updatedAt,
|
|
7510
7921
|
// The park route, so the client never has to guess the prefix, and
|
|
7511
7922
|
// the goals this plugin already disarmed (they are re-armed after
|
|
@@ -7630,27 +8041,64 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
7630
8041
|
// THE READY MARKER: this host answers its own health route, so it is
|
|
7631
8042
|
// up. The watchdog (which waits for the relaunched host) sees the
|
|
7632
8043
|
// marker and stands down — no rollback of a host that is fine.
|
|
8044
|
+
//
|
|
8045
|
+
// THE STATE DIRECTORY COMES FROM `runtime`, NEVER FROM A CLOSURE:
|
|
8046
|
+
// 0.8.2 read it through a variable that only existed inside the runtime
|
|
8047
|
+
// factory, so every request threw `ReferenceError: stateDirOf is not
|
|
8048
|
+
// defined`, the `catch {}` swallowed it, and a healthy host left the
|
|
8049
|
+
// pending marker AND the scheduled task behind (the watchdog would have
|
|
8050
|
+
// rolled a healthy profile back). The failure is recorded now, and the
|
|
8051
|
+
// marker is written first because the marker alone is what stops a
|
|
8052
|
+
// rollback.
|
|
7633
8053
|
try {
|
|
7634
|
-
const
|
|
7635
|
-
if (
|
|
7636
|
-
|
|
7637
|
-
|
|
7638
|
-
|
|
7639
|
-
|
|
7640
|
-
|
|
7641
|
-
|
|
7642
|
-
|
|
7643
|
-
|
|
7644
|
-
|
|
8054
|
+
const stateDir = typeof runtime.stateDir === 'string' ? runtime.stateDir : ''
|
|
8055
|
+
if (stateDir !== '') {
|
|
8056
|
+
const marked = markReady({
|
|
8057
|
+
stateDir,
|
|
8058
|
+
bootId: runtime.bootId,
|
|
8059
|
+
port: runtime.webPort ?? null,
|
|
8060
|
+
})
|
|
8061
|
+
if (marked.ok !== true) {
|
|
8062
|
+
runtime.watchdogReadyError = {
|
|
8063
|
+
at: Date.now(),
|
|
8064
|
+
message: marked.detail,
|
|
8065
|
+
name: marked.error && marked.error.name ? String(marked.error.name) : null,
|
|
7645
8066
|
}
|
|
7646
8067
|
}
|
|
8068
|
+
// The marker alone stands the watchdog down; the pending marker and
|
|
8069
|
+
// the scheduling item are cleaned with retries in the background, so
|
|
8070
|
+
// `schtasks` never delays this answer.
|
|
8071
|
+
runtime.disarmWatchdog('health-route')
|
|
8072
|
+
}
|
|
8073
|
+
} catch (error) {
|
|
8074
|
+
// NEVER silent: a swallowed error here leaves an armed watchdog behind.
|
|
8075
|
+
// It is recorded on `runtime` (this scope has no logger) so the settings
|
|
8076
|
+
// page and the diagnostics can show it instead of pretending all is well.
|
|
8077
|
+
runtime.watchdogReadyError = {
|
|
8078
|
+
at: Date.now(),
|
|
8079
|
+
name: error && error.name ? String(error.name) : 'Error',
|
|
8080
|
+
message: String(error && error.message ? error.message : error),
|
|
8081
|
+
}
|
|
8082
|
+
// One retry with the bare minimum: the marker alone is what the
|
|
8083
|
+
// watchdog actually waits for.
|
|
8084
|
+
try {
|
|
8085
|
+
markReady({
|
|
8086
|
+
stateDir: typeof runtime.stateDir === 'string' ? runtime.stateDir : '',
|
|
8087
|
+
bootId: runtime.bootId,
|
|
8088
|
+
port: runtime.webPort ?? null,
|
|
8089
|
+
})
|
|
8090
|
+
runtime.watchdogReadyError.retry = 'ok'
|
|
8091
|
+
} catch (again) {
|
|
8092
|
+
runtime.watchdogReadyError.retry = String(again && again.message ? again.message : again)
|
|
7647
8093
|
}
|
|
7648
|
-
} catch {
|
|
7649
|
-
// The marker is a convenience: the watchdog also asks the health route.
|
|
7650
8094
|
}
|
|
7651
8095
|
const body = JSON.stringify({
|
|
7652
8096
|
ok: true,
|
|
7653
8097
|
plugin: name,
|
|
8098
|
+
// The version of the code that is LOADED (not the fallback constant):
|
|
8099
|
+
// a real-machine check can tell "the new build is running" from "the
|
|
8100
|
+
// old one is still in memory".
|
|
8101
|
+
version: runtime.pluginVersion ?? PLUGIN_VERSION_FALLBACK,
|
|
7654
8102
|
bootId: runtime.bootId,
|
|
7655
8103
|
pid: process.pid,
|
|
7656
8104
|
phase: state.phase,
|
|
@@ -7662,6 +8110,15 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
7662
8110
|
manualRestartBlocks: runtime.manualRestartBlocks ?? 0,
|
|
7663
8111
|
indexTaps: runtime.indexTaps ?? 0,
|
|
7664
8112
|
configSchema: Config !== undefined,
|
|
8113
|
+
// DID THE DISARM WORK? A failed disarm means an armed watchdog is still
|
|
8114
|
+
// out there; it must be visible from outside the process, not only in
|
|
8115
|
+
// a log line nobody reads.
|
|
8116
|
+
watchdog: summarizeDisarm(runtime.watchdogDisarm),
|
|
8117
|
+
// THE RESTART STAGE, read from the health route as well: a script (or a
|
|
8118
|
+
// user watching the console) can tell "waiting" from "backing up 120 MB
|
|
8119
|
+
// of 156 MB" without opening the interface.
|
|
8120
|
+
restartPhase: runtime.restartStageBrief(),
|
|
8121
|
+
progress: runtime.restartStageBrief().progress,
|
|
7665
8122
|
schemastery: Config === undefined
|
|
7666
8123
|
? (schemastery === undefined ? 'unresolved' : 'construction-failed')
|
|
7667
8124
|
: (schemasterySource === 'bare-specifier' ? 'bare' : 'asar-fallback'),
|
|
@@ -7938,6 +8395,12 @@ export function apply(ctx, rawConfig, options = {}) {
|
|
|
7938
8395
|
// Record this boot in the checkpoint even when there is nothing to recover.
|
|
7939
8396
|
persistSafely(runtime)
|
|
7940
8397
|
|
|
8398
|
+
// THE LAST STAGE OF THE RESTART: this host IS the new one, so the interface
|
|
8399
|
+
// says "back — carrying on". Written here (not after the recovery) so a boot
|
|
8400
|
+
// that finds nothing to resume still shows the end of the story instead of a
|
|
8401
|
+
// spinner left over from the old process.
|
|
8402
|
+
if (runtime.isNewBoot) runtime.markRestartStage('back')
|
|
8403
|
+
|
|
7941
8404
|
trackAgents(ctx, runtime, config)
|
|
7942
8405
|
|
|
7943
8406
|
// The boot recovery pass is the plugin's primary boot duty, so it is scheduled
|
|
@@ -7955,8 +8418,7 @@ export function apply(ctx, rawConfig, options = {}) {
|
|
|
7955
8418
|
})
|
|
7956
8419
|
sleep.restore()
|
|
7957
8420
|
}
|
|
7958
|
-
if (runtime.isNewBoot && config.resumeOnBoot) {
|
|
7959
|
-
runtime.recoverTimer = ctx.timeout(() => {
|
|
8421
|
+
if (runtime.isNewBoot && config.resumeOnBoot) { runtime.recoverTimer = ctx.timeout(() => {
|
|
7960
8422
|
recoverAfterRestart(ctx, runtime, config)
|
|
7961
8423
|
.then(() => {
|
|
7962
8424
|
runtime.recovery = { ran: true, ok: true, detail: 'the boot recovery finished' }
|
|
@@ -7995,6 +8457,19 @@ export function apply(ctx, rawConfig, options = {}) {
|
|
|
7995
8457
|
registerSystemPrompt(ctx, runtime)
|
|
7996
8458
|
registerHealthRoute(ctx, runtime, config, sleep)
|
|
7997
8459
|
|
|
8460
|
+
// DISARM THE WATCHDOG AT BOOT — WITHOUT WAITING FOR ANYBODY TO ASK.
|
|
8461
|
+
//
|
|
8462
|
+
// This is the fix of the 0.8.2 incident: the ready marker and the cleanup of
|
|
8463
|
+
// the pending marker / scheduled task hung on the health route being hit.
|
|
8464
|
+
// The plugin is fully loaded at this point (every route is registered, the
|
|
8465
|
+
// boot recovery is scheduled), so this instance is "up" in the only sense
|
|
8466
|
+
// the watchdog cares about: it must not be rolled back.
|
|
8467
|
+
//
|
|
8468
|
+
// The call never rejects (see runtime.disarmWatchdog) and never blocks the
|
|
8469
|
+
// load: the marker is written synchronously on its first tick, the
|
|
8470
|
+
// `schtasks` cleanup runs with retries behind it.
|
|
8471
|
+
runtime.disarmWatchdog('boot')
|
|
8472
|
+
|
|
7998
8473
|
ctx.effect(() => () => {
|
|
7999
8474
|
runtime.markDisposed()
|
|
8000
8475
|
runtime.stopPolling()
|