dsh-autorestart 0.7.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/README.md +137 -6
- package/client.js +387 -0
- package/cordis.patch.yml +7 -0
- package/docs/README.zh.md +83 -5
- package/docs/platform-matrix.md +92 -0
- package/docs/rescue-drill.md +606 -0
- package/index.js +598 -3
- package/lib/diagnostics.js +8 -0
- package/lib/rescue-paths.js +253 -0
- package/lib/rescue.js +596 -0
- package/lib/scheduler-capability.js +300 -0
- package/lib/watchdog-arbiter.js +117 -0
- package/lib/watchdog-schedule.js +310 -0
- package/lib/watchdog-tier.js +182 -0
- package/lib/watchdog.js +479 -0
- package/locale/en.json +28 -1
- package/locale/zh.json +28 -1
- package/package.json +6 -1
- package/scripts/README.md +65 -0
- package/scripts/drill-rescue.ps1 +508 -0
- package/scripts/drill-rescue.sh +427 -0
- package/scripts/drill-watchdog.mjs +341 -0
- package/scripts/emergency-disable.sh +314 -0
- package/scripts/emergency-toggle.ps1 +136 -65
- package/scripts/rescue-strings.zh.json +168 -0
- package/scripts/restore-profile.cmd +44 -0
- package/scripts/restore-profile.ps1 +608 -0
- package/scripts/restore-profile.sh +593 -0
- package/scripts/run-tests.ps1 +1 -1
- package/scripts/watchdog.mjs +203 -0
package/index.js
CHANGED
|
@@ -19,9 +19,11 @@
|
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
21
|
import crypto from 'node:crypto'
|
|
22
|
+
import { spawn } from 'node:child_process'
|
|
22
23
|
import fs from 'node:fs'
|
|
23
24
|
import os from 'node:os'
|
|
24
25
|
import path from 'node:path'
|
|
26
|
+
import { fileURLToPath } from 'node:url'
|
|
25
27
|
|
|
26
28
|
import {
|
|
27
29
|
armSleep,
|
|
@@ -93,6 +95,48 @@ import {
|
|
|
93
95
|
} from './lib/supervisor.js'
|
|
94
96
|
import { composeFragment, detectInitSystems } from './lib/supervisor-install.js'
|
|
95
97
|
import { classifyStartup } from './lib/startup-mode.js'
|
|
98
|
+
import {
|
|
99
|
+
createDesktopShortcut,
|
|
100
|
+
desktopDir,
|
|
101
|
+
rescueInfo,
|
|
102
|
+
rescueScriptPaths,
|
|
103
|
+
} from './lib/rescue-paths.js'
|
|
104
|
+
import {
|
|
105
|
+
emptyCapabilities,
|
|
106
|
+
isCapabilities,
|
|
107
|
+
markSchedulerWarned,
|
|
108
|
+
probeSchedulerCapability,
|
|
109
|
+
SCHEDULER_UNKNOWN,
|
|
110
|
+
shouldWarnScheduler,
|
|
111
|
+
} from './lib/scheduler-capability.js'
|
|
112
|
+
import {
|
|
113
|
+
detectWatchdogCapabilities,
|
|
114
|
+
tierProvidesRollback,
|
|
115
|
+
tierWithProof,
|
|
116
|
+
TIERS,
|
|
117
|
+
} from './lib/watchdog-tier.js'
|
|
118
|
+
import {
|
|
119
|
+
buildPending,
|
|
120
|
+
clearReady,
|
|
121
|
+
readReady,
|
|
122
|
+
readReport,
|
|
123
|
+
writePending,
|
|
124
|
+
writeReady,
|
|
125
|
+
writeReport,
|
|
126
|
+
} from './lib/watchdog.js'
|
|
127
|
+
import {
|
|
128
|
+
ARBITERS,
|
|
129
|
+
planWatchdogArbiter,
|
|
130
|
+
} from './lib/watchdog-arbiter.js'
|
|
131
|
+
import {
|
|
132
|
+
cleanupSystemdUnit,
|
|
133
|
+
detachedPidPath,
|
|
134
|
+
localIso,
|
|
135
|
+
registerDetachedProcess,
|
|
136
|
+
registerScheduledTask,
|
|
137
|
+
registerSystemdRun,
|
|
138
|
+
removeScheduledTask,
|
|
139
|
+
} from './lib/watchdog-schedule.js'
|
|
96
140
|
import { detectHostKind } from './lib/host-kind.js'
|
|
97
141
|
import { planAutostartAdvice } from './lib/autostart-advice.js'
|
|
98
142
|
import { buildDiagnostics, ISSUE_URL_BASE, SUPPORT_EMAIL } from './lib/diagnostics.js'
|
|
@@ -146,6 +190,16 @@ import {
|
|
|
146
190
|
snapCountdownChoice,
|
|
147
191
|
} from './lib/sleep.js'
|
|
148
192
|
|
|
193
|
+
/**
|
|
194
|
+
* THE PLUGIN DIRECTORY — where this very file lives.
|
|
195
|
+
*
|
|
196
|
+
* The rescue scripts (restore-profile.cmd, emergency-disable.cmd) sit next to
|
|
197
|
+
* it. The settings page shows the user that REAL path, so the path is taken
|
|
198
|
+
* from here and never guessed: nothing assumes `node_modules/dsh-autorestart`
|
|
199
|
+
* and nothing assumes a profile layout.
|
|
200
|
+
*/
|
|
201
|
+
const PLUGIN_DIR = path.dirname(fileURLToPath(import.meta.url))
|
|
202
|
+
|
|
149
203
|
/**
|
|
150
204
|
* The RESTART reserve: how long the host waits after the LAST conversation
|
|
151
205
|
* finished before it is really restarted.
|
|
@@ -320,7 +374,7 @@ function snapGraceChoice(value, fallback = DEFAULT_RESTART_GRACE_MS) {
|
|
|
320
374
|
* `pluginVersion()`), but a bundled installation may not ship that file — so
|
|
321
375
|
* the constant below is the last resort. A test keeps the two in sync.
|
|
322
376
|
*/
|
|
323
|
-
const PLUGIN_VERSION_FALLBACK = '0.
|
|
377
|
+
const PLUGIN_VERSION_FALLBACK = '0.8.1'
|
|
324
378
|
|
|
325
379
|
/**
|
|
326
380
|
* The version of the graceful-shutdown FEATURE, not the version of the package.
|
|
@@ -614,6 +668,12 @@ function buildConfigSchema() {
|
|
|
614
668
|
.description('How many snapshots are kept; older ones are deleted automatically.'),
|
|
615
669
|
rollbackTimeoutMs: schemastery.number().min(1000).default(90000)
|
|
616
670
|
.description('Milliseconds the detached watcher waits for the relaunched host before it restores the snapshot.'),
|
|
671
|
+
// The watchdog: the process that waits for the relaunched host and puts
|
|
672
|
+
// the snapshot back when it never answers. It has to be triggered
|
|
673
|
+
// LATER, and the machine decides how (a one-off scheduled task, a
|
|
674
|
+
// systemd timer, or a detached process of our own).
|
|
675
|
+
watchdogEnabled: schemastery.boolean().default(true)
|
|
676
|
+
.description('Whether the watchdog may be armed before a restart: it waits for the new host and puts the snapshot back when it never answers. With it off there is no automatic rollback at all.'),
|
|
617
677
|
// The self-starting supervisor: a separate process that brings the host
|
|
618
678
|
// back when it dies. See the "开机自启" section of the settings.
|
|
619
679
|
supervisorMode: schemastery.union(['auto', 'installed', 'external', 'off']).default('auto')
|
|
@@ -695,6 +755,11 @@ const DEFAULTS = {
|
|
|
695
755
|
backupOnRestart: true,
|
|
696
756
|
backupIncludePlugins: true,
|
|
697
757
|
maxBackups: 2,
|
|
758
|
+
// The WATCHDOG: the process that waits for the relaunched host and puts the
|
|
759
|
+
// snapshot back when it never answers. It needs a way to be triggered
|
|
760
|
+
// later (a scheduled task, a systemd timer, a detached process) — the
|
|
761
|
+
// machine decides which one, and the settings page says which.
|
|
762
|
+
watchdogEnabled: true,
|
|
698
763
|
rollbackTimeoutMs: 90000,
|
|
699
764
|
// How long the restart command may run before the plugin stops waiting for
|
|
700
765
|
// it, and how long ONE restore step may take before the watcher gives up.
|
|
@@ -1542,6 +1607,13 @@ export function createRuntime(ctx, config, options = {}) {
|
|
|
1542
1607
|
const probeInstanceFn = typeof options.probeInstance === 'function'
|
|
1543
1608
|
? options.probeInstance
|
|
1544
1609
|
: waitForHealthyInstance
|
|
1610
|
+
// The NEW watchdog (v2). It is injected so a test can decide who is the
|
|
1611
|
+
// arbiter of the restart WITHOUT registering anything on the machine.
|
|
1612
|
+
// (The fallback is a wrapper: `armWatchdog` is declared further down, so it
|
|
1613
|
+
// must not be READ here.)
|
|
1614
|
+
const armWatchdogFn = typeof options.armWatchdog === 'function'
|
|
1615
|
+
? options.armWatchdog
|
|
1616
|
+
: (...args) => armWatchdog(...args)
|
|
1545
1617
|
const stateFile = resolveStateFile(config)
|
|
1546
1618
|
|
|
1547
1619
|
const loaded = readState(stateFile, Date.now())
|
|
@@ -2658,6 +2730,225 @@ const titleOfSummary = (item) => {
|
|
|
2658
2730
|
}
|
|
2659
2731
|
}
|
|
2660
2732
|
|
|
2733
|
+
/**
|
|
2734
|
+
* WHERE THE RESCUE SCRIPTS ARE.
|
|
2735
|
+
*
|
|
2736
|
+
* The rescue kit is for everybody who installed the plugin: when DSH does
|
|
2737
|
+
* not come up, the user cannot read a README — they need the path, and they
|
|
2738
|
+
* need to be able to copy it. The browser cannot read the disk, so the Host
|
|
2739
|
+
* resolves it: the plugin directory is where THIS file lives, and the
|
|
2740
|
+
* scripts sit next to it.
|
|
2741
|
+
*
|
|
2742
|
+
* @returns {object}
|
|
2743
|
+
*/
|
|
2744
|
+
const rescueKitInfo = () => rescueInfo({
|
|
2745
|
+
pluginDir: PLUGIN_DIR,
|
|
2746
|
+
env: process.env,
|
|
2747
|
+
platform: env.platform,
|
|
2748
|
+
exists: (target) => fs.existsSync(target),
|
|
2749
|
+
})
|
|
2750
|
+
|
|
2751
|
+
/**
|
|
2752
|
+
* THE CAPABILITIES of this machine: can it create a one-off scheduled
|
|
2753
|
+
* task?
|
|
2754
|
+
*
|
|
2755
|
+
* The automatic rollback hangs on that answer: the host is restarted and a
|
|
2756
|
+
* task that fires a minute later brings it back if it did not come up. On a
|
|
2757
|
+
* machine where such a task cannot be created the safety net is silently
|
|
2758
|
+
* missing — so the user is told what they lose.
|
|
2759
|
+
*
|
|
2760
|
+
* The answer is checked ONCE — the first time the plugin runs here — and
|
|
2761
|
+
* then kept in a file of its own next to the state. It is NOT checked when
|
|
2762
|
+
* the settings page opens: that would register a task every time somebody
|
|
2763
|
+
* looks at the page.
|
|
2764
|
+
*
|
|
2765
|
+
* A failure of the probe is NEVER a failure of the plugin: the plugin
|
|
2766
|
+
* works, only the safety net may be missing.
|
|
2767
|
+
*
|
|
2768
|
+
* @returns {object} the capabilities record
|
|
2769
|
+
*/
|
|
2770
|
+
const capabilitiesInfo = () => {
|
|
2771
|
+
const file = path.join(stateDirOf(), 'capabilities.json')
|
|
2772
|
+
const stored = readCapabilitiesRecord(file)
|
|
2773
|
+
if (stored) return stored
|
|
2774
|
+
let checked
|
|
2775
|
+
try {
|
|
2776
|
+
checked = probeSchedulerCapability({ platform: env.platform })
|
|
2777
|
+
} catch (error) {
|
|
2778
|
+
checked = { scheduler: SCHEDULER_UNKNOWN, checkedAt: Date.now(), evidence: { error: String(error) } }
|
|
2779
|
+
}
|
|
2780
|
+
// The tier is decided by ONE function (lib/watchdog-tier.js), so the
|
|
2781
|
+
// warning in the settings page and the watchdog that is really registered
|
|
2782
|
+
// cannot drift apart.
|
|
2783
|
+
let picked = { tier: TIERS.NONE, reason: 'this machine was not checked yet' }
|
|
2784
|
+
try {
|
|
2785
|
+
const caps = detectWatchdogCapabilities({ platform: env.platform })
|
|
2786
|
+
picked = tierWithProof({
|
|
2787
|
+
platform: env.platform,
|
|
2788
|
+
probeOk: checked.scheduler === 'ok',
|
|
2789
|
+
hasSystemdRun: caps.hasSystemdRun,
|
|
2790
|
+
hasAt: caps.hasAt,
|
|
2791
|
+
canDetach: caps.canDetach,
|
|
2792
|
+
})
|
|
2793
|
+
} catch (error) {
|
|
2794
|
+
picked = { tier: TIERS.NONE, reason: 'the machine could not be checked: ' + String(error) }
|
|
2795
|
+
}
|
|
2796
|
+
const record = { ...emptyCapabilities(), ...checked, tier: picked.tier, tierReason: picked.reason }
|
|
2797
|
+
// The warning is only for the TRANSITION into blocked: a machine that was
|
|
2798
|
+
// blocked and stays blocked does not nag.
|
|
2799
|
+
if (shouldWarnScheduler(emptyCapabilities(), record) && !tierProvidesRollback(picked.tier)) {
|
|
2800
|
+
runtime.log.warn('this machine cannot run the watchdog: the automatic rollback is not available')
|
|
2801
|
+
}
|
|
2802
|
+
writeCapabilitiesRecord(file, record)
|
|
2803
|
+
return record
|
|
2804
|
+
}
|
|
2805
|
+
|
|
2806
|
+
/**
|
|
2807
|
+
* THE WATCHDOG: which tier this machine uses, and what the last run did.
|
|
2808
|
+
*
|
|
2809
|
+
* @returns {{ enabled: boolean, tier: string, tierReason: string, available: boolean, report: object|null }}
|
|
2810
|
+
*/
|
|
2811
|
+
const watchdogInfo = () => {
|
|
2812
|
+
const capabilities = capabilitiesInfo()
|
|
2813
|
+
const tier = typeof capabilities.tier === 'string' && capabilities.tier !== ''
|
|
2814
|
+
? capabilities.tier
|
|
2815
|
+
: TIERS.NONE
|
|
2816
|
+
const report = readReport(stateDirOf())
|
|
2817
|
+
return {
|
|
2818
|
+
enabled: config.watchdogEnabled !== false,
|
|
2819
|
+
tier,
|
|
2820
|
+
tierReason: typeof capabilities.tierReason === 'string' ? capabilities.tierReason : '',
|
|
2821
|
+
available: tierProvidesRollback(tier),
|
|
2822
|
+
report,
|
|
2823
|
+
}
|
|
2824
|
+
}
|
|
2825
|
+
|
|
2826
|
+
/**
|
|
2827
|
+
* The capabilities with the warning marked as seen (the user read it).
|
|
2828
|
+
*
|
|
2829
|
+
* @returns {object}
|
|
2830
|
+
*/
|
|
2831
|
+
const acknowledgeSchedulerWarning = () => {
|
|
2832
|
+
const file = path.join(stateDirOf(), 'capabilities.json')
|
|
2833
|
+
const stored = readCapabilitiesRecord(file) || { ...emptyCapabilities() }
|
|
2834
|
+
const marked = markSchedulerWarned(stored)
|
|
2835
|
+
writeCapabilitiesRecord(file, marked)
|
|
2836
|
+
return marked
|
|
2837
|
+
}
|
|
2838
|
+
|
|
2839
|
+
/**
|
|
2840
|
+
* ARM THE WATCHDOG: the one-off thing that waits for the relaunched host.
|
|
2841
|
+
*
|
|
2842
|
+
* The tier is decided by ONE function (lib/watchdog-tier.js) — the same one
|
|
2843
|
+
* that the settings page and the diagnostics ask — so the warning and the
|
|
2844
|
+
* behaviour cannot drift apart. When no tier is available NOTHING is
|
|
2845
|
+
* registered: there is no watchdog, and the report says so.
|
|
2846
|
+
*
|
|
2847
|
+
* @param {{
|
|
2848
|
+
* snapshotDir?: string,
|
|
2849
|
+
* profileDir?: string|null,
|
|
2850
|
+
* relaunchCommand?: object|null,
|
|
2851
|
+
* healthUrl?: string,
|
|
2852
|
+
* portFile?: string,
|
|
2853
|
+
* expectedBootId?: string|null,
|
|
2854
|
+
* }} options
|
|
2855
|
+
* @returns {{ ok: boolean, tier: string, reason: string, error: string|null }}
|
|
2856
|
+
*/
|
|
2857
|
+
const armWatchdog = ({
|
|
2858
|
+
snapshotDir = '',
|
|
2859
|
+
profileDir = null,
|
|
2860
|
+
relaunchCommand = null,
|
|
2861
|
+
healthUrl = '',
|
|
2862
|
+
portFile = '',
|
|
2863
|
+
expectedBootId = null,
|
|
2864
|
+
} = {}) => {
|
|
2865
|
+
const stateDir = stateDirOf()
|
|
2866
|
+
if (config.watchdogEnabled === false) {
|
|
2867
|
+
return { ok: false, tier: TIERS.NONE, reason: 'the watchdog is switched off', error: null }
|
|
2868
|
+
}
|
|
2869
|
+
if (profileDir === null || snapshotDir === '') {
|
|
2870
|
+
return { ok: false, tier: TIERS.NONE, reason: 'there is no snapshot to watch', error: null }
|
|
2871
|
+
}
|
|
2872
|
+
const info = watchdogInfo()
|
|
2873
|
+
if (info.tier === TIERS.NONE) {
|
|
2874
|
+
return { ok: false, tier: TIERS.NONE, reason: info.tierReason || 'this machine cannot run the watchdog', error: null }
|
|
2875
|
+
}
|
|
2876
|
+
const timeoutMs = Number(config.rollbackTimeoutMs) > 0 ? Number(config.rollbackTimeoutMs) : 180000
|
|
2877
|
+
// ONE name for the scheduling item: a new registration replaces the old
|
|
2878
|
+
// one, and there is never more than one of them on the machine.
|
|
2879
|
+
const taskName = 'dsh-autorestart-watchdog'
|
|
2880
|
+
const unitId = 'dsh-autorestart-watchdog'
|
|
2881
|
+
const pidPath = detachedPidPath(stateDir)
|
|
2882
|
+
const watchdogCommand = '"' + process.execPath + '" "' + path.join(PLUGIN_DIR, 'scripts', 'watchdog.mjs') + '" --state-dir "' + stateDir + '"'
|
|
2883
|
+
const pending = buildPending({
|
|
2884
|
+
tier: info.tier,
|
|
2885
|
+
tierReason: info.tierReason,
|
|
2886
|
+
taskName,
|
|
2887
|
+
unitId,
|
|
2888
|
+
pidPath,
|
|
2889
|
+
snapshotDir,
|
|
2890
|
+
profileDir,
|
|
2891
|
+
relaunchCommand,
|
|
2892
|
+
healthUrl,
|
|
2893
|
+
healthPath: resolveHealthPath(config),
|
|
2894
|
+
portFile,
|
|
2895
|
+
expectedBootId,
|
|
2896
|
+
timeoutMs,
|
|
2897
|
+
recheckMs: 20000,
|
|
2898
|
+
readyTimeoutMs: 90000,
|
|
2899
|
+
hostKind: hostKindOf().kind,
|
|
2900
|
+
now: Date.now(),
|
|
2901
|
+
})
|
|
2902
|
+
writePending(stateDir, pending)
|
|
2903
|
+
|
|
2904
|
+
let registered = { ok: false, error: 'the watchdog was not registered' }
|
|
2905
|
+
try {
|
|
2906
|
+
if (info.tier === TIERS.SCHEDULED_TASK) {
|
|
2907
|
+
const start = new Date(Date.now() + Math.ceil(timeoutMs / 1000) * 1000);
|
|
2908
|
+
const end = new Date(start.getTime() + 30 * 60 * 1000);
|
|
2909
|
+
registered = registerScheduledTask({
|
|
2910
|
+
taskName,
|
|
2911
|
+
startIso: localIso(start),
|
|
2912
|
+
endIso: localIso(end),
|
|
2913
|
+
command: watchdogCommand,
|
|
2914
|
+
xmlPath: path.join(stateDir, 'watchdog-task.xml'),
|
|
2915
|
+
});
|
|
2916
|
+
} else if (info.tier === TIERS.SYSTEMD_RUN) {
|
|
2917
|
+
// The unit of an earlier run must be gone before a new one can take
|
|
2918
|
+
// the name.
|
|
2919
|
+
cleanupSystemdUnit({ unitId });
|
|
2920
|
+
registered = registerSystemdRun({
|
|
2921
|
+
unitId,
|
|
2922
|
+
delaySeconds: Math.ceil(timeoutMs / 1000),
|
|
2923
|
+
command: [process.execPath, path.join(PLUGIN_DIR, 'scripts', 'watchdog.mjs'), '--state-dir', stateDir],
|
|
2924
|
+
});
|
|
2925
|
+
} else if (info.tier === TIERS.DETACHED_PROCESS) {
|
|
2926
|
+
registered = registerDetachedProcess({
|
|
2927
|
+
delaySeconds: Math.ceil(timeoutMs / 1000),
|
|
2928
|
+
command: process.execPath + ' "' + path.join(PLUGIN_DIR, 'scripts', 'watchdog.mjs') + '" --state-dir "' + stateDir + '"',
|
|
2929
|
+
pidPath,
|
|
2930
|
+
});
|
|
2931
|
+
}
|
|
2932
|
+
} catch (error) {
|
|
2933
|
+
registered = { ok: false, error: error && error.message ? error.message : String(error) };
|
|
2934
|
+
}
|
|
2935
|
+
if (!registered || registered.ok !== true) {
|
|
2936
|
+
// Nothing half-armed: the marker goes, and so does the scheduling item.
|
|
2937
|
+
clearPending(stateDir);
|
|
2938
|
+
try {
|
|
2939
|
+
if (info.tier === TIERS.SCHEDULED_TASK) removeScheduledTask({ taskName });
|
|
2940
|
+
else if (info.tier === TIERS.SYSTEMD_RUN) cleanupSystemdUnit({ unitId });
|
|
2941
|
+
} catch {
|
|
2942
|
+
// The next run cleans up.
|
|
2943
|
+
}
|
|
2944
|
+
const reason = (registered && registered.error) || 'the watchdog could not be registered';
|
|
2945
|
+
log.warn('the watchdog was not armed: ' + reason);
|
|
2946
|
+
return { ok: false, tier: info.tier, reason: info.tierReason, error: reason };
|
|
2947
|
+
}
|
|
2948
|
+
log.warn('the watchdog is armed (' + info.tier + '): ' + info.tierReason);
|
|
2949
|
+
return { ok: true, tier: info.tier, reason: info.tierReason, error: null };
|
|
2950
|
+
}
|
|
2951
|
+
|
|
2661
2952
|
/**
|
|
2662
2953
|
* THE DIAGNOSTICS of "report a problem".
|
|
2663
2954
|
*
|
|
@@ -2678,6 +2969,7 @@ const titleOfSummary = (item) => {
|
|
|
2678
2969
|
restartMode: config.restartMode,
|
|
2679
2970
|
lastRestart: lastRestartVerdict(),
|
|
2680
2971
|
stateDir: stateDirOf(),
|
|
2972
|
+
watchdog: watchdogInfo(),
|
|
2681
2973
|
})
|
|
2682
2974
|
|
|
2683
2975
|
/**
|
|
@@ -3165,6 +3457,37 @@ const titleOfSummary = (item) => {
|
|
|
3165
3457
|
return { ok: true, method: result.method, error: null, scriptPath }
|
|
3166
3458
|
}
|
|
3167
3459
|
|
|
3460
|
+
/**
|
|
3461
|
+
* Say in the report WHO is watching this restart.
|
|
3462
|
+
*
|
|
3463
|
+
* The v2 watchdog writes its own report when it acts. When v1 is the
|
|
3464
|
+
* arbiter (v2 could not be armed), the host writes it right away — so
|
|
3465
|
+
* `last-watchdog-report.json` always names the arbiter, and a problem can
|
|
3466
|
+
* be read at a glance.
|
|
3467
|
+
*
|
|
3468
|
+
* @param {string} arbiter
|
|
3469
|
+
* @param {string} reason
|
|
3470
|
+
*/
|
|
3471
|
+
const writeArbiterReport = (arbiter, reason, tier = 'none') => {
|
|
3472
|
+
try {
|
|
3473
|
+
writeReport(stateDirOf(), {
|
|
3474
|
+
at: new Date().toISOString(),
|
|
3475
|
+
result: 'armed',
|
|
3476
|
+
tier,
|
|
3477
|
+
tierReason: reason || '',
|
|
3478
|
+
arbiter,
|
|
3479
|
+
waitedMs: 0,
|
|
3480
|
+
rechecked: false,
|
|
3481
|
+
restored: false,
|
|
3482
|
+
relaunched: false,
|
|
3483
|
+
ready: false,
|
|
3484
|
+
error: null,
|
|
3485
|
+
})
|
|
3486
|
+
} catch (error) {
|
|
3487
|
+
log.warn('could not write the arbiter report:', String(error))
|
|
3488
|
+
}
|
|
3489
|
+
}
|
|
3490
|
+
|
|
3168
3491
|
/** Execute the restart now; the detached helper survives our own death. */
|
|
3169
3492
|
const executeRestart = async () => {
|
|
3170
3493
|
stopPolling()
|
|
@@ -3228,6 +3551,30 @@ const titleOfSummary = (item) => {
|
|
|
3228
3551
|
const backup = snapshot && snapshot.backup ? snapshot.backup : null
|
|
3229
3552
|
const profileDir = snapshot && snapshot.profileDir ? snapshot.profileDir : null
|
|
3230
3553
|
|
|
3554
|
+
// ARM THE WATCHDOG: it waits for the relaunched host, and when that
|
|
3555
|
+
// host never answers, it puts the snapshot back and starts it again.
|
|
3556
|
+
// Which way it is triggered is decided by the machine (the tier).
|
|
3557
|
+
//
|
|
3558
|
+
// The result matters: when v2 is armed, it is the ONLY watchdog. The
|
|
3559
|
+
// old one (v1) is NOT started next to it — they take the same lock, and
|
|
3560
|
+
// the one that loses it gives up. If v2 lost, the profile would be
|
|
3561
|
+
// restored but nothing would be running any more.
|
|
3562
|
+
const watchdogArm = backup && backup.ok === true && profileDir !== null
|
|
3563
|
+
? armWatchdogFn({
|
|
3564
|
+
snapshotDir: backup.dir,
|
|
3565
|
+
profileDir,
|
|
3566
|
+
relaunchCommand: useCustomCommand && typeof config.restartCommand === 'string' && config.restartCommand.trim() !== ''
|
|
3567
|
+
? {
|
|
3568
|
+
file: config.restartCommand.trim().split(/\s+/)[0],
|
|
3569
|
+
args: config.restartCommand.trim().split(/\s+/).slice(1),
|
|
3570
|
+
}
|
|
3571
|
+
: (env.appExePath ? { file: env.appExePath, args: [] } : null),
|
|
3572
|
+
healthUrl: restartHealthUrl(),
|
|
3573
|
+
portFile: accessFilePath(stateDir),
|
|
3574
|
+
expectedBootId: bootId,
|
|
3575
|
+
})
|
|
3576
|
+
: { ok: false, tier: TIERS.NONE, reason: 'there is no snapshot to watch', error: null }
|
|
3577
|
+
|
|
3231
3578
|
// THE DURABLE RESTART RECORD. The instance that is being replaced
|
|
3232
3579
|
// writes the "pending" half BEFORE it runs the restart command: the
|
|
3233
3580
|
// snapshot it took (directory, local time, duration, which older
|
|
@@ -3305,7 +3652,25 @@ const titleOfSummary = (item) => {
|
|
|
3305
3652
|
// It is NOT armed when the supervisor of this plugin is in charge:
|
|
3306
3653
|
// that one restores the snapshot itself, and two of them would write
|
|
3307
3654
|
// the same files at the same time.
|
|
3308
|
-
|
|
3655
|
+
//
|
|
3656
|
+
// It is NOT armed when the watchdog v2 is armed either: v2 does the
|
|
3657
|
+
// same job AND starts the host again. Two watchdogs take the same
|
|
3658
|
+
// lock, and the one that loses it gives up — if v2 lost, the profile
|
|
3659
|
+
// would be restored but nothing would be running any more.
|
|
3660
|
+
const arbiter = planWatchdogArbiter({
|
|
3661
|
+
v2Armed: watchdogArm !== null && watchdogArm.ok === true,
|
|
3662
|
+
supervised: detected.supervised,
|
|
3663
|
+
})
|
|
3664
|
+
if (arbiter.startV1 === true) {
|
|
3665
|
+
armRollbackWatcher({ backup, profileDir })
|
|
3666
|
+
writeArbiterReport(ARBITERS.V1, arbiter.reason, 'none')
|
|
3667
|
+
} else if (arbiter.arbiter === ARBITERS.V2) {
|
|
3668
|
+
writeArbiterReport(ARBITERS.V2, arbiter.reason, watchdogArm.tier)
|
|
3669
|
+
log.warn('the watchdog v2 takes over: the old rollback watcher is NOT started', {
|
|
3670
|
+
tier: watchdogArm.tier,
|
|
3671
|
+
tierReason: watchdogArm.tierReason || '',
|
|
3672
|
+
})
|
|
3673
|
+
}
|
|
3309
3674
|
|
|
3310
3675
|
try {
|
|
3311
3676
|
// (2) THE COMMAND, with a real success judgement. The old code only
|
|
@@ -3398,9 +3763,26 @@ const titleOfSummary = (item) => {
|
|
|
3398
3763
|
// The watcher does NOT need the port of the old instance (it reads the
|
|
3399
3764
|
// port of the NEW one from the instance record), so it is armed as
|
|
3400
3765
|
// soon as there is a snapshot and a profile directory.
|
|
3401
|
-
|
|
3766
|
+
//
|
|
3767
|
+
// ONLY when the watchdog v2 could not be armed: v2 is the better
|
|
3768
|
+
// watcher (it starts the host again), and two watchers take the same
|
|
3769
|
+
// lock — the loser gives up.
|
|
3770
|
+
const arbiter = planWatchdogArbiter({
|
|
3771
|
+
v2Armed: watchdogArm !== null && watchdogArm.ok === true,
|
|
3772
|
+
supervised: false,
|
|
3773
|
+
})
|
|
3774
|
+
const watchScriptPath = arbiter.startV1 === true && backup && backup.ok === true && profileDir !== null
|
|
3402
3775
|
? path.join(stateDir, 'restart-' + state.generation + '-watch.ps1')
|
|
3403
3776
|
: null
|
|
3777
|
+
if (arbiter.startV1 === true) {
|
|
3778
|
+
writeArbiterReport(ARBITERS.V1, arbiter.reason, 'none')
|
|
3779
|
+
} else if (arbiter.arbiter === ARBITERS.V2) {
|
|
3780
|
+
writeArbiterReport(ARBITERS.V2, arbiter.reason, watchdogArm.tier)
|
|
3781
|
+
log.warn('the watchdog v2 takes over: the old rollback watcher is NOT started', {
|
|
3782
|
+
tier: watchdogArm.tier,
|
|
3783
|
+
tierReason: watchdogArm.tierReason || '',
|
|
3784
|
+
})
|
|
3785
|
+
}
|
|
3404
3786
|
const watchScript = watchScriptPath === null
|
|
3405
3787
|
? null
|
|
3406
3788
|
: buildWatchScript({
|
|
@@ -3747,6 +4129,14 @@ const titleOfSummary = (item) => {
|
|
|
3747
4129
|
* WHAT IS ON DISK RIGHT NOW: the backups of this host.
|
|
3748
4130
|
*/
|
|
3749
4131
|
backupsInfo,
|
|
4132
|
+
// Where the rescue scripts are, and whether the desktop shortcut exists.
|
|
4133
|
+
rescueKitInfo,
|
|
4134
|
+
// What this machine can do: can it create a one-off scheduled task? (The
|
|
4135
|
+
// automatic rollback hangs on that answer.)
|
|
4136
|
+
capabilitiesInfo,
|
|
4137
|
+
acknowledgeSchedulerWarning,
|
|
4138
|
+
// The watchdog: which tier this machine uses, and what the last run did.
|
|
4139
|
+
watchdogInfo,
|
|
3750
4140
|
/**
|
|
3751
4141
|
* THE DIAGNOSTICS of "report a problem" (the text the user copies).
|
|
3752
4142
|
*/
|
|
@@ -6224,6 +6614,176 @@ function buildChoiceRoute(ctx, runtime, config) {
|
|
|
6224
6614
|
* @param {object} config the normalized config
|
|
6225
6615
|
* @returns {object} the route
|
|
6226
6616
|
*/
|
|
6617
|
+
/**
|
|
6618
|
+
* Read the capabilities record of this machine.
|
|
6619
|
+
*
|
|
6620
|
+
* It lives in a file of its own next to the state: the checkpoint has a
|
|
6621
|
+
* versioned shape, and a capability that is checked once does not belong in
|
|
6622
|
+
* the record of the running sessions.
|
|
6623
|
+
*
|
|
6624
|
+
* @param {string} file
|
|
6625
|
+
* @returns {object|null}
|
|
6626
|
+
*/
|
|
6627
|
+
function readCapabilitiesRecord(file) {
|
|
6628
|
+
try {
|
|
6629
|
+
const raw = fs.readFileSync(file, 'utf8')
|
|
6630
|
+
const parsed = JSON.parse(raw)
|
|
6631
|
+
if (!isCapabilities(parsed)) return null
|
|
6632
|
+
return {
|
|
6633
|
+
scheduler: parsed.scheduler,
|
|
6634
|
+
checkedAt: typeof parsed.checkedAt === 'number' ? parsed.checkedAt : null,
|
|
6635
|
+
evidence: parsed.evidence && typeof parsed.evidence === 'object' ? parsed.evidence : null,
|
|
6636
|
+
warnedAt: typeof parsed.warnedAt === 'number' ? parsed.warnedAt : null,
|
|
6637
|
+
tier: typeof parsed.tier === 'string' ? parsed.tier : '',
|
|
6638
|
+
tierReason: typeof parsed.tierReason === 'string' ? parsed.tierReason : '',
|
|
6639
|
+
}
|
|
6640
|
+
} catch {
|
|
6641
|
+
// A record that cannot be read is a record that does not exist yet.
|
|
6642
|
+
return null
|
|
6643
|
+
}
|
|
6644
|
+
}
|
|
6645
|
+
|
|
6646
|
+
/**
|
|
6647
|
+
* Write the capabilities record of this machine.
|
|
6648
|
+
*
|
|
6649
|
+
* @param {string} file
|
|
6650
|
+
* @param {object} record
|
|
6651
|
+
*/
|
|
6652
|
+
function writeCapabilitiesRecord(file, record) {
|
|
6653
|
+
try {
|
|
6654
|
+
fs.mkdirSync(path.dirname(file), { recursive: true })
|
|
6655
|
+
fs.writeFileSync(file, JSON.stringify(record, null, 2), 'utf8')
|
|
6656
|
+
} catch {
|
|
6657
|
+
// A capability that cannot be written is a capability that is checked
|
|
6658
|
+
// again next time — not a reason to break the plugin.
|
|
6659
|
+
}
|
|
6660
|
+
}
|
|
6661
|
+
|
|
6662
|
+
/**
|
|
6663
|
+
* Resolve the rescue-route path: a sibling of the health route, under the same
|
|
6664
|
+
* prefix.
|
|
6665
|
+
*
|
|
6666
|
+
* @param {object} config
|
|
6667
|
+
* @returns {string}
|
|
6668
|
+
*/
|
|
6669
|
+
function resolveRescuePath(config) {
|
|
6670
|
+
const healthPath = resolveHealthPath(config)
|
|
6671
|
+
const index = healthPath.lastIndexOf('/')
|
|
6672
|
+
const prefix = index > 0 ? healthPath.slice(0, index) : '/__dsh_autorestart'
|
|
6673
|
+
return `${prefix}/rescue`
|
|
6674
|
+
}
|
|
6675
|
+
|
|
6676
|
+
/**
|
|
6677
|
+
* The rescue route: the two scripts, and the desktop shortcut.
|
|
6678
|
+
*
|
|
6679
|
+
* It does three things, all of them on behalf of the USER:
|
|
6680
|
+
*
|
|
6681
|
+
* - `status` : where the scripts are (the paths and the state of the
|
|
6682
|
+
* desktop shortcut) — nothing is written;
|
|
6683
|
+
* - `shortcut` : write the desktop shortcut. This happens ONLY because the
|
|
6684
|
+
* user pressed the button: loading the settings page never
|
|
6685
|
+
* writes anything to the desktop;
|
|
6686
|
+
* - `reveal` : open the folder of a script in the file manager.
|
|
6687
|
+
*
|
|
6688
|
+
* Like every other route of this plugin it only answers the loopback.
|
|
6689
|
+
*
|
|
6690
|
+
* @param {object} ctx
|
|
6691
|
+
* @param {object} runtime
|
|
6692
|
+
* @param {object} config
|
|
6693
|
+
* @returns {{ kind: string, path: string, handler: Function }}
|
|
6694
|
+
*/
|
|
6695
|
+
function buildRescueRoute(ctx, runtime, config) {
|
|
6696
|
+
const routePath = resolveRescuePath(config)
|
|
6697
|
+
return {
|
|
6698
|
+
kind: 'exact',
|
|
6699
|
+
path: routePath,
|
|
6700
|
+
handler: (req, res) => {
|
|
6701
|
+
const remote = req && req.socket ? req.socket.remoteAddress : undefined
|
|
6702
|
+
const loopback = remote === undefined
|
|
6703
|
+
|| remote === '127.0.0.1'
|
|
6704
|
+
|| remote === '::1'
|
|
6705
|
+
|| remote === '::ffff:127.0.0.1'
|
|
6706
|
+
if (!loopback && !admittedByConnection(ctx, req)) {
|
|
6707
|
+
res.statusCode = 403
|
|
6708
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6709
|
+
res.end('{"ok":false,"error":"loopback-only"}')
|
|
6710
|
+
return
|
|
6711
|
+
}
|
|
6712
|
+
if (!req || req.method !== 'POST') {
|
|
6713
|
+
res.statusCode = 405
|
|
6714
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6715
|
+
res.setHeader('allow', 'POST')
|
|
6716
|
+
res.end('{"ok":false,"error":"method-not-allowed"}')
|
|
6717
|
+
return
|
|
6718
|
+
}
|
|
6719
|
+
void readJsonBody(req).then((payload) => {
|
|
6720
|
+
const body = payload && typeof payload === 'object' ? payload : {}
|
|
6721
|
+
const action = typeof body.action === 'string' ? body.action.trim() : 'status'
|
|
6722
|
+
const info = runtime.rescueKitInfo()
|
|
6723
|
+
if (action === 'shortcut') {
|
|
6724
|
+
// The user pressed the button: NOW the shortcut may be written.
|
|
6725
|
+
const written = createDesktopShortcut({
|
|
6726
|
+
desktop: info.desktopDir,
|
|
6727
|
+
restoreProfilePath: info.restoreProfile,
|
|
6728
|
+
home: os.homedir(),
|
|
6729
|
+
writeFile: (target, content) => fs.writeFileSync(target, content, 'ascii'),
|
|
6730
|
+
exists: (target) => fs.existsSync(target),
|
|
6731
|
+
})
|
|
6732
|
+
res.statusCode = written.ok ? 200 : 400
|
|
6733
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6734
|
+
res.setHeader('cache-control', 'no-store')
|
|
6735
|
+
res.end(JSON.stringify({
|
|
6736
|
+
ok: written.ok,
|
|
6737
|
+
action,
|
|
6738
|
+
path: written.path,
|
|
6739
|
+
error: written.error ?? null,
|
|
6740
|
+
rescue: runtime.rescueKitInfo(),
|
|
6741
|
+
}))
|
|
6742
|
+
return
|
|
6743
|
+
}
|
|
6744
|
+
if (action === 'ack-scheduler') {
|
|
6745
|
+
// The user has read the warning about the missing automatic
|
|
6746
|
+
// rollback: it is not shown again.
|
|
6747
|
+
const marked = runtime.acknowledgeSchedulerWarning()
|
|
6748
|
+
res.statusCode = 200
|
|
6749
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6750
|
+
res.setHeader('cache-control', 'no-store')
|
|
6751
|
+
res.end(JSON.stringify({ ok: true, action, capabilities: marked }))
|
|
6752
|
+
return
|
|
6753
|
+
}
|
|
6754
|
+
if (action === 'reveal') {
|
|
6755
|
+
const wanted = typeof body.path === 'string' ? body.path.trim() : ''
|
|
6756
|
+
// Only a directory of the plugin may be opened — never anything the
|
|
6757
|
+
// caller names.
|
|
6758
|
+
const allowed = [info.scriptsDir].filter((entry) => entry !== '')
|
|
6759
|
+
const folder = allowed.includes(wanted) ? wanted : info.scriptsDir
|
|
6760
|
+
let opened = false
|
|
6761
|
+
try {
|
|
6762
|
+
const child = spawn(process.platform === 'win32' ? 'explorer.exe' : 'xdg-open', [folder], {
|
|
6763
|
+
detached: true,
|
|
6764
|
+
stdio: 'ignore',
|
|
6765
|
+
windowsHide: true,
|
|
6766
|
+
})
|
|
6767
|
+
child.unref()
|
|
6768
|
+
opened = true
|
|
6769
|
+
} catch (error) {
|
|
6770
|
+
opened = false
|
|
6771
|
+
}
|
|
6772
|
+
res.statusCode = opened ? 200 : 500
|
|
6773
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6774
|
+
res.setHeader('cache-control', 'no-store')
|
|
6775
|
+
res.end(JSON.stringify({ ok: opened, action, path: folder, error: opened ? null : 'the file manager could not be opened' }))
|
|
6776
|
+
return
|
|
6777
|
+
}
|
|
6778
|
+
res.statusCode = 200
|
|
6779
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
6780
|
+
res.setHeader('cache-control', 'no-store')
|
|
6781
|
+
res.end(JSON.stringify({ ok: true, action, rescue: info }))
|
|
6782
|
+
})
|
|
6783
|
+
},
|
|
6784
|
+
}
|
|
6785
|
+
}
|
|
6786
|
+
|
|
6227
6787
|
function buildSupervisorRoute(ctx, runtime, config) {
|
|
6228
6788
|
const routePath = resolveSupervisorPath(config)
|
|
6229
6789
|
return {
|
|
@@ -6968,6 +7528,18 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
6968
7528
|
// where it lives). The browser cannot read the disk, so the Host
|
|
6969
7529
|
// reads it — only the directory, never the contents.
|
|
6970
7530
|
backups: runtime.backupsInfo(),
|
|
7531
|
+
// THE RESCUE KIT: where the two scripts are, so a user whose DSH
|
|
7532
|
+
// does not come up can find them (and copy the path) even though
|
|
7533
|
+
// they cannot open a README.
|
|
7534
|
+
rescue: runtime.rescueKitInfo(),
|
|
7535
|
+
rescueUrl: resolveRescuePath(config),
|
|
7536
|
+
// What this machine can do (the automatic rollback needs a
|
|
7537
|
+
// one-off scheduled task; when that is refused the user is told
|
|
7538
|
+
// what they lose).
|
|
7539
|
+
capabilities: runtime.capabilitiesInfo(),
|
|
7540
|
+
// The watchdog: which tier this machine uses to trigger it, and
|
|
7541
|
+
// what the last run did.
|
|
7542
|
+
watchdog: runtime.watchdogInfo(),
|
|
6971
7543
|
// THE RESTART COMMAND that is in effect: what is written, and what
|
|
6972
7544
|
// an undo would restore. The settings page shows it in a folded
|
|
6973
7545
|
// "for a technical colleague" section.
|
|
@@ -7055,6 +7627,27 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
7055
7627
|
return
|
|
7056
7628
|
}
|
|
7057
7629
|
const probeIndex = typeof req.url === 'string' && req.url.includes('probeIndex=1')
|
|
7630
|
+
// THE READY MARKER: this host answers its own health route, so it is
|
|
7631
|
+
// up. The watchdog (which waits for the relaunched host) sees the
|
|
7632
|
+
// marker and stands down — no rollback of a host that is fine.
|
|
7633
|
+
try {
|
|
7634
|
+
const marker = readReady(stateDirOf())
|
|
7635
|
+
if (!marker || marker.bootId !== runtime.bootId) {
|
|
7636
|
+
writeReady(stateDirOf(), { bootId: runtime.bootId, port: runtime.port ?? null })
|
|
7637
|
+
// The host is up: the watchdog has nothing to do. Its scheduling
|
|
7638
|
+
// item and its pending marker go NOW — nothing may be left behind.
|
|
7639
|
+
if (readPending(stateDirOf()) !== null) {
|
|
7640
|
+
clearPending(stateDirOf())
|
|
7641
|
+
try {
|
|
7642
|
+
removeScheduledTask({ taskName: 'dsh-autorestart-watchdog' })
|
|
7643
|
+
} catch {
|
|
7644
|
+
// The watchdog removes it itself when it runs.
|
|
7645
|
+
}
|
|
7646
|
+
}
|
|
7647
|
+
}
|
|
7648
|
+
} catch {
|
|
7649
|
+
// The marker is a convenience: the watchdog also asks the health route.
|
|
7650
|
+
}
|
|
7058
7651
|
const body = JSON.stringify({
|
|
7059
7652
|
ok: true,
|
|
7060
7653
|
plugin: name,
|
|
@@ -7104,6 +7697,7 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
7104
7697
|
const cancelRoute = buildCancelRoute(ctx, runtime, config)
|
|
7105
7698
|
const announcementsRoute = buildAnnouncementsRoute(ctx, runtime, config)
|
|
7106
7699
|
const supervisorRoute = buildSupervisorRoute(ctx, runtime, config)
|
|
7700
|
+
const rescueRoute = buildRescueRoute(ctx, runtime, config)
|
|
7107
7701
|
const sleepRoute = sleep ? buildSleepRoute(ctx, runtime, config, sleep) : null
|
|
7108
7702
|
try {
|
|
7109
7703
|
const origin = typeof webServer.getOrigin === 'function' ? String(webServer.getOrigin()) : ''
|
|
@@ -7129,6 +7723,7 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
7129
7723
|
{ name: 'cancel', route: cancelRoute },
|
|
7130
7724
|
{ name: 'announcements', route: announcementsRoute },
|
|
7131
7725
|
{ name: 'supervisor', route: supervisorRoute },
|
|
7726
|
+
{ name: 'rescue', route: rescueRoute },
|
|
7132
7727
|
]
|
|
7133
7728
|
if (sleepRoute !== null) routes.push({ name: 'sleep', route: sleepRoute })
|
|
7134
7729
|
for (const entry of routes) {
|