dsh-autorestart 0.8.4 → 0.8.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +313 -0
- package/README.md +465 -40
- package/assets/icon-9c245e5b.ico +0 -0
- package/assets/icon-rescue-1902bb15.ico +0 -0
- package/assets/icon-rescue-1902bb15.png +0 -0
- package/client.js +3002 -170
- package/cordis.patch.yml +12 -0
- package/docs/README.zh.md +231 -23
- package/docs/platform-matrix.md +171 -55
- package/docs/real-machine-verification-0.8.6.md +202 -0
- package/index.js +13272 -8513
- package/lib/autostart-advice.js +31 -2
- package/lib/backup.js +2768 -21
- package/lib/checkpoint.js +214 -7
- package/lib/cookie-clear.js +260 -0
- package/lib/lifecycle.js +23 -40
- package/lib/plugin-compare.js +499 -0
- package/lib/plugin-failure.js +382 -0
- package/lib/plugin-recover.js +1557 -0
- package/lib/plugin-watch.js +213 -0
- package/lib/rescue-paths.js +982 -56
- package/lib/restart-phase.js +5 -0
- package/lib/restart-verdict.js +379 -0
- package/lib/restart.js +244 -7
- package/lib/scheduler-capability.js +19 -3
- package/lib/sleep.js +190 -29
- package/lib/watchdog-disarm.js +46 -2
- package/lib/watchdog-schedule.js +834 -17
- package/lib/watchdog-tier.js +8 -1
- package/lib/watchdog.js +1222 -47
- package/locale/en.json +175 -68
- package/locale/zh.json +175 -68
- package/package.json +3 -2
- package/screenshots.json +6 -6
- package/scripts/README.md +6 -3
- package/scripts/build-icon.mjs +998 -0
- package/scripts/clear-dsh-auth-cookies.mjs +257 -0
- package/scripts/emergency-disable.sh +69 -2
- package/scripts/emergency-toggle.ps1 +85 -69
- package/scripts/rescue-strings.zh.json +36 -18
- package/scripts/restore-profile.ps1 +182 -1
- package/scripts/watchdog.mjs +432 -79
package/scripts/watchdog.mjs
CHANGED
|
@@ -6,10 +6,24 @@
|
|
|
6
6
|
* does is always the same, and it lives in lib/watchdog.js:
|
|
7
7
|
*
|
|
8
8
|
* wait for the ready marker -> ask again -> is an instance alive? -> take
|
|
9
|
-
* the lock ->
|
|
9
|
+
* the lock -> (up to maxAttempts times) ask again -> CLEAR THE STALE
|
|
10
|
+
* HANDSHAKE COOKIES -> restore the snapshot -> start the host again -> wait
|
|
11
|
+
* for the ready marker -> report.
|
|
10
12
|
*
|
|
11
13
|
* This file only provides the parts that touch the machine: the health probe,
|
|
12
|
-
* the restore command, the start-again command, and the
|
|
14
|
+
* the cookie cleanup, the restore command, the start-again command, and the
|
|
15
|
+
* lock.
|
|
16
|
+
*
|
|
17
|
+
* NOTHING MAY EXIT SILENTLY. On 2026-10-30 this process never ran a single
|
|
18
|
+
* line (the registered task started the desktop executable instead of node,
|
|
19
|
+
* through an extra `cmd /c`), and the report still said `armed` for days: the
|
|
20
|
+
* watchdog was dead and nothing said so. Every way out of this file therefore
|
|
21
|
+
* leaves a report behind - a failure to LOAD the plugin included:
|
|
22
|
+
*
|
|
23
|
+
* stage 'import' - the modules could not be loaded;
|
|
24
|
+
* stage 'config' - there is no state directory, or no pending marker;
|
|
25
|
+
* stage 'flow' - the flow ran and could not rescue the host;
|
|
26
|
+
* stage 'relaunch' - the flow could not even start the host again.
|
|
13
27
|
*
|
|
14
28
|
* It is ASCII on purpose (like every other script of this plugin): the
|
|
15
29
|
* console reads it with the code page of the machine.
|
|
@@ -18,17 +32,91 @@
|
|
|
18
32
|
import { execFileSync, spawn } from 'node:child_process';
|
|
19
33
|
import fs from 'node:fs';
|
|
20
34
|
import http from 'node:http';
|
|
35
|
+
import os from 'node:os';
|
|
21
36
|
import path from 'node:path';
|
|
37
|
+
import { fileURLToPath } from 'node:url';
|
|
38
|
+
|
|
39
|
+
/** Where this file lives (`<plugin>/scripts/watchdog.mjs`). */
|
|
40
|
+
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
|
41
|
+
/** The plugin directory. */
|
|
42
|
+
const PLUGIN_DIR = path.dirname(HERE);
|
|
43
|
+
|
|
44
|
+
/** The report file name (the plugin, the supervisor and the UI read it). */
|
|
45
|
+
const REPORT_NAME = 'last-watchdog-report.json';
|
|
46
|
+
/** The names of the scheduling items this plugin uses (one name each). */
|
|
47
|
+
const TASK_NAME = 'dsh-autorestart-watchdog';
|
|
48
|
+
const UNIT_ID = 'dsh-autorestart-watchdog';
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* The state directory of the plugin when nothing was passed.
|
|
52
|
+
*
|
|
53
|
+
* It is only used to leave a report on the "there is no --state-dir" path: a
|
|
54
|
+
* watchdog that cannot even say where its state lives still has to say that
|
|
55
|
+
* it refused, in the place the plugin reads.
|
|
56
|
+
*/
|
|
57
|
+
const DEFAULT_STATE_DIR = path.join(os.homedir(), '.dsh', 'storages', 'dsh-autorestart');
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* The file a REFUSAL is written to when the state directory is unknown.
|
|
61
|
+
*
|
|
62
|
+
* WHY NOT `last-watchdog-report.json` (2026-10-30, review finding F3): the old
|
|
63
|
+
* code called `reportAndWarn(DEFAULT_STATE_DIR, ...)` on this path, so a run
|
|
64
|
+
* that had no `--state-dir` OVERWROTE the report of the last REAL rescue with
|
|
65
|
+
* its own "I never started" note (t6 drill, measured: a 320-byte rescue record
|
|
66
|
+
* replaced by a 586-byte refusal). A refusal is not a rescue result and must
|
|
67
|
+
* never be filed as one - so it goes to a file of its own, in a directory that
|
|
68
|
+
* is NOT the installation (`DSH_WATCHDOG_REFUSAL_DIR`, else the temp dir).
|
|
69
|
+
*/
|
|
70
|
+
const REFUSAL_NAME = 'dsh-autorestart-watchdog-refusal.json';
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Where a refusal record may be written (never the plugin state directory).
|
|
74
|
+
*
|
|
75
|
+
* @returns {string}
|
|
76
|
+
*/
|
|
77
|
+
function refusalDir() {
|
|
78
|
+
const override = typeof process.env.DSH_WATCHDOG_REFUSAL_DIR === 'string'
|
|
79
|
+
? process.env.DSH_WATCHDOG_REFUSAL_DIR.trim()
|
|
80
|
+
: '';
|
|
81
|
+
return override !== '' ? override : os.tmpdir();
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Write the record of a refusal, and never throw while doing it.
|
|
86
|
+
*
|
|
87
|
+
* @param {{ result?: string, stage?: string, error?: string, reason?: string }} fields
|
|
88
|
+
* @returns {string} the path it was written to ('' when it could not be written)
|
|
89
|
+
*/
|
|
90
|
+
function writeRefusalRecord(fields = {}) {
|
|
91
|
+
const dir = refusalDir();
|
|
92
|
+
const target = path.join(dir, REFUSAL_NAME);
|
|
93
|
+
const record = {
|
|
94
|
+
at: new Date().toISOString(),
|
|
95
|
+
result: typeof fields.result === 'string' ? fields.result : 'skipped',
|
|
96
|
+
stage: typeof fields.stage === 'string' ? fields.stage : 'config',
|
|
97
|
+
error: typeof fields.error === 'string' ? fields.error : null,
|
|
98
|
+
reason: typeof fields.reason === 'string' ? fields.reason : null,
|
|
99
|
+
// Say where it WOULD have been written, so an operator can finish by hand.
|
|
100
|
+
stateDirWhenKnown: null,
|
|
101
|
+
note: 'this run never started: it is NOT a rescue result and must not overwrite last-watchdog-report.json',
|
|
102
|
+
};
|
|
103
|
+
try {
|
|
104
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
105
|
+
fs.writeFileSync(target, JSON.stringify(record, null, 2), 'utf8');
|
|
106
|
+
return target;
|
|
107
|
+
} catch {
|
|
108
|
+
return '';
|
|
109
|
+
}
|
|
110
|
+
}
|
|
22
111
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
import { removeScheduleItem } from '../lib/watchdog-schedule.js';
|
|
112
|
+
/**
|
|
113
|
+
* WHERE THE RUN STOOD when it ended: 'import' | 'config' | 'flow' | 'relaunch'.
|
|
114
|
+
* It is a module-level value because the crash handlers below have to name the
|
|
115
|
+
* stage the process was in when it died.
|
|
116
|
+
*/
|
|
117
|
+
let stage = 'config';
|
|
118
|
+
/** The state directory as soon as it is known (the crash handlers use it). */
|
|
119
|
+
let resolvedStateDir = DEFAULT_STATE_DIR;
|
|
32
120
|
|
|
33
121
|
/**
|
|
34
122
|
* The value of a command line flag.
|
|
@@ -44,6 +132,111 @@ function valueOf(args, name) {
|
|
|
44
132
|
return typeof next === 'string' && !next.startsWith('--') ? next : '';
|
|
45
133
|
}
|
|
46
134
|
|
|
135
|
+
/**
|
|
136
|
+
* Write a report, and never throw while doing it.
|
|
137
|
+
*
|
|
138
|
+
* This is the raw writer on purpose: it has to work when the plugin modules
|
|
139
|
+
* could NOT be loaded (that is the 'import' failure this file has to report).
|
|
140
|
+
*
|
|
141
|
+
* @param {string} stateDir
|
|
142
|
+
* @param {object} report
|
|
143
|
+
* @returns {boolean} whether the report is on disk now
|
|
144
|
+
*/
|
|
145
|
+
function writeReportFile(stateDir, report) {
|
|
146
|
+
const where = typeof stateDir === 'string' && stateDir !== '' ? stateDir : DEFAULT_STATE_DIR;
|
|
147
|
+
try {
|
|
148
|
+
fs.mkdirSync(where, { recursive: true });
|
|
149
|
+
fs.writeFileSync(path.join(where, REPORT_NAME), JSON.stringify(report, null, 2), 'utf8');
|
|
150
|
+
return true;
|
|
151
|
+
} catch {
|
|
152
|
+
return false;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* The report of a run that never got as far as the flow.
|
|
158
|
+
*
|
|
159
|
+
* The shape is the same as the one the flow writes (the plugins and the UI
|
|
160
|
+
* read the same file): nothing is renamed, and the fields that cannot be known
|
|
161
|
+
* yet say so (`maxAttempts: 0`, `attempts: 0`).
|
|
162
|
+
*
|
|
163
|
+
* @param {{
|
|
164
|
+
* result?: string,
|
|
165
|
+
* stage?: string,
|
|
166
|
+
* error?: string,
|
|
167
|
+
* reason?: string,
|
|
168
|
+
* tier?: string,
|
|
169
|
+
* }} fields
|
|
170
|
+
* @returns {object}
|
|
171
|
+
*/
|
|
172
|
+
function startupReport(fields = {}) {
|
|
173
|
+
const error = typeof fields.error === 'string' ? fields.error : null;
|
|
174
|
+
const reason = typeof fields.reason === 'string' ? fields.reason : null;
|
|
175
|
+
return {
|
|
176
|
+
at: new Date().toISOString(),
|
|
177
|
+
result: fields.result || 'failed',
|
|
178
|
+
tier: fields.tier || 'none',
|
|
179
|
+
tierReason: '',
|
|
180
|
+
arbiter: 'v2',
|
|
181
|
+
waitedMs: 0,
|
|
182
|
+
rechecked: false,
|
|
183
|
+
healthCheck: 'unknown',
|
|
184
|
+
healthDetail: null,
|
|
185
|
+
health: null,
|
|
186
|
+
extraRound: false,
|
|
187
|
+
restored: false,
|
|
188
|
+
relaunched: false,
|
|
189
|
+
ready: false,
|
|
190
|
+
error: error === null ? reason : error,
|
|
191
|
+
stage: fields.stage || 'config',
|
|
192
|
+
// The flow never ran, so no attempt was allowed and none was made.
|
|
193
|
+
maxAttempts: 0,
|
|
194
|
+
attempts: 0,
|
|
195
|
+
cookieClear: null,
|
|
196
|
+
attemptLog: [],
|
|
197
|
+
reason,
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* Say why this process did not rescue anything, on the disk AND on stderr.
|
|
203
|
+
*
|
|
204
|
+
* @param {string} stateDir
|
|
205
|
+
* @param {object} fields
|
|
206
|
+
* @returns {boolean} whether the report is on disk now
|
|
207
|
+
*/
|
|
208
|
+
function reportAndWarn(stateDir, fields) {
|
|
209
|
+
const report = startupReport(fields);
|
|
210
|
+
const wrote = writeReportFile(stateDir, report);
|
|
211
|
+
const text = report.error || report.reason || 'the watchdog did not run';
|
|
212
|
+
console.error(
|
|
213
|
+
'dsh-autorestart watchdog: ' + text
|
|
214
|
+
+ (wrote ? '' : ' (and no report could be written to ' + stateDir + ')'),
|
|
215
|
+
);
|
|
216
|
+
return wrote;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* A crash is not an excuse to leave no trace.
|
|
221
|
+
*
|
|
222
|
+
* @param {string} kind
|
|
223
|
+
* @param {unknown} error
|
|
224
|
+
*/
|
|
225
|
+
function onFatal(kind, error) {
|
|
226
|
+
const detail = error && error.message ? error.message : String(error);
|
|
227
|
+
stage = stage === '' ? 'flow' : stage;
|
|
228
|
+
reportAndWarn(resolvedStateDir, {
|
|
229
|
+
result: 'failed',
|
|
230
|
+
stage,
|
|
231
|
+
error: kind + ': ' + detail,
|
|
232
|
+
reason: kind + ': ' + detail,
|
|
233
|
+
});
|
|
234
|
+
process.exit(1);
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
process.on('uncaughtException', (error) => onFatal('uncaught exception', error));
|
|
238
|
+
process.on('unhandledRejection', (reason) => onFatal('unhandled rejection', reason));
|
|
239
|
+
|
|
47
240
|
/**
|
|
48
241
|
* Ask a health route whether it is alive.
|
|
49
242
|
*
|
|
@@ -117,41 +310,6 @@ function healthCandidates(pending) {
|
|
|
117
310
|
return urls;
|
|
118
311
|
}
|
|
119
312
|
|
|
120
|
-
/**
|
|
121
|
-
* ASK THE HOST ITSELF: is it alive?
|
|
122
|
-
*
|
|
123
|
-
* This is the last question before a restore (see step 3b of
|
|
124
|
-
* lib/watchdog.js). It answers:
|
|
125
|
-
*
|
|
126
|
-
* 'healthy' -- a health route answered `{"ok":true}`, or an instance of the
|
|
127
|
-
* NEW boot is clearly running: DO NOT RESTORE
|
|
128
|
-
* 'unhealthy' -- every known URL was asked and none answered, and no instance
|
|
129
|
-
* of the new boot is running: the restore may go ahead
|
|
130
|
-
* 'unknown' -- there was no URL to ask at all (no instance record, no
|
|
131
|
-
* health URL): the caller waits one more round
|
|
132
|
-
*
|
|
133
|
-
* The instance record is checked as well as the ports, because a port in that
|
|
134
|
-
* record can be the WRONG one (it may carry the configured port instead of the
|
|
135
|
-
* one the server really listens on). The pid of the new boot is a harder fact
|
|
136
|
-
* than a failed HTTP request, and "never restore a live host" is the rule that
|
|
137
|
-
* matters here.
|
|
138
|
-
*
|
|
139
|
-
* @param {object} pending
|
|
140
|
-
* @returns {Promise<{ state: 'healthy'|'unhealthy'|'unknown', detail: string|null }>}
|
|
141
|
-
*/
|
|
142
|
-
async function askHealth(pending) {
|
|
143
|
-
const urls = healthCandidates(pending);
|
|
144
|
-
for (const url of urls) {
|
|
145
|
-
if (await healthOk(url)) return { state: 'healthy', detail: 'the health route answered: ' + url };
|
|
146
|
-
}
|
|
147
|
-
const record = readInstanceRecord(pending.portFile);
|
|
148
|
-
if (record && record.pid && record.bootId !== pending.expectedBootId && isAlive(record.pid)) {
|
|
149
|
-
return { state: 'healthy', detail: 'the instance of the new boot is running (pid ' + record.pid + ')' };
|
|
150
|
-
}
|
|
151
|
-
if (urls.length === 0) return { state: 'unknown', detail: null };
|
|
152
|
-
return { state: 'unhealthy', detail: 'no health route answered (' + urls.length + ' asked), no live instance' };
|
|
153
|
-
}
|
|
154
|
-
|
|
155
313
|
/**
|
|
156
314
|
* The instance record the plugin writes at boot (port, pid, boot id).
|
|
157
315
|
*
|
|
@@ -184,24 +342,43 @@ function isAlive(pid) {
|
|
|
184
342
|
}
|
|
185
343
|
|
|
186
344
|
/**
|
|
187
|
-
*
|
|
345
|
+
* ASK THE HOST ITSELF: is it alive?
|
|
346
|
+
*
|
|
347
|
+
* This is the last question before a restore (see step 3b of
|
|
348
|
+
* lib/watchdog.js), and it is asked again before every further attempt. It
|
|
349
|
+
* answers:
|
|
188
350
|
*
|
|
189
|
-
*
|
|
190
|
-
*
|
|
191
|
-
*
|
|
351
|
+
* 'healthy' -- a health route answered `{"ok":true}`, or an instance of the
|
|
352
|
+
* NEW boot is clearly running: DO NOT RESTORE, DO NOT START
|
|
353
|
+
* 'unhealthy' -- every known URL was asked and none answered, and no instance
|
|
354
|
+
* of the new boot is running: a restore/start may go ahead
|
|
355
|
+
* 'unknown' -- there was no URL to ask at all (no instance record, no
|
|
356
|
+
* health URL): the caller waits one more round
|
|
357
|
+
*
|
|
358
|
+
* @param {object} pending
|
|
359
|
+
* @returns {Promise<{ state: string, detail: string|null }>}
|
|
360
|
+
*/
|
|
361
|
+
async function askHealth(pending) {
|
|
362
|
+
const urls = healthCandidates(pending);
|
|
363
|
+
for (const url of urls) {
|
|
364
|
+
if (await healthOk(url)) return { state: 'healthy', detail: 'the health route answered: ' + url };
|
|
365
|
+
}
|
|
366
|
+
const record = readInstanceRecord(pending.portFile);
|
|
367
|
+
if (record && record.pid && record.bootId !== pending.expectedBootId && isAlive(record.pid)) {
|
|
368
|
+
return { state: 'healthy', detail: 'the instance of the new boot is running (pid ' + record.pid + ')' };
|
|
369
|
+
}
|
|
370
|
+
if (urls.length === 0) return { state: 'unknown', detail: null };
|
|
371
|
+
return { state: 'unhealthy', detail: 'no health route answered (' + urls.length + ' asked), no live instance' };
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
/**
|
|
375
|
+
* Is an instance of the host alive?
|
|
192
376
|
*
|
|
193
377
|
* @param {object} pending
|
|
194
378
|
* @returns {Promise<boolean>}
|
|
195
379
|
*/
|
|
196
380
|
async function probeAlive(pending) {
|
|
197
|
-
const record = (
|
|
198
|
-
if (!pending.portFile) return null;
|
|
199
|
-
try {
|
|
200
|
-
return JSON.parse(fs.readFileSync(pending.portFile, 'utf8'));
|
|
201
|
-
} catch {
|
|
202
|
-
return null;
|
|
203
|
-
}
|
|
204
|
-
})();
|
|
381
|
+
const record = readInstanceRecord(pending.portFile);
|
|
205
382
|
if (record && record.bootId && record.port && record.bootId !== pending.expectedBootId) {
|
|
206
383
|
const url = 'http://127.0.0.1:' + record.port + (pending.healthPath || '/__dsh_autorestart/health');
|
|
207
384
|
if (await healthOk(url)) return true;
|
|
@@ -257,29 +434,167 @@ function startAgain(command) {
|
|
|
257
434
|
}
|
|
258
435
|
}
|
|
259
436
|
|
|
260
|
-
|
|
261
|
-
|
|
437
|
+
// ---------------------------------------------------------------------------
|
|
438
|
+
// (0) The state directory.
|
|
439
|
+
// ---------------------------------------------------------------------------
|
|
440
|
+
|
|
441
|
+
const argv = process.argv.slice(2);
|
|
442
|
+
const fromArgs = valueOf(argv, '--state-dir');
|
|
443
|
+
const fromEnv = typeof process.env.DSH_WATCHDOG_STATE_DIR === 'string' ? process.env.DSH_WATCHDOG_STATE_DIR : '';
|
|
444
|
+
const stateDir = fromArgs !== '' ? fromArgs : fromEnv;
|
|
445
|
+
|
|
262
446
|
if (stateDir === '') {
|
|
263
|
-
|
|
447
|
+
// A watchdog that does not know where the markers live cannot watch
|
|
448
|
+
// anything - and "it did nothing" must not be invisible.
|
|
449
|
+
//
|
|
450
|
+
// BUT IT MUST NOT FILE ITSELF AS A RESCUE RESULT (review finding F3): the
|
|
451
|
+
// state directory is UNKNOWN here, so writing `last-watchdog-report.json`
|
|
452
|
+
// would mean writing into the default installation's directory and
|
|
453
|
+
// overwriting the report of the last REAL rescue with this refusal. The
|
|
454
|
+
// refusal goes to its own file (outside the installation) plus stderr.
|
|
455
|
+
const reason = 'no state directory (--state-dir or DSH_WATCHDOG_STATE_DIR): nothing could be watched';
|
|
456
|
+
const refusalPath = writeRefusalRecord({ result: 'skipped', stage: 'config', error: reason, reason });
|
|
457
|
+
console.error(
|
|
458
|
+
'dsh-autorestart watchdog: ' + reason
|
|
459
|
+
+ ' (refusal recorded at ' + (refusalPath === '' ? 'nothing: the refusal record could not be written' : refusalPath) + ')',
|
|
460
|
+
);
|
|
461
|
+
process.exit(2);
|
|
462
|
+
}
|
|
463
|
+
resolvedStateDir = stateDir;
|
|
464
|
+
|
|
465
|
+
// ---------------------------------------------------------------------------
|
|
466
|
+
// (1) The plugin modules. A plugin that cannot be loaded is a watchdog that
|
|
467
|
+
// cannot rescue anything - and it says so on disk, not only on a console
|
|
468
|
+
// nobody is looking at.
|
|
469
|
+
// ---------------------------------------------------------------------------
|
|
470
|
+
|
|
471
|
+
let watchdog = null;
|
|
472
|
+
let schedule = null;
|
|
473
|
+
let cookieClear = null;
|
|
474
|
+
let nodeRuntime = null;
|
|
475
|
+
try {
|
|
476
|
+
watchdog = await import('../lib/watchdog.js');
|
|
477
|
+
schedule = await import('../lib/watchdog-schedule.js');
|
|
478
|
+
cookieClear = await import('../lib/cookie-clear.js');
|
|
479
|
+
nodeRuntime = await import('../lib/node-runtime.js');
|
|
480
|
+
} catch (error) {
|
|
481
|
+
const detail = error && error.message ? error.message : String(error);
|
|
482
|
+
reportAndWarn(stateDir, {
|
|
483
|
+
result: 'failed',
|
|
484
|
+
stage: 'import',
|
|
485
|
+
error: detail,
|
|
486
|
+
reason: 'the plugin could not be loaded: ' + detail,
|
|
487
|
+
});
|
|
488
|
+
process.exit(1);
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
// ---------------------------------------------------------------------------
|
|
492
|
+
// (2) The cookie cleanup (step 1 of the task: BEFORE the host is started).
|
|
493
|
+
// ---------------------------------------------------------------------------
|
|
494
|
+
|
|
495
|
+
/**
|
|
496
|
+
* Remove the stale `dsh-auth-*` handshake cookies.
|
|
497
|
+
*
|
|
498
|
+
* The cleanup script (scripts/clear-dsh-auth-cookies.mjs) always exits 0: a
|
|
499
|
+
* restart must never be blocked by a cleanup that could not run. So `ok` here
|
|
500
|
+
* means "the cleanup process ran to its end", NOT "cookies were deleted" - the
|
|
501
|
+
* script writes its own count into clear-cookies.log, and `removed` stays null
|
|
502
|
+
* because this runner did not measure it. A count that was never measured is
|
|
503
|
+
* reported as not measured.
|
|
504
|
+
*
|
|
505
|
+
* The interpreter is RESOLVED (lib/node-runtime.js), never `process.execPath`:
|
|
506
|
+
* inside the desktop app that is the app executable, which would start a
|
|
507
|
+
* second client instead of running the script.
|
|
508
|
+
*
|
|
509
|
+
* @returns {{ ok: boolean, removed: number|null, error: string|null }}
|
|
510
|
+
*/
|
|
511
|
+
function clearStaleCookies() {
|
|
512
|
+
const script = path.join(PLUGIN_DIR, 'scripts', cookieClear.COOKIE_CLEAR_SCRIPT_NAME);
|
|
513
|
+
if (!fs.existsSync(script)) {
|
|
514
|
+
return { ok: false, removed: null, error: 'the cleanup script is not installed: ' + script };
|
|
515
|
+
}
|
|
516
|
+
const interpreter = nodeRuntime.resolveNodeInterpreter({
|
|
517
|
+
execPath: process.execPath,
|
|
518
|
+
platform: process.platform,
|
|
519
|
+
});
|
|
520
|
+
if (interpreter.ok !== true) {
|
|
521
|
+
return { ok: false, removed: null, error: interpreter.reason || 'no node interpreter was found' };
|
|
522
|
+
}
|
|
523
|
+
try {
|
|
524
|
+
execFileSync(interpreter.path, [script], {
|
|
525
|
+
timeout: 60000,
|
|
526
|
+
windowsHide: true,
|
|
527
|
+
stdio: 'ignore',
|
|
528
|
+
});
|
|
529
|
+
return { ok: true, removed: null, error: null };
|
|
530
|
+
} catch (error) {
|
|
531
|
+
return { ok: false, removed: null, error: error && error.message ? error.message : String(error) };
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
// ---------------------------------------------------------------------------
|
|
536
|
+
// (3) The pending marker. No marker: nothing to watch - and still a trace.
|
|
537
|
+
// ---------------------------------------------------------------------------
|
|
538
|
+
|
|
539
|
+
// THE MARKER IS READ WITH ITS REASON (review finding F4): "there is no marker"
|
|
540
|
+
// and "there is a marker this file cannot read" are different answers, and the
|
|
541
|
+
// second one used to be reported as the first - the process cleaned up its
|
|
542
|
+
// scheduling item and exited 0 while looking like a run that had done its job.
|
|
543
|
+
const pendingState = watchdog.readPendingState(stateDir);
|
|
544
|
+
if (pendingState.state === 'corrupt') {
|
|
545
|
+
reportAndWarn(stateDir, {
|
|
546
|
+
result: 'failed',
|
|
547
|
+
stage: 'config',
|
|
548
|
+
error: 'the pending marker exists but cannot be read: ' + String(pendingState.error || 'unknown reason'),
|
|
549
|
+
reason: 'the pending marker exists but cannot be read (nothing was restored)',
|
|
550
|
+
});
|
|
551
|
+
console.error('dsh-autorestart watchdog: the pending marker exists but cannot be read - nothing was restored');
|
|
264
552
|
process.exit(2);
|
|
265
553
|
}
|
|
266
|
-
|
|
554
|
+
|
|
555
|
+
const pending = pendingState.pending;
|
|
267
556
|
if (!pending) {
|
|
268
|
-
//
|
|
269
|
-
//
|
|
270
|
-
//
|
|
557
|
+
// ORDER (2026-11-01): the report goes FIRST, the cleanup second. The cleanup
|
|
558
|
+
// below can `systemctl --user stop dsh-autorestart-watchdog.service` - the
|
|
559
|
+
// service this very process runs in - and systemd answers that with SIGTERM:
|
|
560
|
+
// anything written after it never reaches the disk. Same rule as the `finally`
|
|
561
|
+
// of runWatchdogFlow (lib/watchdog.js).
|
|
562
|
+
//
|
|
563
|
+
// The empty exit is written down: on 2026-10-30 this path deleted the task
|
|
564
|
+
// and left not one line behind, so nobody could tell it had ever run.
|
|
565
|
+
const wrote = reportAndWarn(stateDir, {
|
|
566
|
+
result: 'skipped',
|
|
567
|
+
stage: 'config',
|
|
568
|
+
error: 'no pending marker: there was nothing to watch',
|
|
569
|
+
reason: 'no pending marker',
|
|
570
|
+
});
|
|
571
|
+
console.log('dsh-autorestart watchdog: no pending marker - nothing to watch');
|
|
572
|
+
// The scheduling item still goes: a task that fires once and then stays in the
|
|
573
|
+
// task scheduler is garbage. The systemd tier is cleaned WITHOUT stopping its
|
|
574
|
+
// service (`includeService: false`): this process RUNS in that service, so a
|
|
575
|
+
// `systemctl stop` would kill the process doing the cleaning - the one-shot
|
|
576
|
+
// service disappears by itself once the process exits.
|
|
271
577
|
try {
|
|
272
|
-
removeScheduleItem({ tier: 'scheduled-task', taskName:
|
|
273
|
-
removeScheduleItem({ tier: 'systemd-run', unitId:
|
|
578
|
+
schedule.removeScheduleItem({ tier: 'scheduled-task', taskName: TASK_NAME });
|
|
579
|
+
schedule.removeScheduleItem({ tier: 'systemd-run', unitId: UNIT_ID, includeService: false });
|
|
274
580
|
} catch {
|
|
275
581
|
// It was never there.
|
|
276
582
|
}
|
|
277
|
-
|
|
278
|
-
|
|
583
|
+
// A run whose report could not be written must not exit 0: no report means
|
|
584
|
+
// nobody can tell what happened.
|
|
585
|
+
process.exit(wrote ? 0 : 1);
|
|
279
586
|
}
|
|
280
|
-
const lockFile = path.join(stateDir, LOCK_NAME);
|
|
281
587
|
|
|
282
|
-
const
|
|
588
|
+
const lockFile = path.join(stateDir, watchdog.LOCK_NAME);
|
|
589
|
+
stage = 'flow';
|
|
590
|
+
|
|
591
|
+
// WHAT MAY BE PUT BACK, read from the marker through the SAME normalizer the
|
|
592
|
+
// flow uses, so the CLI and the flow cannot disagree about which copy is the
|
|
593
|
+
// target (2026-11-01: the field used to be called `snapshotDir` and was in fact
|
|
594
|
+
// the certified copy, not a snapshot).
|
|
595
|
+
const restoreTarget = watchdog.restoreTargetInfoOf(pending);
|
|
596
|
+
|
|
597
|
+
const report = await watchdog.runWatchdogFlow({
|
|
283
598
|
pending,
|
|
284
599
|
stateDir,
|
|
285
600
|
deps: {
|
|
@@ -288,23 +603,61 @@ const report = await runWatchdogFlow({
|
|
|
288
603
|
// host that answers must never be rolled back, even when it forgot to
|
|
289
604
|
// write the ready marker.
|
|
290
605
|
askHealth: () => askHealth(pending),
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
606
|
+
// BEFORE the host is started again: the stale handshake cookies go. A
|
|
607
|
+
// host that is started with them dies again with 431.
|
|
608
|
+
clearCookies: () => clearStaleCookies(),
|
|
609
|
+
// STRICT MODE (2026-11-01): with no trusted restore point this refuses
|
|
610
|
+
// instead of restoring something. The flow already stops before calling it;
|
|
611
|
+
// this is the second lock on the same door, because restoring an unknown
|
|
612
|
+
// copy is the one mistake this whole feature must not make.
|
|
613
|
+
runRestore: () => {
|
|
614
|
+
if (restoreTarget.blocked === true || restoreTarget.dir === '') {
|
|
615
|
+
return {
|
|
616
|
+
ok: false,
|
|
617
|
+
error: 'no trusted restore point: ' + (restoreTarget.reason || 'nothing to restore'),
|
|
618
|
+
copied: [],
|
|
619
|
+
links: [],
|
|
620
|
+
};
|
|
621
|
+
}
|
|
622
|
+
return pending.restoreCommand
|
|
623
|
+
? runCommand(pending.restoreCommand)
|
|
624
|
+
: watchdog.restoreFromSnapshot({ restoreTargetDir: restoreTarget.dir, profileDir: pending.profileDir });
|
|
625
|
+
},
|
|
294
626
|
runRelaunch: () => startAgain(pending.relaunchCommand),
|
|
295
|
-
acquireLock: () => acquireLock(lockFile),
|
|
296
|
-
releaseLock: () => releaseLock(lockFile),
|
|
627
|
+
acquireLock: () => watchdog.acquireLock(lockFile),
|
|
628
|
+
releaseLock: () => watchdog.releaseLock(lockFile),
|
|
629
|
+
// SELF-CLEANUP MUST NOT KILL ITSELF (2026-11-01). `includeService: false`
|
|
630
|
+
// makes the systemd tier stop the TIMER only: this CLI runs inside
|
|
631
|
+
// `dsh-autorestart-watchdog.service`, so `systemctl --user stop` of that
|
|
632
|
+
// unit hands this process a SIGTERM. The report is already on the disk by
|
|
633
|
+
// then (the flow writes it before any cleanup), but a process that kills
|
|
634
|
+
// itself mid-cleanup is still a bug, so both halves are fixed here.
|
|
297
635
|
removeSchedule: () => {
|
|
298
|
-
removeScheduleItem({
|
|
636
|
+
schedule.removeScheduleItem({
|
|
299
637
|
tier: pending.tier,
|
|
300
638
|
taskName: pending.taskName,
|
|
301
639
|
unitId: pending.unitId,
|
|
302
640
|
pidPath: pending.pidPath,
|
|
641
|
+
includeService: false,
|
|
303
642
|
});
|
|
304
643
|
},
|
|
305
644
|
},
|
|
306
645
|
});
|
|
307
646
|
|
|
308
|
-
|
|
647
|
+
// The report itself is written by the flow (from its finally, BEFORE any cleanup
|
|
648
|
+
// that could stop this process), so it exists whatever the result was. This is
|
|
649
|
+
// only the console summary.
|
|
650
|
+
console.log(
|
|
651
|
+
'dsh-autorestart watchdog: ' + report.result
|
|
652
|
+
+ ' (tier ' + report.tier + ', attempt ' + report.attempts + '/' + report.maxAttempts + ')',
|
|
653
|
+
);
|
|
654
|
+
if (report.cookieClear) {
|
|
655
|
+
console.log(' cookie cleanup: ' + (report.cookieClear.ok === true ? 'ran' : 'FAILED: ' + report.cookieClear.error));
|
|
656
|
+
}
|
|
309
657
|
if (report.error) console.log(' ' + report.error);
|
|
310
|
-
|
|
658
|
+
// A report that could not be written is a failed run, whatever the rescue did:
|
|
659
|
+
// the next reader would see a rescue nobody can prove.
|
|
660
|
+
if (report.reportWriteError) {
|
|
661
|
+
console.error('dsh-autorestart watchdog: the report could not be written: ' + report.reportWriteError);
|
|
662
|
+
}
|
|
663
|
+
process.exit(report.result === 'failed' || report.reportWriteError ? 1 : 0);
|