dsh-autorestart 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +97 -0
- package/README.md +55 -6
- package/client.js +102 -9
- package/docs/README.zh.md +4 -2
- package/index.js +514 -39
- package/lib/backup.js +279 -9
- package/lib/checkpoint.js +134 -0
- package/lib/diagnostics.js +18 -0
- package/lib/restart-phase.js +263 -0
- package/lib/watchdog-disarm.js +506 -0
- package/lib/watchdog.js +86 -0
- package/locale/en.json +1 -0
- package/locale/zh.json +1 -0
- package/package.json +6 -1
- package/scripts/drill-rescue.sh +427 -0
- package/scripts/drill-watchdog.mjs +474 -0
- package/scripts/emergency-disable.sh +314 -0
- package/scripts/rescue-strings.zh.json +168 -0
- package/scripts/restore-profile.sh +593 -0
- package/scripts/watchdog.mjs +310 -0
|
@@ -0,0 +1,506 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DISARMING THE WATCHDOG — the host is up, so nothing may roll it back.
|
|
3
|
+
*
|
|
4
|
+
* THE INCIDENT THIS FILE EXISTS FOR (0.8.1, 2026-10-29):
|
|
5
|
+
*
|
|
6
|
+
* A restart armed the watchdog (`watchdog-pending.json` + a one-off Windows
|
|
7
|
+
* scheduled task). The new host came up and answered its own health route
|
|
8
|
+
* with 200 — and still the ready marker was never written, the pending
|
|
9
|
+
* marker stayed, and the scheduled task stayed `Ready`. Had nobody
|
|
10
|
+
* interfered, the watchdog would have restored the snapshot of a HEALTHY
|
|
11
|
+
* profile 90 seconds later.
|
|
12
|
+
*
|
|
13
|
+
* The reason was not a filesystem problem at all: the health route read the
|
|
14
|
+
* state directory from a variable that only existed inside the runtime
|
|
15
|
+
* factory, so the lookup threw `ReferenceError: stateDirOf is not defined`
|
|
16
|
+
* on EVERY request. The `catch {}` around it swallowed that, and the
|
|
17
|
+
* "nothing may be left behind" branch never ran. Nobody could see it: the
|
|
18
|
+
* route still answered 200, and nothing was recorded anywhere.
|
|
19
|
+
*
|
|
20
|
+
* THE TWO RULES THAT FOLLOW FROM IT:
|
|
21
|
+
*
|
|
22
|
+
* 1. The ready marker comes FIRST and on its own. It is the one thing that
|
|
23
|
+
* stops a rollback, it must never depend on the scheduling item, and a
|
|
24
|
+
* failure to write it must be recorded (never swallowed).
|
|
25
|
+
* 2. Removing the scheduling item is an OPERATION ON THE MACHINE
|
|
26
|
+
* (`schtasks`, `systemctl`, a kill). It can fail, so it is retried with a
|
|
27
|
+
* backoff, and when every attempt fails the record says so (and says with
|
|
28
|
+
* which error). A silently failed disarm is the incident all over again.
|
|
29
|
+
*
|
|
30
|
+
* Everything here is ESM and Node builtins only, and every effect is
|
|
31
|
+
* injectable: a test drives the whole thing without touching the machine.
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
import { execFileSync } from 'node:child_process';
|
|
35
|
+
import fs from 'node:fs';
|
|
36
|
+
import path from 'node:path';
|
|
37
|
+
|
|
38
|
+
import {
|
|
39
|
+
clearPending,
|
|
40
|
+
readPending,
|
|
41
|
+
readReady,
|
|
42
|
+
writeReady,
|
|
43
|
+
} from './watchdog.js';
|
|
44
|
+
import { removeScheduleItem } from './watchdog-schedule.js';
|
|
45
|
+
import { TIERS } from './watchdog-tier.js';
|
|
46
|
+
|
|
47
|
+
/** The file that records how the last disarm went. */
|
|
48
|
+
export const DISARM_FILE_NAME = 'watchdog-disarm.json';
|
|
49
|
+
|
|
50
|
+
/** How often the disarm is tried before it gives up (and says so). */
|
|
51
|
+
export const DEFAULT_DISARM_ATTEMPTS = 3;
|
|
52
|
+
|
|
53
|
+
/** The first wait between two attempts; it grows with every attempt. */
|
|
54
|
+
export const DEFAULT_DISARM_BACKOFF_MS = 120;
|
|
55
|
+
|
|
56
|
+
/** @param {string} stateDir */
|
|
57
|
+
export function disarmPath(stateDir) {
|
|
58
|
+
return path.join(String(stateDir ?? ''), DISARM_FILE_NAME);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** @param {unknown} value */
|
|
62
|
+
function errorText(value) {
|
|
63
|
+
if (value && typeof value.message === 'string' && value.message) return value.message;
|
|
64
|
+
return String(value);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** @param {unknown} value */
|
|
68
|
+
function errorName(value) {
|
|
69
|
+
if (value && typeof value.name === 'string' && value.name) return value.name;
|
|
70
|
+
return 'Error';
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* One failure, in the shape the diagnostics and the settings page show.
|
|
75
|
+
*
|
|
76
|
+
* @param {number} attempt
|
|
77
|
+
* @param {string} step
|
|
78
|
+
* @param {unknown} error
|
|
79
|
+
*/
|
|
80
|
+
function failure(attempt, step, error) {
|
|
81
|
+
return { attempt, step, name: errorName(error), message: errorText(error) };
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* The record of the last disarm (null when nothing was recorded yet).
|
|
86
|
+
*
|
|
87
|
+
* @param {string} stateDir
|
|
88
|
+
* @returns {object|null}
|
|
89
|
+
*/export function readDisarmRecord(stateDir) {
|
|
90
|
+
const file = disarmPath(stateDir);
|
|
91
|
+
try {
|
|
92
|
+
const parsed = JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
93
|
+
return parsed && typeof parsed === 'object' ? parsed : null;
|
|
94
|
+
} catch {
|
|
95
|
+
return null;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Write the record (never throws: a failed disarm must not fail the host).
|
|
101
|
+
*
|
|
102
|
+
* @param {string} stateDir
|
|
103
|
+
* @param {object} record
|
|
104
|
+
*/
|
|
105
|
+
export function writeDisarmRecord(stateDir, record) {
|
|
106
|
+
const dir = String(stateDir ?? '');
|
|
107
|
+
if (dir === '') return false;
|
|
108
|
+
try {
|
|
109
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
110
|
+
fs.writeFileSync(disarmPath(dir), JSON.stringify(record, null, 2) + '\n', 'utf8');
|
|
111
|
+
return true;
|
|
112
|
+
} catch {
|
|
113
|
+
return false;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* STEP 1: the ready marker — "the new host is up".
|
|
119
|
+
*
|
|
120
|
+
* It is written even when there is no pending marker at all (an idle boot says
|
|
121
|
+
* "this instance is alive"), and it is READ BACK: a write that returned is not
|
|
122
|
+
* a marker that is there.
|
|
123
|
+
*
|
|
124
|
+
* @param {{
|
|
125
|
+
* stateDir?: string,
|
|
126
|
+
* bootId?: string,
|
|
127
|
+
* port?: number|null,
|
|
128
|
+
* now?: Function,
|
|
129
|
+
* read?: Function,
|
|
130
|
+
* write?: Function,
|
|
131
|
+
* }} [options]
|
|
132
|
+
* @returns {{ ok: boolean, wrote: boolean, error: unknown, detail: string }}
|
|
133
|
+
*/
|
|
134
|
+
export function markReady({
|
|
135
|
+
stateDir = '',
|
|
136
|
+
bootId = '',
|
|
137
|
+
port = null,
|
|
138
|
+
now = Date.now,
|
|
139
|
+
read = readReady,
|
|
140
|
+
write = writeReady,
|
|
141
|
+
} = {}) {
|
|
142
|
+
const dir = typeof stateDir === 'string' ? stateDir : '';
|
|
143
|
+
if (dir === '') {
|
|
144
|
+
return { ok: false, wrote: false, error: null, detail: 'there is no state directory to write the marker into' };
|
|
145
|
+
}
|
|
146
|
+
const id = typeof bootId === 'string' ? bootId : '';
|
|
147
|
+
let wrote = false;
|
|
148
|
+
try {
|
|
149
|
+
const existing = read(dir);
|
|
150
|
+
if (id !== '' && existing !== null && existing.bootId === id) {
|
|
151
|
+
// Already ours: this boot has marked itself.
|
|
152
|
+
return { ok: true, wrote: false, error: null, detail: 'the ready marker of this boot is already there' };
|
|
153
|
+
}
|
|
154
|
+
const number = Number(port);
|
|
155
|
+
write(dir, {
|
|
156
|
+
bootId: id === '' ? null : id,
|
|
157
|
+
port: Number.isFinite(number) && number > 0 ? number : null,
|
|
158
|
+
at: now(),
|
|
159
|
+
});
|
|
160
|
+
wrote = true;
|
|
161
|
+
const written = read(dir);
|
|
162
|
+
if (id !== '' && (written === null || written.bootId !== id)) {
|
|
163
|
+
return { ok: false, wrote, error: null, detail: 'the ready marker could not be read back' };
|
|
164
|
+
}
|
|
165
|
+
return { ok: true, wrote, error: null, detail: 'the ready marker was written' };
|
|
166
|
+
} catch (error) {
|
|
167
|
+
return { ok: false, wrote, error, detail: 'the ready marker could not be written: ' + errorText(error) };
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* WHICH scheduling item does this pending marker belong to?
|
|
173
|
+
*
|
|
174
|
+
* The tier is the source of truth; when it is unknown the names are used, so a
|
|
175
|
+
* marker written by an older version is still cleaned up instead of blocking
|
|
176
|
+
* the disarm for ever.
|
|
177
|
+
*
|
|
178
|
+
* @param {object|null} pending
|
|
179
|
+
* @returns {{ tier: string, taskName: string, unitId: string, pidPath: string }}
|
|
180
|
+
*/
|
|
181
|
+
export function scheduleTargetOf(pending) {
|
|
182
|
+
const source = pending && typeof pending === 'object' ? pending : {};
|
|
183
|
+
const taskName = typeof source.taskName === 'string' ? source.taskName : '';
|
|
184
|
+
const unitId = typeof source.unitId === 'string' ? source.unitId : '';
|
|
185
|
+
const pidPath = typeof source.pidPath === 'string' ? source.pidPath : '';
|
|
186
|
+
const known = new Set([TIERS.SCHEDULED_TASK, TIERS.SYSTEMD_RUN, TIERS.DETACHED_PROCESS]);
|
|
187
|
+
let tier = typeof source.tier === 'string' ? source.tier : '';
|
|
188
|
+
if (!known.has(tier)) {
|
|
189
|
+
if (taskName !== '') tier = TIERS.SCHEDULED_TASK;
|
|
190
|
+
else if (unitId !== '') tier = TIERS.SYSTEMD_RUN;
|
|
191
|
+
else if (pidPath !== '') tier = TIERS.DETACHED_PROCESS;
|
|
192
|
+
else tier = TIERS.NONE;
|
|
193
|
+
}
|
|
194
|
+
return { tier, taskName, unitId, pidPath };
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* Read the pending marker without letting a broken read break the disarm.
|
|
199
|
+
*
|
|
200
|
+
* @param {Function} readPendingFn
|
|
201
|
+
* @param {string} stateDir
|
|
202
|
+
* @returns {object|null}
|
|
203
|
+
*/
|
|
204
|
+
function readPendingSafely(readPendingFn, stateDir) {
|
|
205
|
+
try {
|
|
206
|
+
const value = readPendingFn(stateDir);
|
|
207
|
+
return value === undefined ? null : value;
|
|
208
|
+
} catch {
|
|
209
|
+
return null;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* Does the Windows task exist right now?
|
|
215
|
+
*
|
|
216
|
+
* `null` means "this machine cannot tell" (not Windows). It exists so the
|
|
217
|
+
* disarm does not call a failed `schtasks /Delete` a failure when the task was
|
|
218
|
+
* already gone — and does not call a refused delete a success.
|
|
219
|
+
*
|
|
220
|
+
* @param {string} taskName
|
|
221
|
+
* @returns {boolean|null}
|
|
222
|
+
*/
|
|
223
|
+
export function defaultTaskExists(taskName) {
|
|
224
|
+
if (process.platform !== 'win32') return null;
|
|
225
|
+
if (typeof taskName !== 'string' || taskName === '') return false;
|
|
226
|
+
try {
|
|
227
|
+
execFileSync('schtasks', ['/Query', '/TN', taskName], { stdio: 'ignore', timeout: 15000, windowsHide: true });
|
|
228
|
+
return true;
|
|
229
|
+
} catch {
|
|
230
|
+
return false;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/**
|
|
235
|
+
* ONE disarm attempt: the marker, then the two leftovers.
|
|
236
|
+
*
|
|
237
|
+
* Scheduled reminders: `attempt` numbers the record, `errors` collects every
|
|
238
|
+
* failure of THIS attempt (so a partial result is never reported as clean).
|
|
239
|
+
*
|
|
240
|
+
* @param {{
|
|
241
|
+
* stateDir?: string,
|
|
242
|
+
* bootId?: string,
|
|
243
|
+
* port?: number|null,
|
|
244
|
+
* attempt?: number,
|
|
245
|
+
* now?: Function,
|
|
246
|
+
* read?: Function,
|
|
247
|
+
* write?: Function,
|
|
248
|
+
* readPendingFn?: Function,
|
|
249
|
+
* clearPendingFn?: Function,
|
|
250
|
+
* removeItem?: Function,
|
|
251
|
+
* taskExists?: Function,
|
|
252
|
+
* }} [options]
|
|
253
|
+
* @returns {{
|
|
254
|
+
* ok: boolean,
|
|
255
|
+
* ready: boolean,
|
|
256
|
+
* pending: boolean,
|
|
257
|
+
* removed: { tier: string, ok: boolean, error: unknown }|null,
|
|
258
|
+
* errors: Array<{ attempt: number, step: string, name: string, message: string }>,
|
|
259
|
+
* }}
|
|
260
|
+
*/
|
|
261
|
+
export function disarmOnce({
|
|
262
|
+
stateDir = '',
|
|
263
|
+
bootId = '',
|
|
264
|
+
port = null,
|
|
265
|
+
attempt = 1,
|
|
266
|
+
now = Date.now,
|
|
267
|
+
read = readReady,
|
|
268
|
+
write = writeReady,
|
|
269
|
+
readPendingFn = readPending,
|
|
270
|
+
clearPendingFn = clearPending,
|
|
271
|
+
removeItem = removeScheduleItem,
|
|
272
|
+
taskExists = defaultTaskExists,
|
|
273
|
+
pendingHint = null,
|
|
274
|
+
} = {}) {
|
|
275
|
+
const dir = typeof stateDir === 'string' ? stateDir : '';
|
|
276
|
+
const errors = [];
|
|
277
|
+
if (dir === '') {
|
|
278
|
+
errors.push(failure(attempt, 'state-dir', 'there is no state directory to disarm in'));
|
|
279
|
+
return { ok: false, ready: false, pending: false, removed: null, errors };
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// (1) THE MARKER. Nothing else in this function may run before it.
|
|
283
|
+
const marked = markReady({ stateDir: dir, bootId, port, now, read, write });
|
|
284
|
+
if (marked.ok !== true) {
|
|
285
|
+
errors.push(failure(attempt, 'ready-marker', marked.error === null
|
|
286
|
+
? new Error(marked.detail)
|
|
287
|
+
: marked.error));
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// (2) THE PENDING MARKER + (3) THE SCHEDULING ITEM.
|
|
291
|
+
//
|
|
292
|
+
// They are handled separately ON PURPOSE: the marker can be gone (the first
|
|
293
|
+
// attempt cleared it) while the scheduling item is still there (the delete
|
|
294
|
+
// was refused). That is why a later attempt re-tries the removal from
|
|
295
|
+
// `pendingHint` — the marker of the first attempt — instead of giving up.
|
|
296
|
+
let pendingAfter = null;
|
|
297
|
+
let removed = null;
|
|
298
|
+
let pendingMarker = null;
|
|
299
|
+
try {
|
|
300
|
+
pendingMarker = readPendingFn(dir);
|
|
301
|
+
} catch (error) {
|
|
302
|
+
errors.push(failure(attempt, 'read-pending', error));
|
|
303
|
+
}
|
|
304
|
+
if (pendingMarker !== null && pendingMarker !== undefined) {
|
|
305
|
+
try {
|
|
306
|
+
clearPendingFn(dir);
|
|
307
|
+
} catch (error) {
|
|
308
|
+
errors.push(failure(attempt, 'clear-pending', error));
|
|
309
|
+
}
|
|
310
|
+
try {
|
|
311
|
+
pendingAfter = readPendingFn(dir);
|
|
312
|
+
} catch (error) {
|
|
313
|
+
errors.push(failure(attempt, 'read-pending', error));
|
|
314
|
+
pendingAfter = pendingMarker;
|
|
315
|
+
}
|
|
316
|
+
if (pendingAfter !== null && pendingAfter !== undefined) {
|
|
317
|
+
errors.push(failure(attempt, 'clear-pending', new Error('the pending marker is still there after it was removed')));
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
const scheduling = pendingMarker !== null && pendingMarker !== undefined
|
|
321
|
+
? pendingMarker
|
|
322
|
+
: (pendingHint !== null && pendingHint !== undefined ? pendingHint : null);
|
|
323
|
+
if (scheduling !== null) {
|
|
324
|
+
const target = scheduleTargetOf(scheduling);
|
|
325
|
+
const anywhere = target.taskName !== '' || target.unitId !== '' || target.pidPath !== '';
|
|
326
|
+
if (target.tier !== TIERS.NONE && anywhere) {
|
|
327
|
+
// "It answers" is not "it is gone", and a delete of something that is
|
|
328
|
+
// already gone is not a failure. Only the Windows task can be asked
|
|
329
|
+
// about (and only there is the distinction needed).
|
|
330
|
+
let exists = null;
|
|
331
|
+
if (target.tier === TIERS.SCHEDULED_TASK) {
|
|
332
|
+
try {
|
|
333
|
+
exists = typeof taskExists === 'function' ? taskExists(target.taskName) : null;
|
|
334
|
+
} catch {
|
|
335
|
+
exists = null;
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
if (exists === false) {
|
|
339
|
+
// Nothing to delete: another run (or the watchdog itself) was first.
|
|
340
|
+
removed = { tier: target.tier, ok: true, error: null };
|
|
341
|
+
} else {
|
|
342
|
+
let answer = null;
|
|
343
|
+
try {
|
|
344
|
+
answer = removeItem(target);
|
|
345
|
+
} catch (error) {
|
|
346
|
+
errors.push(failure(attempt, 'remove-schedule', error));
|
|
347
|
+
}
|
|
348
|
+
if (answer !== null && answer !== undefined) {
|
|
349
|
+
removed = {
|
|
350
|
+
tier: target.tier,
|
|
351
|
+
ok: answer.ok === true,
|
|
352
|
+
error: answer.ok === true ? null : (answer.error || 'the scheduling item could not be removed'),
|
|
353
|
+
};
|
|
354
|
+
if (removed.ok !== true) {
|
|
355
|
+
errors.push(failure(attempt, 'remove-schedule', new Error(String(removed.error))));
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
let stillThere = null;
|
|
359
|
+
if (target.tier === TIERS.SCHEDULED_TASK) {
|
|
360
|
+
try {
|
|
361
|
+
stillThere = typeof taskExists === 'function' ? taskExists(target.taskName) : null;
|
|
362
|
+
} catch {
|
|
363
|
+
stillThere = null;
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
if (stillThere === true) {
|
|
367
|
+
// A removal that CLAIMED success and left the task behind is the
|
|
368
|
+
// dangerous case; a removal that already reported its failure needs
|
|
369
|
+
// no second line for the same attempt.
|
|
370
|
+
const alreadyReported = removed !== null && removed.ok !== true;
|
|
371
|
+
removed = { tier: target.tier, ok: false, error: 'the scheduling item is still there' };
|
|
372
|
+
if (!alreadyReported) {
|
|
373
|
+
errors.push(failure(attempt, 'remove-schedule', new Error('the scheduling item is still there after it was removed')));
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
const ready = (() => {
|
|
381
|
+
try {
|
|
382
|
+
const marker = read(dir);
|
|
383
|
+
if (marker === null) return false;
|
|
384
|
+
return typeof bootId !== 'string' || bootId === '' ? true : marker.bootId === bootId;
|
|
385
|
+
} catch {
|
|
386
|
+
return false;
|
|
387
|
+
}
|
|
388
|
+
})();
|
|
389
|
+
const pendingLeft = pendingAfter !== null && pendingAfter !== undefined;
|
|
390
|
+
const ok = errors.length === 0 && ready === true && pendingLeft === false
|
|
391
|
+
&& (removed === null || removed.ok === true);
|
|
392
|
+
return { ok, ready, pending: pendingLeft, removed, errors };
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
/**
|
|
396
|
+
* DISARM, WITH RETRIES — "the host is up, so the watchdog has nothing to do".
|
|
397
|
+
*
|
|
398
|
+
* The first attempt runs SYNCHRONOUSLY up to the marker, so a caller inside an
|
|
399
|
+
* HTTP route has the marker on disk before it answers. Only the retries wait.
|
|
400
|
+
*
|
|
401
|
+
* The record is always returned and (unless `writeRecord` is false) written to
|
|
402
|
+
* `<stateDir>/watchdog-disarm.json`, so a failed disarm is visible afterwards
|
|
403
|
+
* instead of being the silent state of 0.8.1.
|
|
404
|
+
*
|
|
405
|
+
* @param {{
|
|
406
|
+
* stateDir?: string,
|
|
407
|
+
* bootId?: string,
|
|
408
|
+
* port?: number|null,
|
|
409
|
+
* source?: string,
|
|
410
|
+
* attempts?: number,
|
|
411
|
+
* backoffMs?: number,
|
|
412
|
+
* sleep?: Function,
|
|
413
|
+
* now?: Function,
|
|
414
|
+
* read?: Function,
|
|
415
|
+
* write?: Function,
|
|
416
|
+
* readPendingFn?: Function,
|
|
417
|
+
* clearPendingFn?: Function,
|
|
418
|
+
* removeItem?: Function,
|
|
419
|
+
* taskExists?: Function,
|
|
420
|
+
* writeRecord?: boolean,
|
|
421
|
+
* log?: { warn?: Function, info?: Function }|null,
|
|
422
|
+
* }} [options]
|
|
423
|
+
* @returns {Promise<object>} the record
|
|
424
|
+
*/
|
|
425
|
+
export async function disarmWatchdog({
|
|
426
|
+
stateDir = '',
|
|
427
|
+
bootId = '',
|
|
428
|
+
port = null,
|
|
429
|
+
source = 'boot',
|
|
430
|
+
attempts = DEFAULT_DISARM_ATTEMPTS,
|
|
431
|
+
backoffMs = DEFAULT_DISARM_BACKOFF_MS,
|
|
432
|
+
sleep = (ms) => new Promise((resolve) => { setTimeout(resolve, ms); }),
|
|
433
|
+
now = Date.now,
|
|
434
|
+
read = readReady,
|
|
435
|
+
write = writeReady,
|
|
436
|
+
readPendingFn = readPending,
|
|
437
|
+
clearPendingFn = clearPending,
|
|
438
|
+
removeItem = removeScheduleItem,
|
|
439
|
+
taskExists = defaultTaskExists,
|
|
440
|
+
writeRecord = true,
|
|
441
|
+
log = null,
|
|
442
|
+
} = {}) {
|
|
443
|
+
const dir = typeof stateDir === 'string' ? stateDir : '';
|
|
444
|
+
const wanted = Number.isFinite(Number(attempts)) && Number(attempts) > 0
|
|
445
|
+
? Math.trunc(Number(attempts))
|
|
446
|
+
: DEFAULT_DISARM_ATTEMPTS;
|
|
447
|
+
const wait = Number.isFinite(Number(backoffMs)) && Number(backoffMs) >= 0
|
|
448
|
+
? Math.trunc(Number(backoffMs))
|
|
449
|
+
: DEFAULT_DISARM_BACKOFF_MS;
|
|
450
|
+
|
|
451
|
+
const errors = [];
|
|
452
|
+
let last = null;
|
|
453
|
+
let used = 0;
|
|
454
|
+
// The pending marker as it was BEFORE the first attempt: the scheduling item
|
|
455
|
+
// it names must be retried even after the marker itself is gone (removing the
|
|
456
|
+
// file can work while the `schtasks /Delete` is refused).
|
|
457
|
+
const hint = readPendingSafely(readPendingFn, dir);
|
|
458
|
+
for (let attempt = 1; attempt <= wanted; attempt += 1) {
|
|
459
|
+
used = attempt;
|
|
460
|
+
last = disarmOnce({
|
|
461
|
+
stateDir: dir,
|
|
462
|
+
bootId,
|
|
463
|
+
port,
|
|
464
|
+
attempt,
|
|
465
|
+
now,
|
|
466
|
+
read,
|
|
467
|
+
write,
|
|
468
|
+
readPendingFn,
|
|
469
|
+
clearPendingFn,
|
|
470
|
+
removeItem,
|
|
471
|
+
taskExists,
|
|
472
|
+
pendingHint: hint,
|
|
473
|
+
});
|
|
474
|
+
errors.push(...last.errors);
|
|
475
|
+
if (last.ok) break;
|
|
476
|
+
if (attempt < wanted && wait > 0) {
|
|
477
|
+
// A failed `schtasks /Delete` is usually a moment later fine (the task
|
|
478
|
+
// scheduler holds a lock while the task is being started): back off.
|
|
479
|
+
await sleep(wait * attempt);
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
const record = {
|
|
484
|
+
at: new Date(typeof now === 'function' ? now() : Date.now()).toISOString(),
|
|
485
|
+
source: String(source ?? 'boot'),
|
|
486
|
+
bootId: typeof bootId === 'string' && bootId !== '' ? bootId : null,
|
|
487
|
+
attempts: used,
|
|
488
|
+
ok: last !== null && last.ok === true,
|
|
489
|
+
ready: last !== null && last.ready === true,
|
|
490
|
+
pending: last !== null && last.pending === true,
|
|
491
|
+
tier: last !== null && last.removed !== null ? last.removed.tier : null,
|
|
492
|
+
scheduleRemoved: last !== null && last.removed !== null ? last.removed.ok === true : null,
|
|
493
|
+
scheduleError: last !== null && last.removed !== null && last.removed.error
|
|
494
|
+
? String(last.removed.error)
|
|
495
|
+
: null,
|
|
496
|
+
errors,
|
|
497
|
+
};
|
|
498
|
+
if (writeRecord) writeDisarmRecord(dir, record);
|
|
499
|
+
if (record.ok !== true && log && typeof log.warn === 'function') {
|
|
500
|
+
log.warn('the watchdog could not be disarmed', {
|
|
501
|
+
attempts: record.attempts,
|
|
502
|
+
errors: record.errors.slice(0, 3),
|
|
503
|
+
});
|
|
504
|
+
}
|
|
505
|
+
return record;
|
|
506
|
+
}
|
package/lib/watchdog.js
CHANGED
|
@@ -11,6 +11,12 @@
|
|
|
11
11
|
* not a host that is dead (the misjudgement protection);
|
|
12
12
|
* 3. still nothing? Is an instance ALIVE? If one is listening, the rollback
|
|
13
13
|
* is not needed: stand down (the no-double-start protection);
|
|
14
|
+
* 3b. THE LAST QUESTION BEFORE A RESTORE: ask the health route ITSELF
|
|
15
|
+
* (HTTP to 127.0.0.1:<port>). Only a host that does not answer may be
|
|
16
|
+
* rolled back. This is what keeps a host that forgot to write the marker
|
|
17
|
+
* — or could not write it — from being restored out from under itself.
|
|
18
|
+
* When there is no port to ask at all (no instance record, no health
|
|
19
|
+
* URL), one more round is waited and the leftovers are looked at again;
|
|
14
20
|
* 4. take the shared lock (`rollback.lock`) — exactly ONE of us restores;
|
|
15
21
|
* 5. restore the snapshot, then START THE HOST AGAIN;
|
|
16
22
|
* 6. wait for the ready marker once more: it came → `recovered`, it did not
|
|
@@ -349,6 +355,26 @@ async function waitForReady(deps, timeoutMs) {
|
|
|
349
355
|
return deps.readReady() === true;
|
|
350
356
|
}
|
|
351
357
|
|
|
358
|
+
/**
|
|
359
|
+
* Normalize the answer of the health question.
|
|
360
|
+
*
|
|
361
|
+
* The dependency may answer with a plain string ('healthy' | 'unhealthy' |
|
|
362
|
+
* 'unknown') or with `{ state, detail }` — the CLI uses the second form, so the
|
|
363
|
+
* report can say WHY the host was believed to be up (a health route answered, or
|
|
364
|
+
* an instance of the new boot is clearly running).
|
|
365
|
+
*
|
|
366
|
+
* @param {unknown} value
|
|
367
|
+
* @returns {{ state: string, detail: string|null }}
|
|
368
|
+
*/
|
|
369
|
+
function healthAnswer(value) {
|
|
370
|
+
if (value && typeof value === 'object') {
|
|
371
|
+
const state = typeof value.state === 'string' ? value.state : 'unknown';
|
|
372
|
+
const detail = typeof value.detail === 'string' && value.detail !== '' ? value.detail : null;
|
|
373
|
+
return { state, detail };
|
|
374
|
+
}
|
|
375
|
+
return { state: typeof value === 'string' ? value : 'unknown', detail: null };
|
|
376
|
+
}
|
|
377
|
+
|
|
352
378
|
/**
|
|
353
379
|
* THE FLOW.
|
|
354
380
|
*
|
|
@@ -367,6 +393,9 @@ export async function runWatchdogFlow({ pending, stateDir = '', deps = {} }) {
|
|
|
367
393
|
readReady: () => readReady(stateDir),
|
|
368
394
|
isWithdrawn: () => readPending(stateDir) === null,
|
|
369
395
|
probeAlive: () => false,
|
|
396
|
+
// THE HEALTH QUESTION (see step 3b): 'healthy' | 'unhealthy' | 'unknown'.
|
|
397
|
+
// The CLI answers it with a real HTTP request; a flow test injects it.
|
|
398
|
+
askHealth: () => 'unknown',
|
|
370
399
|
runRestore: () => ({ ok: false, error: 'no restore command' }),
|
|
371
400
|
runRelaunch: () => ({ ok: false, error: 'no relaunch command' }),
|
|
372
401
|
acquireLock: () => true,
|
|
@@ -386,6 +415,13 @@ export async function runWatchdogFlow({ pending, stateDir = '', deps = {} }) {
|
|
|
386
415
|
arbiter: pending && pending.arbiter ? pending.arbiter : 'v2',
|
|
387
416
|
waitedMs: 0,
|
|
388
417
|
rechecked: false,
|
|
418
|
+
// What the health route of the host itself said before anything was
|
|
419
|
+
// restored: 'healthy' (stood down), 'unhealthy' (really down) or
|
|
420
|
+
// 'unknown' (there was no port to ask).
|
|
421
|
+
healthCheck: 'unknown',
|
|
422
|
+
healthDetail: null,
|
|
423
|
+
health: null,
|
|
424
|
+
extraRound: false,
|
|
389
425
|
restored: false,
|
|
390
426
|
relaunched: false,
|
|
391
427
|
ready: false,
|
|
@@ -426,6 +462,56 @@ export async function runWatchdogFlow({ pending, stateDir = '', deps = {} }) {
|
|
|
426
462
|
return report;
|
|
427
463
|
}
|
|
428
464
|
|
|
465
|
+
// (3b) THE LAST QUESTION BEFORE A RESTORE: ask the health route itself.
|
|
466
|
+
//
|
|
467
|
+
// The marker is written by the plugin, so it can be missing while the host
|
|
468
|
+
// is perfectly healthy (0.8.1: the write threw and the error was
|
|
469
|
+
// swallowed). A restore of a live host is the worst thing this file can
|
|
470
|
+
// do, so the host gets asked directly — and only a host that does not
|
|
471
|
+
// answer may be rolled back.
|
|
472
|
+
const health = healthAnswer(await d.askHealth());
|
|
473
|
+
report.healthCheck = health.state;
|
|
474
|
+
report.healthDetail = health.detail;
|
|
475
|
+
if (health.state === 'healthy') {
|
|
476
|
+
report.result = 'not-needed';
|
|
477
|
+
report.health = 'healthy';
|
|
478
|
+
report.error = 'the host answered: it is up, the snapshot was NOT restored'
|
|
479
|
+
+ (health.detail === null ? '' : ' (' + health.detail + ')');
|
|
480
|
+
return report;
|
|
481
|
+
}
|
|
482
|
+
if (health.state === 'unknown') {
|
|
483
|
+
// There was no port to ask (no instance record, no health URL). One more
|
|
484
|
+
// round is waited, and the leftovers are looked at again: a host that is
|
|
485
|
+
// still starting gets this last chance before anything is restored.
|
|
486
|
+
report.extraRound = true;
|
|
487
|
+
await d.sleep(Math.max(0, pending.recheckMs));
|
|
488
|
+
if (d.readReady()) {
|
|
489
|
+
report.result = 'not-needed';
|
|
490
|
+
report.ready = true;
|
|
491
|
+
return report;
|
|
492
|
+
}
|
|
493
|
+
if (d.isWithdrawn()) {
|
|
494
|
+
report.result = 'decided';
|
|
495
|
+
report.error = 'the plugin decided on its own: the watchdog stood down';
|
|
496
|
+
return report;
|
|
497
|
+
}
|
|
498
|
+
if (await d.probeAlive()) {
|
|
499
|
+
report.result = 'abandoned';
|
|
500
|
+
report.error = 'an instance is alive: the snapshot was not restored';
|
|
501
|
+
return report;
|
|
502
|
+
}
|
|
503
|
+
const lateHealth = healthAnswer(await d.askHealth());
|
|
504
|
+
report.healthCheck = lateHealth.state;
|
|
505
|
+
report.healthDetail = lateHealth.detail;
|
|
506
|
+
if (lateHealth.state === 'healthy') {
|
|
507
|
+
report.result = 'not-needed';
|
|
508
|
+
report.health = 'healthy';
|
|
509
|
+
report.error = 'the host answered: it is up, the snapshot was NOT restored'
|
|
510
|
+
+ (lateHealth.detail === null ? '' : ' (' + lateHealth.detail + ')');
|
|
511
|
+
return report;
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
|
|
429
515
|
// (4) The lock: exactly ONE of us restores.
|
|
430
516
|
if (!d.acquireLock()) {
|
|
431
517
|
report.result = 'abandoned';
|
package/locale/en.json
CHANGED
|
@@ -54,6 +54,7 @@
|
|
|
54
54
|
"cancelPanelAria": "Pending restart details",
|
|
55
55
|
"cancelButton": "Cancel my restart",
|
|
56
56
|
"cancelButtonAria": "Cancel my restart",
|
|
57
|
+
"cancelLocked": "Restarting — this can no longer be cancelled",
|
|
57
58
|
"cancelConfirmTitle": "Cancel your restart?",
|
|
58
59
|
"cancelConfirmOnlyMe": "After this, the restart will not happen.",
|
|
59
60
|
"cancelConfirmOthers": "{n} other session(s) are waiting for this restart too — cancelling only affects you; they will still restart.",
|
package/locale/zh.json
CHANGED
|
@@ -54,6 +54,7 @@
|
|
|
54
54
|
"cancelPanelAria": "待重启详情",
|
|
55
55
|
"cancelButton": "取消我的这次重启",
|
|
56
56
|
"cancelButtonAria": "取消我的这次重启",
|
|
57
|
+
"cancelLocked": "正在重启,已经不能取消了",
|
|
57
58
|
"cancelConfirmTitle": "取消你的这次重启?",
|
|
58
59
|
"cancelConfirmOnlyMe": "取消后这次重启就不会发生了。",
|
|
59
60
|
"cancelConfirmOthers": "还有 {n} 个会话也在等这次重启——取消只影响你自己,他们仍会重启。",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-autorestart",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.2",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "DeepSeek Harness plugin: restart without losing work. Waits until every conversation is idle, restarts the host, then automatically resumes the interrupted sessions. AI-generated, not human-reviewed: use with caution.",
|
|
6
6
|
"license": "MIT",
|
|
@@ -34,12 +34,17 @@
|
|
|
34
34
|
"supervisor/supervisor.js",
|
|
35
35
|
"scripts/*.ps1",
|
|
36
36
|
"scripts/*.cmd",
|
|
37
|
+
"scripts/*.sh",
|
|
38
|
+
"scripts/*.mjs",
|
|
39
|
+
"scripts/README.md",
|
|
40
|
+
"scripts/rescue-strings.zh.json",
|
|
37
41
|
"cordis.patch.yml",
|
|
38
42
|
"locale/*.json",
|
|
39
43
|
"icon.svg",
|
|
40
44
|
"assets/*.png",
|
|
41
45
|
"screenshots.json",
|
|
42
46
|
"README.md",
|
|
47
|
+
"CHANGELOG.md",
|
|
43
48
|
"docs/*.md"
|
|
44
49
|
],
|
|
45
50
|
"dsh": {
|