dsh-autorestart 0.7.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/README.md +137 -6
- package/client.js +387 -0
- package/cordis.patch.yml +7 -0
- package/docs/README.zh.md +83 -5
- package/docs/platform-matrix.md +92 -0
- package/docs/rescue-drill.md +606 -0
- package/index.js +598 -3
- package/lib/diagnostics.js +8 -0
- package/lib/rescue-paths.js +253 -0
- package/lib/rescue.js +596 -0
- package/lib/scheduler-capability.js +300 -0
- package/lib/watchdog-arbiter.js +117 -0
- package/lib/watchdog-schedule.js +310 -0
- package/lib/watchdog-tier.js +182 -0
- package/lib/watchdog.js +479 -0
- package/locale/en.json +28 -1
- package/locale/zh.json +28 -1
- package/package.json +6 -1
- package/scripts/README.md +65 -0
- package/scripts/drill-rescue.ps1 +508 -0
- package/scripts/drill-rescue.sh +427 -0
- package/scripts/drill-watchdog.mjs +341 -0
- package/scripts/emergency-disable.sh +314 -0
- package/scripts/emergency-toggle.ps1 +136 -65
- package/scripts/rescue-strings.zh.json +168 -0
- package/scripts/restore-profile.cmd +44 -0
- package/scripts/restore-profile.ps1 +608 -0
- package/scripts/restore-profile.sh +593 -0
- package/scripts/run-tests.ps1 +1 -1
- package/scripts/watchdog.mjs +203 -0
package/lib/watchdog.js
ADDED
|
@@ -0,0 +1,479 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* THE WATCHDOG — one flow, three ways to trigger it.
|
|
3
|
+
*
|
|
4
|
+
* The flow is the same everywhere; only the way the watchdog is made to run
|
|
5
|
+
* differs (see lib/watchdog-tier.js and lib/watchdog-schedule.js). This file
|
|
6
|
+
* is that flow:
|
|
7
|
+
*
|
|
8
|
+
* 1. wait for the READY marker (the new host wrote it after its health
|
|
9
|
+
* check passed);
|
|
10
|
+
* 2. no marker? Wait a bit and ASK AGAIN — a host that is slow to start is
|
|
11
|
+
* not a host that is dead (the misjudgement protection);
|
|
12
|
+
* 3. still nothing? Is an instance ALIVE? If one is listening, the rollback
|
|
13
|
+
* is not needed: stand down (the no-double-start protection);
|
|
14
|
+
* 4. take the shared lock (`rollback.lock`) — exactly ONE of us restores;
|
|
15
|
+
* 5. restore the snapshot, then START THE HOST AGAIN;
|
|
16
|
+
* 6. wait for the ready marker once more: it came → `recovered`, it did not
|
|
17
|
+
* → `failed` — and then we STOP (at most one restore and one start).
|
|
18
|
+
*
|
|
19
|
+
* Whatever happens, the scheduling item and the pending marker go; the report
|
|
20
|
+
* stays.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import fs from 'node:fs';
|
|
24
|
+
import path from 'node:path';
|
|
25
|
+
|
|
26
|
+
/** The marker that says "a restart is coming, a watchdog is watching". */
|
|
27
|
+
export const PENDING_NAME = 'watchdog-pending.json';
|
|
28
|
+
/** The marker that says "the new host is up and healthy". */
|
|
29
|
+
export const READY_NAME = 'watchdog-ready.json';
|
|
30
|
+
/** The report of the last watchdog run. */
|
|
31
|
+
export const REPORT_NAME = 'last-watchdog-report.json';
|
|
32
|
+
/** The lock that both the plugin and the watchdog take before a restore. */
|
|
33
|
+
export const LOCK_NAME = 'rollback.lock';
|
|
34
|
+
|
|
35
|
+
/** How long the watchdog waits for the new host. */
|
|
36
|
+
export const DEFAULT_TIMEOUT_MS = 180000;
|
|
37
|
+
/** How long it waits before it asks again (the misjudgement protection). */
|
|
38
|
+
export const DEFAULT_RECHECK_MS = 20000;
|
|
39
|
+
/** How often it looks at the ready marker. */
|
|
40
|
+
export const DEFAULT_POLL_MS = 2000;
|
|
41
|
+
/** How long it waits for the host after the restore. */
|
|
42
|
+
export const DEFAULT_READY_TIMEOUT_MS = 90000;
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Write a JSON file (no BOM, so every reader can parse it).
|
|
46
|
+
*
|
|
47
|
+
* @param {string} file
|
|
48
|
+
* @param {object} value
|
|
49
|
+
*/
|
|
50
|
+
function writeJson(file, value) {
|
|
51
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
52
|
+
fs.writeFileSync(file, JSON.stringify(value, null, 2), 'utf8');
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Read a JSON file (a file that cannot be read is a file that is not there).
|
|
57
|
+
*
|
|
58
|
+
* @param {string} file
|
|
59
|
+
* @returns {object|null}
|
|
60
|
+
*/
|
|
61
|
+
function readJson(file) {
|
|
62
|
+
try {
|
|
63
|
+
const parsed = JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
64
|
+
return parsed && typeof parsed === 'object' ? parsed : null;
|
|
65
|
+
} catch {
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* The pending marker: what the watchdog has to know.
|
|
72
|
+
*
|
|
73
|
+
* @param {{
|
|
74
|
+
* tier?: string,
|
|
75
|
+
* tierReason?: string,
|
|
76
|
+
* taskName?: string,
|
|
77
|
+
* unitId?: string,
|
|
78
|
+
* pidPath?: string,
|
|
79
|
+
* snapshotDir?: string,
|
|
80
|
+
* profileDir?: string,
|
|
81
|
+
* restoreCommand?: object|null,
|
|
82
|
+
* relaunchCommand?: object|null,
|
|
83
|
+
* healthUrl?: string,
|
|
84
|
+
* healthPath?: string,
|
|
85
|
+
* portFile?: string,
|
|
86
|
+
* expectedBootId?: string|null,
|
|
87
|
+
* timeoutMs?: number,
|
|
88
|
+
* recheckMs?: number,
|
|
89
|
+
* readyTimeoutMs?: number,
|
|
90
|
+
* hostKind?: string,
|
|
91
|
+
* attempts?: number,
|
|
92
|
+
* now?: number,
|
|
93
|
+
* }} options
|
|
94
|
+
* @returns {object}
|
|
95
|
+
*/
|
|
96
|
+
export function buildPending({
|
|
97
|
+
tier = 'none',
|
|
98
|
+
tierReason = '',
|
|
99
|
+
arbiter = 'v2',
|
|
100
|
+
taskName = '',
|
|
101
|
+
unitId = '',
|
|
102
|
+
pidPath = '',
|
|
103
|
+
snapshotDir = '',
|
|
104
|
+
profileDir = '',
|
|
105
|
+
restoreCommand = null,
|
|
106
|
+
relaunchCommand = null,
|
|
107
|
+
healthUrl = '',
|
|
108
|
+
healthPath = '/__dsh_autorestart/health',
|
|
109
|
+
portFile = '',
|
|
110
|
+
expectedBootId = null,
|
|
111
|
+
timeoutMs = DEFAULT_TIMEOUT_MS,
|
|
112
|
+
recheckMs = DEFAULT_RECHECK_MS,
|
|
113
|
+
readyTimeoutMs = DEFAULT_READY_TIMEOUT_MS,
|
|
114
|
+
hostKind = '',
|
|
115
|
+
attempts = 1,
|
|
116
|
+
now = Date.now(),
|
|
117
|
+
} = {}) {
|
|
118
|
+
return {
|
|
119
|
+
version: 1,
|
|
120
|
+
at: new Date(now).toISOString(),
|
|
121
|
+
tier,
|
|
122
|
+
tierReason,
|
|
123
|
+
arbiter,
|
|
124
|
+
taskName,
|
|
125
|
+
unitId,
|
|
126
|
+
pidPath,
|
|
127
|
+
snapshotDir,
|
|
128
|
+
profileDir,
|
|
129
|
+
restoreCommand,
|
|
130
|
+
relaunchCommand,
|
|
131
|
+
healthUrl,
|
|
132
|
+
healthPath,
|
|
133
|
+
portFile,
|
|
134
|
+
expectedBootId,
|
|
135
|
+
timeoutMs,
|
|
136
|
+
recheckMs,
|
|
137
|
+
readyTimeoutMs,
|
|
138
|
+
hostKind,
|
|
139
|
+
attempts,
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* @param {string} stateDir
|
|
145
|
+
* @returns {string}
|
|
146
|
+
*/
|
|
147
|
+
export function pendingPath(stateDir) {
|
|
148
|
+
return path.join(stateDir, PENDING_NAME);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* @param {string} stateDir
|
|
153
|
+
* @returns {string}
|
|
154
|
+
*/
|
|
155
|
+
export function readyPath(stateDir) {
|
|
156
|
+
return path.join(stateDir, READY_NAME);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* @param {string} stateDir
|
|
161
|
+
* @returns {string}
|
|
162
|
+
*/
|
|
163
|
+
export function reportPath(stateDir) {
|
|
164
|
+
return path.join(stateDir, REPORT_NAME);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Write the pending marker.
|
|
169
|
+
*
|
|
170
|
+
* @param {string} stateDir
|
|
171
|
+
* @param {object} pending
|
|
172
|
+
*/
|
|
173
|
+
export function writePending(stateDir, pending) {
|
|
174
|
+
writeJson(pendingPath(stateDir), pending);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* @param {string} stateDir
|
|
179
|
+
* @returns {object|null}
|
|
180
|
+
*/
|
|
181
|
+
export function readPending(stateDir) {
|
|
182
|
+
return readJson(pendingPath(stateDir));
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* Remove the pending marker (the watchdog is over).
|
|
187
|
+
*
|
|
188
|
+
* @param {string} stateDir
|
|
189
|
+
*/
|
|
190
|
+
export function clearPending(stateDir) {
|
|
191
|
+
try {
|
|
192
|
+
fs.rmSync(pendingPath(stateDir), { force: true });
|
|
193
|
+
} catch {
|
|
194
|
+
// It is gone either way.
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/**
|
|
199
|
+
* Write the ready marker: "the new host is up".
|
|
200
|
+
*
|
|
201
|
+
* @param {string} stateDir
|
|
202
|
+
* @param {{ bootId?: string, port?: number, at?: number }} [info]
|
|
203
|
+
*/
|
|
204
|
+
export function writeReady(stateDir, info = {}) {
|
|
205
|
+
writeJson(readyPath(stateDir), {
|
|
206
|
+
at: new Date(info.at ?? Date.now()).toISOString(),
|
|
207
|
+
bootId: info.bootId ?? null,
|
|
208
|
+
port: info.port ?? null,
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* @param {string} stateDir
|
|
214
|
+
* @returns {object|null}
|
|
215
|
+
*/
|
|
216
|
+
export function readReady(stateDir) {
|
|
217
|
+
return readJson(readyPath(stateDir));
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Remove the ready marker (it belongs to one restart).
|
|
222
|
+
*
|
|
223
|
+
* @param {string} stateDir
|
|
224
|
+
*/
|
|
225
|
+
export function clearReady(stateDir) {
|
|
226
|
+
try {
|
|
227
|
+
fs.rmSync(readyPath(stateDir), { force: true });
|
|
228
|
+
} catch {
|
|
229
|
+
// It is gone either way.
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* Write the report of the last watchdog run.
|
|
235
|
+
*
|
|
236
|
+
* @param {string} stateDir
|
|
237
|
+
* @param {object} report
|
|
238
|
+
*/
|
|
239
|
+
export function writeReport(stateDir, report) {
|
|
240
|
+
writeJson(reportPath(stateDir), report);
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* @param {string} stateDir
|
|
245
|
+
* @returns {object|null}
|
|
246
|
+
*/
|
|
247
|
+
export function readReport(stateDir) {
|
|
248
|
+
return readJson(reportPath(stateDir));
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* Take the lock (exactly one restore at a time).
|
|
253
|
+
*
|
|
254
|
+
* @param {string} lockFile
|
|
255
|
+
* @param {{ now?: number, exists?: Function, mkdir?: Function, open?: Function }} [deps]
|
|
256
|
+
* @returns {boolean}
|
|
257
|
+
*/
|
|
258
|
+
export function acquireLock(lockFile, { now = Date.now(), exists = fs.existsSync, mkdir = (dir) => fs.mkdirSync(dir, { recursive: true }), open = null } = {}) {
|
|
259
|
+
try {
|
|
260
|
+
mkdir(path.dirname(lockFile));
|
|
261
|
+
if (exists(lockFile)) {
|
|
262
|
+
// A lock that is older than 10 minutes belongs to a process that died
|
|
263
|
+
// in the middle of its restore: it must not block the rollback for ever.
|
|
264
|
+
let age = 0;
|
|
265
|
+
try {
|
|
266
|
+
age = now - fs.statSync(lockFile).mtimeMs;
|
|
267
|
+
} catch {
|
|
268
|
+
age = 0;
|
|
269
|
+
}
|
|
270
|
+
if (age > 10 * 60 * 1000) fs.rmSync(lockFile, { force: true });
|
|
271
|
+
else return false;
|
|
272
|
+
}
|
|
273
|
+
const handle = open === null
|
|
274
|
+
? fs.openSync(lockFile, 'wx')
|
|
275
|
+
: open(lockFile);
|
|
276
|
+
if (handle !== null && handle !== undefined && typeof handle.close === 'function') handle.close();
|
|
277
|
+
return true;
|
|
278
|
+
} catch {
|
|
279
|
+
return false;
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/**
|
|
284
|
+
* Give the lock back.
|
|
285
|
+
*
|
|
286
|
+
* @param {string} lockFile
|
|
287
|
+
*/
|
|
288
|
+
export function releaseLock(lockFile) {
|
|
289
|
+
try {
|
|
290
|
+
fs.rmSync(lockFile, { force: true });
|
|
291
|
+
} catch {
|
|
292
|
+
// It is gone either way.
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* Put the snapshot back.
|
|
298
|
+
*
|
|
299
|
+
* The same for every platform: the configuration files and the plugin tree
|
|
300
|
+
* are copied out of the snapshot into the profile. (The profile is NOT
|
|
301
|
+
* deleted first — the broken files are overwritten, so a half-restored
|
|
302
|
+
* profile is never worse than the broken one.)
|
|
303
|
+
*
|
|
304
|
+
* @param {{ snapshotDir?: string, profileDir?: string, copy?: Function }} options
|
|
305
|
+
* @returns {{ ok: boolean, error: string|null, copied: string[] }}
|
|
306
|
+
*/
|
|
307
|
+
export function restoreFromSnapshot({ snapshotDir = '', profileDir = '', copy = null } = {}) {
|
|
308
|
+
const from = String(snapshotDir ?? '');
|
|
309
|
+
const into = String(profileDir ?? '');
|
|
310
|
+
if (from === '' || into === '') {
|
|
311
|
+
return { ok: false, error: 'the snapshot or the profile directory is unknown', copied: [] };
|
|
312
|
+
}
|
|
313
|
+
const copied = [];
|
|
314
|
+
const copyInto = copy === null
|
|
315
|
+
? (source, target) => { fs.cpSync(source, target, { recursive: true, force: true }); }
|
|
316
|
+
: copy;
|
|
317
|
+
try {
|
|
318
|
+
for (const [part, target] of [['config', into], ['node_modules', path.join(into, 'node_modules')]]) {
|
|
319
|
+
const source = path.join(from, part);
|
|
320
|
+
if (!fs.existsSync(source)) continue;
|
|
321
|
+
copyInto(source, target);
|
|
322
|
+
copied.push(part);
|
|
323
|
+
}
|
|
324
|
+
} catch (error) {
|
|
325
|
+
return {
|
|
326
|
+
ok: false,
|
|
327
|
+
error: error && error.message ? error.message : String(error),
|
|
328
|
+
copied,
|
|
329
|
+
};
|
|
330
|
+
}
|
|
331
|
+
return { ok: copied.length > 0, error: copied.length > 0 ? null : 'the snapshot carries nothing to restore', copied };
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
/**
|
|
335
|
+
* Wait for the ready marker.
|
|
336
|
+
*
|
|
337
|
+
* @param {object} deps
|
|
338
|
+
* @param {number} timeoutMs
|
|
339
|
+
* @returns {Promise<boolean>}
|
|
340
|
+
*/
|
|
341
|
+
async function waitForReady(deps, timeoutMs) {
|
|
342
|
+
const deadline = deps.now() + Math.max(0, timeoutMs);
|
|
343
|
+
while (deps.now() < deadline) {
|
|
344
|
+
if (deps.readReady()) return true;
|
|
345
|
+
// The plugin may have decided on its own (the restart failed): stand down.
|
|
346
|
+
if (deps.isWithdrawn && deps.isWithdrawn()) return false;
|
|
347
|
+
await deps.sleep(Math.min(deps.pollMs, Math.max(0, deadline - deps.now())));
|
|
348
|
+
}
|
|
349
|
+
return deps.readReady() === true;
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* THE FLOW.
|
|
354
|
+
*
|
|
355
|
+
* @param {{
|
|
356
|
+
* pending: object,
|
|
357
|
+
* stateDir?: string,
|
|
358
|
+
* deps?: object,
|
|
359
|
+
* }} options
|
|
360
|
+
* @returns {Promise<object>} the report
|
|
361
|
+
*/
|
|
362
|
+
export async function runWatchdogFlow({ pending, stateDir = '', deps = {} }) {
|
|
363
|
+
const d = {
|
|
364
|
+
sleep: async (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
365
|
+
now: () => Date.now(),
|
|
366
|
+
pollMs: DEFAULT_POLL_MS,
|
|
367
|
+
readReady: () => readReady(stateDir),
|
|
368
|
+
isWithdrawn: () => readPending(stateDir) === null,
|
|
369
|
+
probeAlive: () => false,
|
|
370
|
+
runRestore: () => ({ ok: false, error: 'no restore command' }),
|
|
371
|
+
runRelaunch: () => ({ ok: false, error: 'no relaunch command' }),
|
|
372
|
+
acquireLock: () => true,
|
|
373
|
+
releaseLock: () => {},
|
|
374
|
+
removeSchedule: () => {},
|
|
375
|
+
writeReport: (report) => writeReport(stateDir, report),
|
|
376
|
+
...deps,
|
|
377
|
+
};
|
|
378
|
+
const startedAt = d.now();
|
|
379
|
+
const report = {
|
|
380
|
+
at: new Date(startedAt).toISOString(),
|
|
381
|
+
result: 'unknown',
|
|
382
|
+
tier: pending && pending.tier ? pending.tier : 'none',
|
|
383
|
+
tierReason: pending && pending.tierReason ? pending.tierReason : '',
|
|
384
|
+
// WHO watched this restart. There is only ever one watchdog at a time
|
|
385
|
+
// (see lib/watchdog-arbiter.js), and the report says which one it was.
|
|
386
|
+
arbiter: pending && pending.arbiter ? pending.arbiter : 'v2',
|
|
387
|
+
waitedMs: 0,
|
|
388
|
+
rechecked: false,
|
|
389
|
+
restored: false,
|
|
390
|
+
relaunched: false,
|
|
391
|
+
ready: false,
|
|
392
|
+
error: null,
|
|
393
|
+
};
|
|
394
|
+
try {
|
|
395
|
+
// (1) The new host writes the ready marker after its health check.
|
|
396
|
+
const ready = await waitForReady(d, pending.timeoutMs);
|
|
397
|
+
if (ready) {
|
|
398
|
+
report.result = 'not-needed';
|
|
399
|
+
report.ready = true;
|
|
400
|
+
return report;
|
|
401
|
+
}
|
|
402
|
+
if (d.isWithdrawn()) {
|
|
403
|
+
report.result = 'decided';
|
|
404
|
+
report.error = 'the plugin decided on its own: the watchdog stood down';
|
|
405
|
+
return report;
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
// (2) ASK AGAIN. A host that is slow to start is not a host that is dead.
|
|
409
|
+
report.rechecked = true;
|
|
410
|
+
await d.sleep(Math.max(0, pending.recheckMs));
|
|
411
|
+
if (d.readReady()) {
|
|
412
|
+
report.result = 'not-needed';
|
|
413
|
+
report.ready = true;
|
|
414
|
+
return report;
|
|
415
|
+
}
|
|
416
|
+
if (d.isWithdrawn()) {
|
|
417
|
+
report.result = 'decided';
|
|
418
|
+
report.error = 'the plugin decided on its own: the watchdog stood down';
|
|
419
|
+
return report;
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
// (3) Is an instance ALIVE? Then there is nothing to restore.
|
|
423
|
+
if (await d.probeAlive()) {
|
|
424
|
+
report.result = 'abandoned';
|
|
425
|
+
report.error = 'an instance is alive: the snapshot was not restored';
|
|
426
|
+
return report;
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
// (4) The lock: exactly ONE of us restores.
|
|
430
|
+
if (!d.acquireLock()) {
|
|
431
|
+
report.result = 'abandoned';
|
|
432
|
+
report.error = 'another rollback is already running';
|
|
433
|
+
return report;
|
|
434
|
+
}
|
|
435
|
+
try {
|
|
436
|
+
// (5) Restore the snapshot.
|
|
437
|
+
const restored = d.runRestore();
|
|
438
|
+
report.restored = restored !== null && restored !== undefined && restored.ok === true;
|
|
439
|
+
if (!report.restored) {
|
|
440
|
+
report.result = 'failed';
|
|
441
|
+
report.error = restored && restored.error ? restored.error : 'the restore failed';
|
|
442
|
+
return report;
|
|
443
|
+
}
|
|
444
|
+
// (6) Start the host again.
|
|
445
|
+
const relaunched = d.runRelaunch();
|
|
446
|
+
report.relaunched = relaunched !== null && relaunched !== undefined && relaunched.ok === true;
|
|
447
|
+
if (!report.relaunched) {
|
|
448
|
+
report.result = 'failed';
|
|
449
|
+
report.error = relaunched && relaunched.error ? relaunched.error : 'the host could not be started again';
|
|
450
|
+
return report;
|
|
451
|
+
}
|
|
452
|
+
// (7) Did it come up this time?
|
|
453
|
+
const readyAgain = await waitForReady(d, pending.readyTimeoutMs);
|
|
454
|
+
report.ready = readyAgain;
|
|
455
|
+
report.result = readyAgain ? 'recovered' : 'failed';
|
|
456
|
+
if (!readyAgain) report.error = 'the host did not come up after the restore';
|
|
457
|
+
} finally {
|
|
458
|
+
d.releaseLock();
|
|
459
|
+
}
|
|
460
|
+
} catch (error) {
|
|
461
|
+
report.result = 'failed';
|
|
462
|
+
report.error = error && error.message ? error.message : String(error);
|
|
463
|
+
} finally {
|
|
464
|
+
report.waitedMs = d.now() - startedAt;
|
|
465
|
+
// The scheduling item and the pending marker go; the report stays.
|
|
466
|
+
try {
|
|
467
|
+
d.removeSchedule();
|
|
468
|
+
} catch {
|
|
469
|
+
// The scheduling item is cleaned up by the next run if this fails.
|
|
470
|
+
}
|
|
471
|
+
try {
|
|
472
|
+
if (stateDir !== '') clearPending(stateDir);
|
|
473
|
+
} catch {
|
|
474
|
+
// It is gone either way.
|
|
475
|
+
}
|
|
476
|
+
d.writeReport(report);
|
|
477
|
+
}
|
|
478
|
+
return report;
|
|
479
|
+
}
|
package/locale/en.json
CHANGED
|
@@ -174,7 +174,8 @@
|
|
|
174
174
|
"restart": "Restart settings",
|
|
175
175
|
"shutdown": "Shutdown settings",
|
|
176
176
|
"supervisor": "Start at login",
|
|
177
|
-
"report": "Report a problem"
|
|
177
|
+
"report": "Report a problem",
|
|
178
|
+
"rescue": "If DSH will not start"
|
|
178
179
|
},
|
|
179
180
|
"seconds": {
|
|
180
181
|
"label": "Shutdown countdown seconds",
|
|
@@ -270,6 +271,32 @@
|
|
|
270
271
|
"issueTitle": "A problem with the restart / start at login",
|
|
271
272
|
"issueBody": "Please paste the diagnostics you just copied below.\n\n## What happened\n\n\n## What you expected\n\n\n## How to reproduce\n"
|
|
272
273
|
},
|
|
274
|
+
"rescue": {
|
|
275
|
+
"hint": "When DSH does not come up, these two scripts rescue it for you (neither of them needs DSH running).",
|
|
276
|
+
"restoreProfile": "Back to the last good state",
|
|
277
|
+
"emergencyDisable": "Take this plugin out only",
|
|
278
|
+
"path": "Path",
|
|
279
|
+
"copyPath": "Copy the path",
|
|
280
|
+
"copied": "The path is on the clipboard",
|
|
281
|
+
"copyFailed": "The copy failed — please select the path above by hand",
|
|
282
|
+
"reveal": "Open the folder",
|
|
283
|
+
"which": "Which one: the whole environment is broken (a changed configuration, another plugin, a bad upgrade) → \"Back to the last good state\"; only this plugin is suspect → \"Take this plugin out only\".",
|
|
284
|
+
"warning": "Important: these two scripts are for when DSH does not come up. Using \"Back to the last good state\" while DSH is running overwrites the configuration you are working with right now.",
|
|
285
|
+
"shortcut": "Put a shortcut on the desktop",
|
|
286
|
+
"shortcutDone": "It is on the desktop",
|
|
287
|
+
"noDesktop": "This machine has no desktop directory, so no shortcut can be placed (the paths are above — copy them by hand).",
|
|
288
|
+
"shortcutFailed": "The shortcut was not placed: {message}",
|
|
289
|
+
"revealFailed": "The folder did not open: {message}",
|
|
290
|
+
"rollbackOk": "Automatic rollback: available (a host that does not come up after a restart goes back to the last good state)",
|
|
291
|
+
"rollbackBlocked": "Automatic rollback: **not available** — this computer does not allow creating scheduled tasks. **What you lose: DSH will NOT recover by itself when it does not come up after a restart**; when it breaks, double-click the restore script in \"If DSH will not start\".",
|
|
292
|
+
"rollbackUnknown": "Automatic rollback: not known yet (this machine has not been checked)",
|
|
293
|
+
"watchdogTier": "How the watchdog is triggered: {tier}",
|
|
294
|
+
"watchdogLast": "The last watchdog run: {result}",
|
|
295
|
+
"tierScheduledTask": "a Windows scheduled task",
|
|
296
|
+
"tierSystemdRun": "a systemd timer",
|
|
297
|
+
"tierDetachedProcess": "a detached background process",
|
|
298
|
+
"tierNone": "(this machine has no way to run it)"
|
|
299
|
+
},
|
|
273
300
|
"error": "Could not save the settings: {message}",
|
|
274
301
|
"autostartKind": {
|
|
275
302
|
"other": "something else",
|
package/locale/zh.json
CHANGED
|
@@ -174,7 +174,8 @@
|
|
|
174
174
|
"restart": "重启相关设置",
|
|
175
175
|
"shutdown": "关机相关设置",
|
|
176
176
|
"supervisor": "开机自启",
|
|
177
|
-
"report": "报告问题"
|
|
177
|
+
"report": "报告问题",
|
|
178
|
+
"rescue": "出事了怎么办"
|
|
178
179
|
},
|
|
179
180
|
"seconds": {
|
|
180
181
|
"label": "关机倒数秒数",
|
|
@@ -270,6 +271,32 @@
|
|
|
270
271
|
"issueTitle": "重启 / 开机自启相关的问题",
|
|
271
272
|
"issueBody": "请把刚才复制的诊断信息粘贴到下面。\n\n## 发生了什么\n\n\n## 期望什么\n\n\n## 怎么复现\n"
|
|
272
273
|
},
|
|
274
|
+
"rescue": {
|
|
275
|
+
"hint": "DSH 打不开的时候,用下面这两个脚本救自己(都不用 DSH 跑起来)。",
|
|
276
|
+
"restoreProfile": "回到上一次好状态",
|
|
277
|
+
"emergencyDisable": "只把这个插件摘掉",
|
|
278
|
+
"path": "路径",
|
|
279
|
+
"copyPath": "复制路径",
|
|
280
|
+
"copied": "路径已复制",
|
|
281
|
+
"copyFailed": "复制失败,请手动选中上面的路径",
|
|
282
|
+
"reveal": "打开所在文件夹",
|
|
283
|
+
"which": "怎么选:整个环境坏了(改了配置、装了别的插件、升级崩了)→ 用「回到上一次好状态」;只怀疑这个插件 → 用「只把这个插件摘掉」。",
|
|
284
|
+
"warning": "重要:这两个脚本是在 DSH 打不开的时候用的。DSH 正开着的时候用「回到上一次好状态」,会把你现在正在用的配置覆盖掉。",
|
|
285
|
+
"shortcut": "在桌面放一个快捷方式",
|
|
286
|
+
"shortcutDone": "已在桌面",
|
|
287
|
+
"noDesktop": "这台机器上没有桌面目录,所以放不了快捷方式(脚本的路径在上面,可以手动复制过去)。",
|
|
288
|
+
"shortcutFailed": "快捷方式没放成:{message}",
|
|
289
|
+
"revealFailed": "文件夹没打开:{message}",
|
|
290
|
+
"rollbackOk": "自动回滚:可用(重启后起不来会自动回到上一个好状态)",
|
|
291
|
+
"rollbackBlocked": "自动回滚:**不可用** —— 这台电脑不允许创建计划任务。**会丢失:重启后 DSH 起不来时不会自动恢复**;出问题时请双击「出事了怎么办」里的还原脚本。",
|
|
292
|
+
"rollbackUnknown": "自动回滚:还不知道(这台机器的能力还没检查过)",
|
|
293
|
+
"watchdogTier": "看门狗的触发方式:{tier}",
|
|
294
|
+
"watchdogLast": "上一次看门狗的结果:{result}",
|
|
295
|
+
"tierScheduledTask": "Windows 计划任务",
|
|
296
|
+
"tierSystemdRun": "systemd 定时器",
|
|
297
|
+
"tierDetachedProcess": "自建后台进程",
|
|
298
|
+
"tierNone": "(这台机器没有可用机制)"
|
|
299
|
+
},
|
|
273
300
|
"error": "没能保存设置:{message}",
|
|
274
301
|
"autostartKind": {
|
|
275
302
|
"other": "其他方式",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-autorestart",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.8.1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "DeepSeek Harness plugin: restart without losing work. Waits until every conversation is idle, restarts the host, then automatically resumes the interrupted sessions. AI-generated, not human-reviewed: use with caution.",
|
|
6
6
|
"license": "MIT",
|
|
@@ -34,12 +34,17 @@
|
|
|
34
34
|
"supervisor/supervisor.js",
|
|
35
35
|
"scripts/*.ps1",
|
|
36
36
|
"scripts/*.cmd",
|
|
37
|
+
"scripts/*.sh",
|
|
38
|
+
"scripts/*.mjs",
|
|
39
|
+
"scripts/README.md",
|
|
40
|
+
"scripts/rescue-strings.zh.json",
|
|
37
41
|
"cordis.patch.yml",
|
|
38
42
|
"locale/*.json",
|
|
39
43
|
"icon.svg",
|
|
40
44
|
"assets/*.png",
|
|
41
45
|
"screenshots.json",
|
|
42
46
|
"README.md",
|
|
47
|
+
"CHANGELOG.md",
|
|
43
48
|
"docs/*.md"
|
|
44
49
|
],
|
|
45
50
|
"dsh": {
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# 自救包(scripts)
|
|
2
|
+
|
|
3
|
+
DSH 起不来的时候,这个文件夹里的东西**不用 DSH 自己**就能救自己。
|
|
4
|
+
|
|
5
|
+
## 按平台挑文件(**别拿错平台的**)
|
|
6
|
+
|
|
7
|
+
| 你的机器 | 还原 profile | 摘掉本插件 |
|
|
8
|
+
|---|---|---|
|
|
9
|
+
| **Windows**(桌面客户端 / Windows 上的 `dsh web`) | 双击 `restore-profile.cmd` | 双击 `emergency-disable.cmd` |
|
|
10
|
+
| **Linux / WSL / macOS**(托管端) | `sh scripts/restore-profile.sh` | `sh scripts/emergency-disable.sh` |
|
|
11
|
+
|
|
12
|
+
Linux/WSL 的机器上**没有** `.cmd` 文件,Windows 的机器上**没有** `.sh` —— 设置页的
|
|
13
|
+
「出事了怎么办」只显示**这台机器上真有的**那两个路径,不会给你一条走不通的死路。
|
|
14
|
+
每个平台到底有什么能力,看 [平台能力矩阵](../docs/platform-matrix.md)。
|
|
15
|
+
|
|
16
|
+
- **双击哪个**:DSH 起不来(插件升级把 profile 弄坏了)→ 还原;只是想把本插件摘掉 → 摘插件。
|
|
17
|
+
- **找不到它们?**:设置页里有一组「出事了怎么办」,上面写着这两个脚本的**绝对路径**(可以一键复制、可以一键打开所在文件夹;Windows 上还能放一个桌面快捷方式)—— DSH 真打不开、连 README 都打不开的时候,那一栏就是给你指路的。
|
|
18
|
+
- **它会做什么**:`restore-profile.cmd` / `restore-profile.sh` 把最新的那份快照(`<stateDir>/backups/` 里带 manifest 的)放回 profile —— 先把现在的 profile 整份挪到 `<profile>.broken-<时间戳>`,再复制快照里的配置和插件树,最后回读校验(package.json 能解析、`dsh.profile.bundles` 有内容、插件版本号读得到);校验不过就如实报错,并告诉你原来那份还在哪个目录。**它不会替你重启 DSH**(重启由你自己来,免得"救一次挂一次")。
|
|
19
|
+
- **会留下什么**:一个 `<profile>.broken-<yyyyMMdd-HHmmss>` 目录(出事之前的那份 profile 原封不动地留着,确认没问题后可以自己删掉),加上最后一行"已还原到 …… 现在可以重新打开 DSH"的结论。
|
|
20
|
+
|
|
21
|
+
## 两条硬规矩(有意收紧,不是 bug)
|
|
22
|
+
|
|
23
|
+
- **目标 profile 目录必须已经存在**:还原器把快照放回一个**已经存在**的 profile,**不会凭空新建**一个 profile(也不会替你建 `profiles` 目录)。路径写错的时候,宁可报错,也不留一个空壳 profile 在那儿。
|
|
24
|
+
- **插件版本号的校验看配置说话**:profile 的 `dsh.profile.bundles` 里**没有** `dsh-autorestart` → 这一项**跳过**;**声明了**、但 `node_modules/dsh-autorestart` 不在 → **必须失败**(配置说一套、磁盘是另一套,这是真坏,不能放过)。
|
|
25
|
+
|
|
26
|
+
## 演练(不碰你正在用的 profile)
|
|
27
|
+
|
|
28
|
+
| 平台 | 命令 |
|
|
29
|
+
|---|---|
|
|
30
|
+
| Windows | `powershell -NoProfile -ExecutionPolicy Bypass -File .\drill-rescue.ps1` |
|
|
31
|
+
| Linux / WSL | `sh scripts/drill-rescue.sh` |
|
|
32
|
+
|
|
33
|
+
两个演练是**同一套断言**(Windows 的 `.ps1` 与 POSIX 的 `.sh`):在临时目录里造一个假 profile → 故意把 `package.json` 写坏、删掉插件目录 → 调还原脚本救回来 → 断言(能解析、插件版本读得到、`broken-*` 目录存在、非 profile 目录被拒绝、目标目录不存在时被拒绝、"配置说装了却没装"时必须失败、**还原脚本的输出里真的有中文**)→ 清理临时目录。全绿时退出码是 0。
|
|
34
|
+
|
|
35
|
+
演练的目录纪律(**并发跑也互不干扰**):每次跑用一个带完整 GUID 的**唯一**目录,目录由本次演练自己创建(已存在就报错退出),目录里放一个独占的锁文件 `.lock`;清理**只删自己那一个绝对路径**,**绝不按前缀/通配符扫临时目录** —— 那会踩到同时在跑的另一个演练。成功、断言不过、抛错中断,走的都是同一个 `finally`,一定会清理;清理不干净就把残留路径打印出来。
|
|
36
|
+
|
|
37
|
+
`restore-profile.ps1` 的参数:`-DryRun`(只打印将要做什么)、`-Snapshot <目录>`(指定某一份快照)、`-ProfileDir <路径>`(演练用的隔离目录)、`-StateDir <路径>`(插件状态目录,默认 `<DSH_HOME>/storages/dsh-autorestart`)。
|
|
38
|
+
`restore-profile.sh` 的参数:`--dry-run`、`--snapshot <目录>`、`--profile <路径>`、`--state-dir <路径>`。`DSH_HOME` 可以用环境变量覆盖,但目标 profile 目录必须已经在那儿。
|
|
39
|
+
|
|
40
|
+
> 编码规矩(为什么脚本是英文的):这台机器只有 Windows PowerShell 5.1,它按 ANSI 读**不带 BOM 的 UTF-8** —— 脚本里只要有一个中文字符,整份文件就变乱码、直接 ParserError,脚本连第一步都走不到。POSIX 那边同理(`/bin/sh` 也按字节读,而且中文注释会让不同的 shell 表现不一致)。这个坑一天之内踩了三次,所以现在是**根治**而不是"记得加 BOM":**所有 `.ps1` / `.sh` / `.cmd` / `.mjs` 必须是纯 ASCII(所有字节 < 128、不带 BOM)**。面向用户的中文全部放在**一个文件**里:`scripts/rescue-strings.zh.json`(UTF-8、**不带 BOM**,带 BOM 会让 `JSON.parse` 直接失败),脚本运行时读它;读不到或读坏了,脚本会用**英文兜底**继续跑,并明确打印一行英文说明(不会崩成乱码)。
|
|
41
|
+
>
|
|
42
|
+
> **以后要改文案,改 `rescue-strings.zh.json`,不要往脚本里塞中文。** 键名是语义化的(`restore.title`、`restore.brokenKept`、`drill.passed`、`emergency.failed` …),值里可以用 `{0}`、`{1}` 这类占位符。这条规矩由 `test/scripts-encoding.test.mjs` 天天盯着:它会扫目录(不是写死文件名)、断言每个 `.ps1` / `.sh` / `.mjs` 都是纯 ASCII、断言 `.sh` 以 `#!/bin/sh` 开头并开着 `set -eu`、断言 JSON 无 BOM 且能被 `JSON.parse`、断言每个脚本里用到的键在 JSON 里都存在、断言 `.cmd` 调的是 `powershell.exe` 而不是 `pwsh`。
|
|
43
|
+
|
|
44
|
+
## 写 POSIX 脚本时踩过的三个坑(别再踩一次)
|
|
45
|
+
|
|
46
|
+
这三个都是**真跑出来的**(不是猜的),症状和原因都写在这儿:
|
|
47
|
+
|
|
48
|
+
1. **`/bin/sh`(dash)会数单引号里的括号。**
|
|
49
|
+
一段 `$( ... )` 里如果有一段**单引号包起来的英文兜底文案**,而那个文案里有一个
|
|
50
|
+
没配平的 `)`,整份脚本会直接语法报错(`Syntax error: ")" unexpected` / `expecting ")"`),
|
|
51
|
+
**而且报错的位置在后面很远的地方**,很难看出是哪一行害的。
|
|
52
|
+
→ **兜底文案里不要出现括号**。要写就写成 `-` 或 `:`。
|
|
53
|
+
2. **双引号包起来的 sed 表达式会被 shell 吃掉转义。**
|
|
54
|
+
`sed -n "s/.*\"$key\".../\1/p"` 里的 `\"` 会被 shell 先处理掉,表达式在引号那里被截断,
|
|
55
|
+
sed 报 `unterminated 's' command`(而且报错只给 char N,看不出是引号的问题)。
|
|
56
|
+
→ **sed 表达式要么全单引号,要么用 `'...'"$var"'...'` 的拼接写法**。
|
|
57
|
+
3. **文案 JSON 是嵌套的**(`restore.title` 是一条路径,不是文件里的键)。
|
|
58
|
+
用一条平铺的 sed 去找 `"restore.title"` 永远找不到 → 脚本会静悄悄地退回英文,
|
|
59
|
+
**功能全对、只是没有中文**,特别难发现。
|
|
60
|
+
→ **按路径取值**:用 node 读 JSON(`process.argv[2].split(".")` 一层层取),
|
|
61
|
+
顺便把 `{0}`/`{1}` 的占位符替换也交给 node —— 中文是 UTF-8,**按字节的 sed 会把它弄坏**。
|
|
62
|
+
|
|
63
|
+
这三个坑的共同点:**症状离原因很远**(语法错在后面、或者干脆不报错只是少了中文),
|
|
64
|
+
所以每个脚本都尽量把"取值"和"拼字符串"这两件事交给 node,只在 node 不在时才退回
|
|
65
|
+
那一小段 ASCII 兜底逻辑。
|