dsh-autorestart 0.8.1 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +81 -0
- package/README.md +60 -10
- package/client.js +1038 -49
- package/docs/README.zh.md +8 -6
- package/index.js +538 -44
- package/lib/backup.js +279 -9
- package/lib/checkpoint.js +153 -0
- package/lib/diagnostics.js +18 -0
- package/lib/restart-phase.js +304 -0
- package/lib/watchdog-disarm.js +506 -0
- package/lib/watchdog.js +86 -0
- package/locale/en.json +27 -5
- package/locale/zh.json +27 -5
- package/package.json +1 -1
- package/scripts/drill-watchdog.mjs +135 -2
- package/scripts/watchdog.mjs +107 -0
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* THE FOUR STAGES OF A RESTART — and what the user is told during each one.
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS FILE EXISTS (user report, 2026-10-29):
|
|
5
|
+
*
|
|
6
|
+
* "I clicked 'restart in 0 seconds' and then nothing moved for ten-odd
|
|
7
|
+
* seconds. It was backing up 156 MB the whole time. I thought the button was
|
|
8
|
+
* dead, or that '0 seconds' was a lie, or that there was a bug."
|
|
9
|
+
*
|
|
10
|
+
* The restart really has four stages, and each one has to be VISIBLE at once,
|
|
11
|
+
* because "the screen does nothing" is indistinguishable from "it is broken":
|
|
12
|
+
*
|
|
13
|
+
* waiting the gate is still waiting for the conversations to become idle
|
|
14
|
+
* snapshotting the pre-restart snapshot is being written (this is the long one)
|
|
15
|
+
* restarting the host is about to exit; from here nothing can be cancelled
|
|
16
|
+
* back the new host is up and the sessions continue
|
|
17
|
+
*
|
|
18
|
+
* TWO RULES that make the difference between "a progress report" and "a fake
|
|
19
|
+
* progress bar":
|
|
20
|
+
*
|
|
21
|
+
* 1. Every number here comes from the real copy loop (`lib/backup.js`): the
|
|
22
|
+
* bytes that were actually written and the bytes there are to write. When
|
|
23
|
+
* there is no measurement yet, the text says nothing about progress
|
|
24
|
+
* instead of inventing it.
|
|
25
|
+
* 2. A slow copy is SAID OUT LOUD (`slow: 'total'` after five minutes,
|
|
26
|
+
* `slow: 'stalled'` when the byte counter has not moved for a minute) —
|
|
27
|
+
* that is the opposite of pretending: the user learns that the disk is
|
|
28
|
+
* busy and that it is still working.
|
|
29
|
+
*
|
|
30
|
+
* Everything in this file is pure: it takes numbers and returns an object. The
|
|
31
|
+
* host, the state route, the health route and the client all read the same
|
|
32
|
+
* shape, so no two surfaces can disagree about what stage the restart is in.
|
|
33
|
+
*
|
|
34
|
+
* ESM only, Node builtins only (nothing here imports anything).
|
|
35
|
+
*/
|
|
36
|
+
|
|
37
|
+
/** The stages, in the order they happen. */
|
|
38
|
+
export const RESTART_PHASES = Object.freeze(['waiting', 'snapshotting', 'restarting', 'back']);
|
|
39
|
+
|
|
40
|
+
/** After this long the snapshot phase is reported as slow (5 minutes). */
|
|
41
|
+
export const SLOW_TOTAL_MS = 300000;
|
|
42
|
+
|
|
43
|
+
/** After this long without a byte of progress the snapshot is reported as stalled. */
|
|
44
|
+
export const SLOW_STALL_MS = 60000;
|
|
45
|
+
|
|
46
|
+
/** @param {unknown} value */
|
|
47
|
+
function finiteNumber(value) {
|
|
48
|
+
const number = Number(value);
|
|
49
|
+
return Number.isFinite(number) ? number : null;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** @param {unknown} value */
|
|
53
|
+
function nonNegative(value) {
|
|
54
|
+
const number = finiteNumber(value);
|
|
55
|
+
return number === null ? null : Math.max(0, number);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Is this one of the four stages?
|
|
60
|
+
*
|
|
61
|
+
* @param {unknown} phase
|
|
62
|
+
* @returns {boolean}
|
|
63
|
+
*/
|
|
64
|
+
export function isRestartPhase(phase) {
|
|
65
|
+
return typeof phase === 'string' && RESTART_PHASES.includes(phase);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* The stage that is shown, when the browser cannot show a progress line.
|
|
70
|
+
*
|
|
71
|
+
* @param {string} phase
|
|
72
|
+
* @returns {boolean} whether the restart may still be called off
|
|
73
|
+
*/
|
|
74
|
+
export function isCancellable(phase) {
|
|
75
|
+
// Once the host is shutting down, a cancel would be a lie: the snapshot is
|
|
76
|
+
// already the rollback point and the process is on its way out.
|
|
77
|
+
return phase === 'waiting' || phase === 'snapshotting';
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Why a cancel is refused at this stage (null when it is allowed).
|
|
82
|
+
*
|
|
83
|
+
* @param {string} phase
|
|
84
|
+
* @returns {string|null}
|
|
85
|
+
*/
|
|
86
|
+
export function cancelRefusalReason(phase) {
|
|
87
|
+
if (isCancellable(phase)) return null;
|
|
88
|
+
if (phase === 'restarting') return 'the host is already restarting';
|
|
89
|
+
if (phase === 'back') return 'the restart is already finished';
|
|
90
|
+
return 'there is no restart to cancel';
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* "1.2 MB" / "156 MB" — the size the user actually sees.
|
|
95
|
+
*
|
|
96
|
+
* Rounded on purpose: `153.7 MB` is noise, `154 MB` is a fact. Below one
|
|
97
|
+
* megabyte the kilobyte value is used, so a small profile still shows motion.
|
|
98
|
+
*
|
|
99
|
+
* @param {number|null} bytes
|
|
100
|
+
* @returns {string|null}
|
|
101
|
+
*/
|
|
102
|
+
export function formatBytes(bytes) {
|
|
103
|
+
const value = nonNegative(bytes);
|
|
104
|
+
if (value === null) return null;
|
|
105
|
+
if (value < 1024) return value + ' B';
|
|
106
|
+
if (value < 1024 * 1024) return Math.round(value / 1024) + ' KB';
|
|
107
|
+
return Math.round((value / (1024 * 1024)) * 10) / 10 + ' MB';
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* How long the rest of the copy will take, from the speed so far.
|
|
112
|
+
*
|
|
113
|
+
* It is only offered when there is something to base it on (real bytes, at
|
|
114
|
+
* least a second of real copying and a total to compare with). The result is
|
|
115
|
+
* capped at "a long time", so a copy that slowed to a crawl does not promise
|
|
116
|
+
* "4 seconds left" for ten minutes.
|
|
117
|
+
*
|
|
118
|
+
* @param {{ bytes?: number|null, total?: number|null, elapsedMs?: number|null }} [progress]
|
|
119
|
+
* @returns {number|null} seconds, or null when it cannot be estimated
|
|
120
|
+
*/
|
|
121
|
+
export function estimateRemainingSeconds({ bytes = null, total = null, elapsedMs = null } = {}) {
|
|
122
|
+
const done = nonNegative(bytes);
|
|
123
|
+
const all = nonNegative(total);
|
|
124
|
+
const elapsed = nonNegative(elapsedMs);
|
|
125
|
+
if (done === null || all === null || elapsed === null) return null;
|
|
126
|
+
if (done <= 0 || all <= 0 || elapsed < 1000) return null;
|
|
127
|
+
if (done >= all) return 0;
|
|
128
|
+
const perByte = elapsed / done;
|
|
129
|
+
const seconds = Math.ceil((all - done) * perByte / 1000);
|
|
130
|
+
if (!Number.isFinite(seconds)) return null;
|
|
131
|
+
return Math.min(Math.max(0, seconds), 3600);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* The stage, as the interface shows it.
|
|
136
|
+
*
|
|
137
|
+
* @param {{
|
|
138
|
+
* phase?: string,
|
|
139
|
+
* startedAt?: number|null,
|
|
140
|
+
* at?: number|null,
|
|
141
|
+
* progress?: { files?: number|null, bytes?: number|null, total?: number|null, filesTotal?: number|null, at?: number|null }|null,
|
|
142
|
+
* blocking?: number|null,
|
|
143
|
+
* now?: number,
|
|
144
|
+
* }} [options]
|
|
145
|
+
* @returns {{
|
|
146
|
+
* phase: string,
|
|
147
|
+
* startedAt: number|null,
|
|
148
|
+
* at: number|null,
|
|
149
|
+
* elapsedMs: number|null,
|
|
150
|
+
* progress: { files: number|null, total: number|null, filesTotal: number|null, bytes: number|null, percent: number|null, ageMs: number|null, at: number|null }|null,
|
|
151
|
+
* slow: 'total'|'stalled'|null,
|
|
152
|
+
* cancellable: boolean,
|
|
153
|
+
* cancelReason: string|null,
|
|
154
|
+
* estimateSeconds: number|null,
|
|
155
|
+
* blocking: number|null,
|
|
156
|
+
* text: { zh: string, en: string },
|
|
157
|
+
* }}
|
|
158
|
+
*/
|
|
159
|
+
export function describeRestartPhase({
|
|
160
|
+
phase = 'idle',
|
|
161
|
+
startedAt = null,
|
|
162
|
+
at = null,
|
|
163
|
+
progress = null,
|
|
164
|
+
blocking = null,
|
|
165
|
+
snapshotBytes = null,
|
|
166
|
+
now = Date.now(),
|
|
167
|
+
} = {}) {
|
|
168
|
+
const stage = isRestartPhase(phase) ? phase : 'idle';
|
|
169
|
+
const started = finiteNumber(startedAt);
|
|
170
|
+
const moment = finiteNumber(at);
|
|
171
|
+
const clock = finiteNumber(now) ?? Date.now();
|
|
172
|
+
const elapsedMs = started === null ? null : Math.max(0, clock - started);
|
|
173
|
+
|
|
174
|
+
let measured = null;
|
|
175
|
+
if (progress && typeof progress === 'object') {
|
|
176
|
+
const bytes = nonNegative(progress.bytes);
|
|
177
|
+
const total = nonNegative(progress.total);
|
|
178
|
+
const files = nonNegative(progress.files);
|
|
179
|
+
const filesTotal = nonNegative(progress.filesTotal);
|
|
180
|
+
const measuredAt = finiteNumber(progress.at);
|
|
181
|
+
const percent = bytes !== null && total !== null && total > 0
|
|
182
|
+
? Math.min(100, Math.round((bytes / total) * 100))
|
|
183
|
+
: null;
|
|
184
|
+
measured = {
|
|
185
|
+
files,
|
|
186
|
+
total,
|
|
187
|
+
filesTotal,
|
|
188
|
+
bytes,
|
|
189
|
+
percent,
|
|
190
|
+
// The age of the measurement: a counter that stopped moving is what
|
|
191
|
+
// "stalled" means, and it is measured from the last update, not invented.
|
|
192
|
+
ageMs: measuredAt === null ? null : Math.max(0, clock - measuredAt),
|
|
193
|
+
at: measuredAt,
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
let slow = null;
|
|
198
|
+
if (stage === 'snapshotting') {
|
|
199
|
+
if (elapsedMs !== null && elapsedMs >= SLOW_TOTAL_MS) slow = 'total';
|
|
200
|
+
else if (measured !== null && measured.ageMs !== null && measured.ageMs >= SLOW_STALL_MS) slow = 'stalled';
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
const seconds = estimateRemainingSeconds({
|
|
204
|
+
bytes: measured === null ? null : measured.bytes,
|
|
205
|
+
total: measured === null ? null : measured.total,
|
|
206
|
+
elapsedMs,
|
|
207
|
+
});
|
|
208
|
+
|
|
209
|
+
const estimateSeconds = stage === 'snapshotting' ? seconds : null;
|
|
210
|
+
const done = measured === null ? null : formatBytes(measured.bytes);
|
|
211
|
+
// HOW MANY TASKS STILL HOLD THE GATE. The centred overlay says
|
|
212
|
+
// "还有 N 个会话" as a number of its own, so the count travels as a NUMBER
|
|
213
|
+
// and not only inside the sentence: a surface that has room for two words
|
|
214
|
+
// must not have to parse the long text.
|
|
215
|
+
//
|
|
216
|
+
// NULL means "the caller did not say", which is not the same as zero:
|
|
217
|
+
// Number(null) is 0, and a stage that was never given a count must not
|
|
218
|
+
// pretend the gate is free.
|
|
219
|
+
const blockingCount = blocking === null || blocking === undefined || blocking === ''
|
|
220
|
+
? null
|
|
221
|
+
: nonNegative(blocking);
|
|
222
|
+
|
|
223
|
+
let text;
|
|
224
|
+
if (stage === 'waiting') {
|
|
225
|
+
const count = blockingCount;
|
|
226
|
+
const suffixZh = count && count > 0 ? '(还有 ' + count + ' 个会话在忙)' : '';
|
|
227
|
+
const suffixEn = count && count > 0 ? ' (' + count + (count === 1 ? ' conversation' : ' conversations') + ' busy)' : '';
|
|
228
|
+
text = {
|
|
229
|
+
zh: '正在等待所有会话空闲…' + suffixZh,
|
|
230
|
+
en: 'Waiting for every conversation to be idle…' + suffixEn,
|
|
231
|
+
};
|
|
232
|
+
} else if (stage === 'snapshotting') {
|
|
233
|
+
// ONE VOICE (user report, 2026-10-29): "正在备份" did not say WHAT was being
|
|
234
|
+
// backed up — the card says "DSH 重启前备份中", and this sentence (the
|
|
235
|
+
// settings page and the tooltip) says the same words, so no surface can
|
|
236
|
+
// disagree about what is running.
|
|
237
|
+
//
|
|
238
|
+
// The numbers appear ONLY when they were measured: "预计 X 秒" needs a real
|
|
239
|
+
// speed behind it, and "已完成 Y MB" a real byte count. With nothing
|
|
240
|
+
// measured yet the sentence claims nothing at all.
|
|
241
|
+
const measuredZh = [];
|
|
242
|
+
const measuredEn = [];
|
|
243
|
+
if (estimateSeconds !== null) {
|
|
244
|
+
measuredZh.push('预计 ' + estimateSeconds + ' 秒');
|
|
245
|
+
measuredEn.push('about ' + estimateSeconds + 's');
|
|
246
|
+
}
|
|
247
|
+
if (done !== null) {
|
|
248
|
+
measuredZh.push('已完成 ' + done);
|
|
249
|
+
measuredEn.push(done + ' done');
|
|
250
|
+
}
|
|
251
|
+
const detailZh = measuredZh.length === 0 ? '' : '(' + measuredZh.join(',') + ')';
|
|
252
|
+
const detailEn = measuredEn.length === 0 ? '' : ' (' + measuredEn.join(', ') + ')';
|
|
253
|
+
text = {
|
|
254
|
+
zh: 'DSH 重启前备份中' + detailZh + '…',
|
|
255
|
+
en: 'Backing up before DSH restart' + detailEn + '…',
|
|
256
|
+
};
|
|
257
|
+
if (slow !== null) {
|
|
258
|
+
// Still the same subject when it is slow: the user must know WHICH backup
|
|
259
|
+
// is slow, not only that "the backup" is.
|
|
260
|
+
text = {
|
|
261
|
+
zh: 'DSH 重启前备份比预期慢(可能磁盘很忙),还在继续…' + (done === null ? '' : '(已完成 ' + done + ')'),
|
|
262
|
+
en: 'The DSH pre-restart backup is slower than expected (the disk may be busy); it is still running…'
|
|
263
|
+
+ (done === null ? '' : ' (' + done + ' done)'),
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
} else if (stage === 'restarting') {
|
|
267
|
+
text = {
|
|
268
|
+
zh: '正在重启,稍后自动继续…',
|
|
269
|
+
en: 'Restarting; the work continues by itself in a moment…',
|
|
270
|
+
};
|
|
271
|
+
} else if (stage === 'back') {
|
|
272
|
+
text = {
|
|
273
|
+
zh: '已回来,接着干',
|
|
274
|
+
en: 'Back — carrying on where we left off',
|
|
275
|
+
};
|
|
276
|
+
} else {
|
|
277
|
+
text = { zh: '', en: '' };
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
return {
|
|
281
|
+
phase: stage,
|
|
282
|
+
startedAt: started,
|
|
283
|
+
at: moment,
|
|
284
|
+
elapsedMs,
|
|
285
|
+
progress: measured,
|
|
286
|
+
slow,
|
|
287
|
+
cancellable: isCancellable(stage),
|
|
288
|
+
cancelReason: cancelRefusalReason(stage),
|
|
289
|
+
estimateSeconds,
|
|
290
|
+
// The number of tasks the gate still waits for, on its own: the centred
|
|
291
|
+
// overlay prints it, and a surface with room for two words must not have
|
|
292
|
+
// to parse the long sentence of `text`.
|
|
293
|
+
blocking: blockingCount,
|
|
294
|
+
// HOW BIG THE PRE-RESTART BACKUP IS (bytes). It is reported on EVERY stage
|
|
295
|
+
// of the restart, not only while the copy runs, so a card that appears a
|
|
296
|
+
// moment late can still say "backup done (92 KB)" instead of nothing.
|
|
297
|
+
// null means "nothing was measured", which is NOT the same as zero bytes:
|
|
298
|
+
// Number(null) is 0, and a receipt for a 0-byte backup would be a lie.
|
|
299
|
+
snapshotBytes: snapshotBytes === null || snapshotBytes === undefined || snapshotBytes === ''
|
|
300
|
+
? null
|
|
301
|
+
: nonNegative(snapshotBytes),
|
|
302
|
+
text,
|
|
303
|
+
};
|
|
304
|
+
}
|