dsh-autorestart 0.8.4 → 0.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -0
- package/README.md +43 -7
- package/client.js +581 -9
- package/docs/README.zh.md +12 -4
- package/index.js +727 -47
- package/lib/backup.js +219 -4
- package/lib/checkpoint.js +93 -3
- package/lib/restart-phase.js +5 -0
- package/locale/en.json +21 -1
- package/locale/zh.json +21 -1
- package/package.json +2 -2
package/index.js
CHANGED
|
@@ -77,7 +77,14 @@ import {
|
|
|
77
77
|
claimRollbackLock,
|
|
78
78
|
clearRollbackReport,
|
|
79
79
|
createBackup,
|
|
80
|
+
DEFAULT_MAX_BACKUPS,
|
|
81
|
+
KIND_KNOWN_GOOD,
|
|
82
|
+
KIND_SNAPSHOT,
|
|
83
|
+
latestKnownGood,
|
|
84
|
+
listBackupIdentities,
|
|
80
85
|
listBackups,
|
|
86
|
+
markKnownGood,
|
|
87
|
+
readManifest,
|
|
81
88
|
readRollbackReport,
|
|
82
89
|
releaseRollbackLock,
|
|
83
90
|
restoreBackup,
|
|
@@ -387,7 +394,7 @@ function snapGraceChoice(value, fallback = DEFAULT_RESTART_GRACE_MS) {
|
|
|
387
394
|
* `pluginVersion()`), but a bundled installation may not ship that file — so
|
|
388
395
|
* the constant below is the last resort. A test keeps the two in sync.
|
|
389
396
|
*/
|
|
390
|
-
const PLUGIN_VERSION_FALLBACK = '0.8.
|
|
397
|
+
const PLUGIN_VERSION_FALLBACK = '0.8.5'
|
|
391
398
|
|
|
392
399
|
/**
|
|
393
400
|
* The version of the graceful-shutdown FEATURE, not the version of the package.
|
|
@@ -677,8 +684,8 @@ function buildConfigSchema() {
|
|
|
677
684
|
// rollback of "a plugin upgrade broke the boot" possible)
|
|
678
685
|
backupIncludePlugins: schemastery.boolean().default(true)
|
|
679
686
|
.description('Include the installed plugin tree in that snapshot: about 117 MB and roughly 3 extra seconds, but it is what makes a rollback of a broken plugin upgrade possible.'),
|
|
680
|
-
maxBackups: schemastery.number().step(1).min(1).default(
|
|
681
|
-
.description('How many snapshots are kept; older ones are deleted automatically.'),
|
|
687
|
+
maxBackups: schemastery.number().step(1).min(1).default(DEFAULT_MAX_BACKUPS)
|
|
688
|
+
.description('How many snapshots are kept (default 3: the current one plus the two before it); older ones are deleted automatically, except the newest known-good one, which is never removed.'),
|
|
682
689
|
rollbackTimeoutMs: schemastery.number().min(1000).default(90000)
|
|
683
690
|
.description('Milliseconds the detached watcher waits for the relaunched host before it restores the snapshot.'),
|
|
684
691
|
// The watchdog: the process that waits for the relaunched host and puts
|
|
@@ -767,7 +774,7 @@ const DEFAULTS = {
|
|
|
767
774
|
// detached watcher puts the snapshot back and launches the host again.
|
|
768
775
|
backupOnRestart: true,
|
|
769
776
|
backupIncludePlugins: true,
|
|
770
|
-
maxBackups:
|
|
777
|
+
maxBackups: DEFAULT_MAX_BACKUPS,
|
|
771
778
|
// The WATCHDOG: the process that waits for the relaunched host and puts the
|
|
772
779
|
// snapshot back when it never answers. It needs a way to be triggered
|
|
773
780
|
// later (a scheduled task, a systemd timer, a detached process) — the
|
|
@@ -813,7 +820,7 @@ function normalizeConfig(raw) {
|
|
|
813
820
|
if (config.deferPolicy !== 'auto' && config.deferPolicy !== 'wait') config.deferPolicy = 'auto'
|
|
814
821
|
if (config.restartMode !== 'app' && config.restartMode !== 'command') config.restartMode = 'app'
|
|
815
822
|
const maxBackups = Number(config.maxBackups)
|
|
816
|
-
config.maxBackups = Number.isFinite(maxBackups) && maxBackups >= 1 ? Math.trunc(maxBackups) :
|
|
823
|
+
config.maxBackups = Number.isFinite(maxBackups) && maxBackups >= 1 ? Math.trunc(maxBackups) : DEFAULT_MAX_BACKUPS
|
|
817
824
|
const rollbackTimeout = Number(config.rollbackTimeoutMs)
|
|
818
825
|
config.rollbackTimeoutMs =
|
|
819
826
|
Number.isFinite(rollbackTimeout) && rollbackTimeout >= 1000 ? Math.trunc(rollbackTimeout) : 90000
|
|
@@ -910,6 +917,20 @@ function resolveHandoffPath(config) {
|
|
|
910
917
|
return `${prefix}/handoff`
|
|
911
918
|
}
|
|
912
919
|
|
|
920
|
+
/**
|
|
921
|
+
* Resolve the manual-rollback route path: a sibling of the health route, under
|
|
922
|
+
* the same prefix (`POST /rollback-to` restores ONE backup the user picked).
|
|
923
|
+
*
|
|
924
|
+
* @param {object} config
|
|
925
|
+
* @returns {string}
|
|
926
|
+
*/
|
|
927
|
+
function resolveManualRollbackPath(config) {
|
|
928
|
+
const healthPath = resolveHealthPath(config)
|
|
929
|
+
const index = healthPath.lastIndexOf('/')
|
|
930
|
+
const prefix = index > 0 ? healthPath.slice(0, index) : '/__dsh_autorestart'
|
|
931
|
+
return `${prefix}/rollback-to`
|
|
932
|
+
}
|
|
933
|
+
|
|
913
934
|
/**
|
|
914
935
|
* Resolve the choice-route path: a sibling of the health route, under the same
|
|
915
936
|
* prefix (`POST /choice` persists the one-time install choice of the user).
|
|
@@ -1378,6 +1399,101 @@ function backupsDirFor(dir) {
|
|
|
1378
1399
|
return backupsRoot(typeof dir === 'string' ? dir : '')
|
|
1379
1400
|
}
|
|
1380
1401
|
|
|
1402
|
+
/**
|
|
1403
|
+
* THE FIRST SAFE COPY (user decision, 2026-10-30).
|
|
1404
|
+
*
|
|
1405
|
+
* A machine that has never restarted has no backup at all, so a boot that fails
|
|
1406
|
+
* after the plugin is installed has nothing to roll back to. The FIRST boot of a
|
|
1407
|
+
* NEW host process therefore seeds one known-good copy of the profile.
|
|
1408
|
+
*
|
|
1409
|
+
* Two rules keep it honest:
|
|
1410
|
+
* - a plugin RELOAD is not an installation: `runtime.isNewBoot` must be true,
|
|
1411
|
+
* and nothing is written on a reload;
|
|
1412
|
+
* - a machine that already has a known-good copy gets no second seed.
|
|
1413
|
+
*
|
|
1414
|
+
* A failure is NOT silent: it is recorded on `runtime.firstInstallBackup` (and
|
|
1415
|
+
* in the checkpoint, so it survives the next boot) and the health route reports
|
|
1416
|
+
* it, because a machine with no safe copy must be able to say so.
|
|
1417
|
+
*
|
|
1418
|
+
* @param {object} runtime
|
|
1419
|
+
* @param {object} [config]
|
|
1420
|
+
* @param {{ profileDir?: string }} [overrides] the profile to copy (tests pass it)
|
|
1421
|
+
* @returns {Promise<object|null>} the record that was written (null = nothing to do)
|
|
1422
|
+
*/
|
|
1423
|
+
export async function seedFirstInstallBackup(runtime, config = {}, overrides = {}) {
|
|
1424
|
+
const state = runtime && runtime.state && typeof runtime.state === 'object' ? runtime.state : null
|
|
1425
|
+
if (state === null) return null
|
|
1426
|
+
if (runtime.isNewBoot !== true) return null
|
|
1427
|
+
// NO BACKUPS AT ALL: the user switched the pre-restart snapshot off, so a
|
|
1428
|
+
// safe copy is not wanted either.
|
|
1429
|
+
if (config.backupOnRestart === false) return null
|
|
1430
|
+
const stateDir = path.dirname(String(runtime.stateFile ?? ''))
|
|
1431
|
+
const backupsDir = backupsDirFor(stateDir)
|
|
1432
|
+
/**
|
|
1433
|
+
* Record the outcome where the interface can see it.
|
|
1434
|
+
*
|
|
1435
|
+
* @param {object} value
|
|
1436
|
+
* @returns {object}
|
|
1437
|
+
*/
|
|
1438
|
+
const record = (value) => {
|
|
1439
|
+
state.firstInstallBackup = value
|
|
1440
|
+
runtime.firstInstallBackup = value
|
|
1441
|
+
persistSafely(runtime)
|
|
1442
|
+
return value
|
|
1443
|
+
}
|
|
1444
|
+
const existing = latestKnownGood(backupsDir)
|
|
1445
|
+
if (existing !== null) {
|
|
1446
|
+
return record({ result: 'present', at: Date.now(), dir: existing.dir, bytes: null, error: null })
|
|
1447
|
+
}
|
|
1448
|
+
/**
|
|
1449
|
+
* Report that this machine has NO safe copy, with the reason.
|
|
1450
|
+
*
|
|
1451
|
+
* @param {string} error
|
|
1452
|
+
* @returns {object}
|
|
1453
|
+
*/
|
|
1454
|
+
const failed = (error) => {
|
|
1455
|
+
const value = { result: 'no-known-good', at: Date.now(), dir: null, bytes: null, error: String(error) }
|
|
1456
|
+
try {
|
|
1457
|
+
runtime.log.warn('the first safe copy could not be written: this machine has nothing to roll back to', { error: String(error) })
|
|
1458
|
+
} catch {}
|
|
1459
|
+
return record(value)
|
|
1460
|
+
}
|
|
1461
|
+
const profileDir = typeof overrides.profileDir === 'string' && overrides.profileDir !== ''
|
|
1462
|
+
? overrides.profileDir
|
|
1463
|
+
: resolveProfileDir()
|
|
1464
|
+
if (profileDir === null) return failed('the profile directory could not be resolved')
|
|
1465
|
+
const maxBackups = Number(config.maxBackups)
|
|
1466
|
+
let result = null
|
|
1467
|
+
try {
|
|
1468
|
+
result = await createBackup({
|
|
1469
|
+
profileDir,
|
|
1470
|
+
backupsDir,
|
|
1471
|
+
// CONFIGURATION ONLY. The seed exists so a fresh installation has
|
|
1472
|
+
// something to roll back to at once; the plugin tree (the 117 MB part)
|
|
1473
|
+
// comes with the snapshot of the first restart, which is promoted to
|
|
1474
|
+
// known-good as soon as that restart proves it boots.
|
|
1475
|
+
includePlugins: false,
|
|
1476
|
+
maxBackups: Number.isFinite(maxBackups) && maxBackups >= 1 ? Math.trunc(maxBackups) : DEFAULT_MAX_BACKUPS,
|
|
1477
|
+
kind: KIND_KNOWN_GOOD,
|
|
1478
|
+
reason: 'first-install',
|
|
1479
|
+
version: runtime.pluginVersion ?? PLUGIN_VERSION_FALLBACK,
|
|
1480
|
+
bootId: runtime.bootId ?? null,
|
|
1481
|
+
})
|
|
1482
|
+
} catch (error) {
|
|
1483
|
+
return failed(error && error.message ? error.message : String(error))
|
|
1484
|
+
}
|
|
1485
|
+
if (!result || result.ok !== true || typeof result.dir !== 'string' || result.dir === '') {
|
|
1486
|
+
return failed(result && result.error ? result.error : 'the safe copy was not written')
|
|
1487
|
+
}
|
|
1488
|
+
return record({
|
|
1489
|
+
result: 'seeded',
|
|
1490
|
+
at: Date.now(),
|
|
1491
|
+
dir: result.dir,
|
|
1492
|
+
bytes: Number.isFinite(result.bytes) ? result.bytes : null,
|
|
1493
|
+
error: null,
|
|
1494
|
+
})
|
|
1495
|
+
}
|
|
1496
|
+
|
|
1381
1497
|
/**
|
|
1382
1498
|
* The pre-restart snapshot: copy the profile into `<stateDir>/backups/<stamp>/`
|
|
1383
1499
|
* and clear the rollback report an older boot may have left behind.
|
|
@@ -2588,12 +2704,18 @@ const titleOfSummary = (item) => {
|
|
|
2588
2704
|
state.restart.failedAttempts = (Number(state.restart.failedAttempts) || 0) + 1
|
|
2589
2705
|
if (state.restart.failedAttempts >= 2) {
|
|
2590
2706
|
log.error('restart could not be scheduled twice; cancelling the pending restart instead of retrying forever:', reason)
|
|
2707
|
+
// The restart is gone: so is its stage (a stage without a restart would
|
|
2708
|
+
// draw a card for nothing).
|
|
2709
|
+
clearRestartStage()
|
|
2591
2710
|
releaseRequesters()
|
|
2592
2711
|
cancelRestart(state, { now: Date.now() })
|
|
2593
2712
|
persist()
|
|
2594
2713
|
stopPolling()
|
|
2595
2714
|
return
|
|
2596
2715
|
}
|
|
2716
|
+
// A retry is scheduled: the stage goes back to waiting, WITH the reason, so
|
|
2717
|
+
// no surface keeps showing the step that already ended.
|
|
2718
|
+
deferRestartStage(String(reason))
|
|
2597
2719
|
setPhase(state, 'deferred')
|
|
2598
2720
|
if (state.restart.attempt > 0) state.restart.attempt -= 1
|
|
2599
2721
|
// Back off before the next attempt: no tight retry loop on a broken scheduler.
|
|
@@ -2615,6 +2737,12 @@ const titleOfSummary = (item) => {
|
|
|
2615
2737
|
const aborts = (Number(state.restart.gateAborts) || 0) + 1
|
|
2616
2738
|
state.restart.gateAborts = aborts
|
|
2617
2739
|
if (state.restart.attempt > 0) state.restart.attempt -= 1
|
|
2740
|
+
// The helper refused to kill this process (a conversation is active): the
|
|
2741
|
+
// restart is postponed, so the stage must say 'waiting' — this is the
|
|
2742
|
+
// exact path that left the backup card on screen for ever.
|
|
2743
|
+
deferRestartStage('detached helper denied the idle gate', {
|
|
2744
|
+
blocked: countBlockerTasks(gate().blockedBy),
|
|
2745
|
+
})
|
|
2618
2746
|
setPhase(state, 'deferred')
|
|
2619
2747
|
if (aborts > 20) {
|
|
2620
2748
|
log.error('the detached helper aborted on the idle gate more than 20 times; the next attempt is scheduled UNGATED:', { gateAborts: aborts })
|
|
@@ -2661,13 +2789,170 @@ const titleOfSummary = (item) => {
|
|
|
2661
2789
|
* @param {{ blocked?: number, now?: number }} [options]
|
|
2662
2790
|
* @returns {object|null} the recorded stage
|
|
2663
2791
|
*/
|
|
2664
|
-
|
|
2792
|
+
/**
|
|
2793
|
+
* HOW OFTEN THE IDLE GATE IS RE-CHECKED WHILE THE HOST WAITS TO BE KILLED.
|
|
2794
|
+
*
|
|
2795
|
+
* THE REPORT (user, 2026-10-30): after the backup the card said "正在重启…" for
|
|
2796
|
+
* 25 s while nothing was restarting — the helper was still waiting for its own
|
|
2797
|
+
* idle re-check, and only THEN did the stage go back to 'waiting'. 300–500 ms
|
|
2798
|
+
* is the window the owner asked for: a restart that is not going to happen must
|
|
2799
|
+
* be downgraded before anybody reads the old word.
|
|
2800
|
+
*/
|
|
2801
|
+
const RESTART_BUSY_CHECK_MS = 350
|
|
2802
|
+
let restartBusyTimer = null
|
|
2803
|
+
|
|
2804
|
+
/** Stop the re-check of the idle gate (idempotent). */
|
|
2805
|
+
const stopRestartBusyWatch = () => {
|
|
2806
|
+
if (restartBusyTimer === null) return
|
|
2807
|
+
try {
|
|
2808
|
+
restartBusyTimer()
|
|
2809
|
+
} catch {
|
|
2810
|
+
// The timer belongs to a disposed fiber already.
|
|
2811
|
+
}
|
|
2812
|
+
restartBusyTimer = null
|
|
2813
|
+
}
|
|
2814
|
+
|
|
2815
|
+
/**
|
|
2816
|
+
* WHILE THE STAGE SAYS 'restarting', NOBODY MAY BECOME BUSY UNNOTICED.
|
|
2817
|
+
*
|
|
2818
|
+
* 'restarting' is a CLAIM: from here on the only thing left is the process
|
|
2819
|
+
* being killed. A conversation that starts while the helper waits makes that
|
|
2820
|
+
* claim false, so it is withdrawn at once (the gate is re-checked four times a
|
|
2821
|
+
* second) instead of being discovered by the helper half a minute later.
|
|
2822
|
+
*/
|
|
2823
|
+
const armRestartBusyWatch = () => {
|
|
2824
|
+
stopRestartBusyWatch()
|
|
2825
|
+
restartBusyTimer = ctx.interval(() => {
|
|
2826
|
+
if (disposed) return stopRestartBusyWatch()
|
|
2827
|
+
const record = state.restartPhase
|
|
2828
|
+
// Only a stage that still claims 'restarting' is watched: every other
|
|
2829
|
+
// stage already says the truth.
|
|
2830
|
+
if (!record || typeof record !== 'object' || record.phase !== 'restarting') {
|
|
2831
|
+
return stopRestartBusyWatch()
|
|
2832
|
+
}
|
|
2833
|
+
if (!state.restart) return stopRestartBusyWatch()
|
|
2834
|
+
let verdict
|
|
2835
|
+
try {
|
|
2836
|
+
verdict = gate()
|
|
2837
|
+
} catch {
|
|
2838
|
+
return undefined
|
|
2839
|
+
}
|
|
2840
|
+
if (verdict && verdict.allow === true) return undefined
|
|
2841
|
+
log.warn('a conversation became active while the restart was being carried out; the stage goes back to waiting', {
|
|
2842
|
+
reason: verdict ? verdict.reason : null,
|
|
2843
|
+
blockedBy: verdict ? verdict.blockedBy : null,
|
|
2844
|
+
})
|
|
2845
|
+
deferRestartStage('conversation-active while restarting', {
|
|
2846
|
+
blocked: verdict && Array.isArray(verdict.blockedBy) ? countBlockerTasks(verdict.blockedBy) : 0,
|
|
2847
|
+
})
|
|
2848
|
+
setPhase(state, 'deferred')
|
|
2849
|
+
persist()
|
|
2850
|
+
startPolling()
|
|
2851
|
+
return stopRestartBusyWatch()
|
|
2852
|
+
}, RESTART_BUSY_CHECK_MS)
|
|
2853
|
+
return restartBusyTimer
|
|
2854
|
+
}
|
|
2855
|
+
|
|
2856
|
+
const markRestartStage = (phase, { blocked = 0, now = Date.now(), reason = null, snapshotBytes } = {}) => {
|
|
2665
2857
|
if (!RESTART_PHASES.includes(phase)) return null
|
|
2666
|
-
const record = setRestartPhase(state, { phase, blocked, now })
|
|
2858
|
+
const record = setRestartPhase(state, { phase, blocked, now, reason, snapshotBytes })
|
|
2667
2859
|
persist()
|
|
2668
2860
|
return record
|
|
2669
2861
|
}
|
|
2670
2862
|
|
|
2863
|
+
/**
|
|
2864
|
+
* REMEMBER THE SNAPSHOT THIS REQUEST ALREADY TOOK.
|
|
2865
|
+
*
|
|
2866
|
+
* THE REPORT (user, 2026-10-30): the copy finished, a conversation postponed the
|
|
2867
|
+
* restart, and the retry started copying the whole 205 MB again. The snapshot is
|
|
2868
|
+
* the rollback point of THIS restart; it does not get stale because the restart
|
|
2869
|
+
* had to wait. It is stored on the request, so withdrawing or carrying out the
|
|
2870
|
+
* request throws it away by itself.
|
|
2871
|
+
*
|
|
2872
|
+
* @param {object} snapshot the value the snapshot entry returned
|
|
2873
|
+
*/
|
|
2874
|
+
const rememberSnapshot = (snapshot) => {
|
|
2875
|
+
if (!state.restart || typeof state.restart !== 'object') return
|
|
2876
|
+
const backup = snapshot && typeof snapshot === 'object' && snapshot.backup && typeof snapshot.backup === 'object'
|
|
2877
|
+
? snapshot.backup
|
|
2878
|
+
: null
|
|
2879
|
+
if (!backup || backup.ok !== true || typeof backup.dir !== 'string' || backup.dir === '') return
|
|
2880
|
+
const number = (value) => (Number.isFinite(value) ? Math.max(0, Math.trunc(value)) : 0)
|
|
2881
|
+
state.restart.snapshot = {
|
|
2882
|
+
// The request this copy belongs to: a different request must start over.
|
|
2883
|
+
requestedAt: Number.isFinite(state.restart.requestedAt) ? state.restart.requestedAt : 0,
|
|
2884
|
+
dir: backup.dir,
|
|
2885
|
+
profileDir: typeof snapshot.profileDir === 'string' && snapshot.profileDir !== '' ? snapshot.profileDir : null,
|
|
2886
|
+
files: number(backup.files),
|
|
2887
|
+
bytes: number(backup.bytes),
|
|
2888
|
+
total: number(backup.totalBytes) || number(backup.bytes),
|
|
2889
|
+
at: Date.now(),
|
|
2890
|
+
}
|
|
2891
|
+
persist()
|
|
2892
|
+
log.info('the pre-restart snapshot of this request is remembered; a retry reuses it', {
|
|
2893
|
+
dir: backup.dir,
|
|
2894
|
+
bytes: state.restart.snapshot.bytes,
|
|
2895
|
+
})
|
|
2896
|
+
}
|
|
2897
|
+
|
|
2898
|
+
/**
|
|
2899
|
+
* THE SNAPSHOT THIS REQUEST ALREADY PAID FOR, if it is still usable.
|
|
2900
|
+
*
|
|
2901
|
+
* All three conditions must hold, or the restart copies again:
|
|
2902
|
+
* 1. it belongs to THE SAME request (same `requestedAt`),
|
|
2903
|
+
* 2. the backup directory is still there,
|
|
2904
|
+
* 3. its manifest is readable and reports the size that was recorded — a
|
|
2905
|
+
* half-deleted or replaced directory never passes for a rollback point.
|
|
2906
|
+
*
|
|
2907
|
+
* @returns {object|null}
|
|
2908
|
+
*/
|
|
2909
|
+
const reusableSnapshot = () => {
|
|
2910
|
+
const request = state.restart
|
|
2911
|
+
if (!request || typeof request !== 'object') return null
|
|
2912
|
+
const saved = request.snapshot
|
|
2913
|
+
if (!saved || typeof saved !== 'object') return null
|
|
2914
|
+
if (!Number.isFinite(saved.requestedAt) || !Number.isFinite(request.requestedAt)) return null
|
|
2915
|
+
if (saved.requestedAt !== request.requestedAt) return null
|
|
2916
|
+
if (typeof saved.dir !== 'string' || saved.dir === '') return null
|
|
2917
|
+
try {
|
|
2918
|
+
if (!fs.existsSync(saved.dir)) return null
|
|
2919
|
+
const manifest = readManifest(saved.dir)
|
|
2920
|
+
if (!manifest || typeof manifest !== 'object') return null
|
|
2921
|
+
if (Number.isFinite(manifest.bytes) && Number.isFinite(saved.bytes) && manifest.bytes !== saved.bytes) return null
|
|
2922
|
+
} catch {
|
|
2923
|
+
return null
|
|
2924
|
+
}
|
|
2925
|
+
return saved
|
|
2926
|
+
}
|
|
2927
|
+
|
|
2928
|
+
/** The size this request's snapshot already has, for the stage record. */
|
|
2929
|
+
const knownSnapshotBytes = () => {
|
|
2930
|
+
const saved = state.restart && state.restart.snapshot
|
|
2931
|
+
return saved && typeof saved === 'object' && Number.isFinite(saved.bytes) ? saved.bytes : null
|
|
2932
|
+
}
|
|
2933
|
+
|
|
2934
|
+
/**
|
|
2935
|
+
* Send the stage BACK TO WAITING — the one way a postponed restart is recorded.
|
|
2936
|
+
*
|
|
2937
|
+
* WHY (user report, 2026-10-29): a restart that had already written
|
|
2938
|
+
* 'snapshotting' and was then postponed (a conversation became active again,
|
|
2939
|
+
* the detached helper refused to kill this process, a retry was scheduled)
|
|
2940
|
+
* left that stage on disk. The interface drew a card for a copy that had long
|
|
2941
|
+
* finished, and the card never went away. Every deferral now goes through
|
|
2942
|
+
* here, so the stage always says the truth: waiting.
|
|
2943
|
+
*
|
|
2944
|
+
* @param {string} reason why the restart was postponed
|
|
2945
|
+
* @param {{ blocked?: number }} [options]
|
|
2946
|
+
* @returns {object|null} the recorded stage
|
|
2947
|
+
*/
|
|
2948
|
+
const deferRestartStage = (reason, { blocked = 0 } = {}) => {
|
|
2949
|
+
const text = typeof reason === 'string' && reason !== '' ? 'deferred: ' + reason : 'deferred'
|
|
2950
|
+
const count = Number.isFinite(blocked) ? blocked : 0
|
|
2951
|
+
// A request that already took its snapshot keeps saying how big it is: the
|
|
2952
|
+
// number is a fact of this restart and does not expire while it waits.
|
|
2953
|
+
return markRestartStage('waiting', { blocked: count, reason: text, snapshotBytes: knownSnapshotBytes() })
|
|
2954
|
+
}
|
|
2955
|
+
|
|
2671
2956
|
/**
|
|
2672
2957
|
* Record the REAL progress of the running snapshot (files and bytes).
|
|
2673
2958
|
*
|
|
@@ -2717,6 +3002,9 @@ const titleOfSummary = (item) => {
|
|
|
2717
3002
|
// The measured size of the pre-restart snapshot: it survives the stage
|
|
2718
3003
|
// change, so a page that missed the copy can still report it.
|
|
2719
3004
|
snapshotBytes: record && typeof record === 'object' ? record.snapshotBytes : null,
|
|
3005
|
+
// WHY the stage is what it is ("deferred: conversation-active", "snapshot
|
|
3006
|
+
// done", …), so a postponed restart can be told from a running one.
|
|
3007
|
+
reason: record && typeof record === 'object' ? record.reason : null,
|
|
2720
3008
|
now: Date.now(),
|
|
2721
3009
|
})
|
|
2722
3010
|
}
|
|
@@ -2840,11 +3128,17 @@ const titleOfSummary = (item) => {
|
|
|
2840
3128
|
* even see the disk. Only the DIRECTORY is read — the manifest of every
|
|
2841
3129
|
* copy — because one copy can be 150 MB and its contents are never walked.
|
|
2842
3130
|
*
|
|
3131
|
+
* WHAT EACH COPY IS travels with it (user decision, 2026-10-30): only a copy
|
|
3132
|
+
* a boot has proven is a rollback target (`known-good`), the snapshot of a
|
|
3133
|
+
* restart is not (`snapshot`), and a copy without an identity file was never
|
|
3134
|
+
* proven (`unknown`). The browser cannot read `meta.json`, so the identity is
|
|
3135
|
+
* resolved HERE and handed to the interface with the numbers.
|
|
3136
|
+
*
|
|
2843
3137
|
* @returns {{ dir: string, max: number, count: number, items: Array<object> }}
|
|
2844
3138
|
*/
|
|
2845
3139
|
const backupsInfo = () => {
|
|
2846
3140
|
const dir = backupsDirFor(stateDirOf())
|
|
2847
|
-
const items =
|
|
3141
|
+
const items = listBackupIdentities(dir).map((item) => ({
|
|
2848
3142
|
name: item.name,
|
|
2849
3143
|
dir: item.dir,
|
|
2850
3144
|
at: item.at,
|
|
@@ -2852,6 +3146,10 @@ const titleOfSummary = (item) => {
|
|
|
2852
3146
|
bytes: item.bytes,
|
|
2853
3147
|
files: item.files,
|
|
2854
3148
|
includePlugins: item.includePlugins,
|
|
3149
|
+
// 'known-good' | 'snapshot' | 'unknown' — never a guess: it is the
|
|
3150
|
+
// identity the copy itself carries.
|
|
3151
|
+
kind: item.kind,
|
|
3152
|
+
version: item.version,
|
|
2855
3153
|
}))
|
|
2856
3154
|
return {
|
|
2857
3155
|
dir,
|
|
@@ -2861,6 +3159,149 @@ const titleOfSummary = (item) => {
|
|
|
2861
3159
|
}
|
|
2862
3160
|
}
|
|
2863
3161
|
|
|
3162
|
+
/** The receipt of the last manual rollback, next to the checkpoint. */
|
|
3163
|
+
const MANUAL_ROLLBACK_RECEIPT = 'last-manual-rollback.json'
|
|
3164
|
+
|
|
3165
|
+
/**
|
|
3166
|
+
* RESTORE ONE BACKUP THE USER PICKED (user decision, 2026-10-30).
|
|
3167
|
+
*
|
|
3168
|
+
* The automatic rollback only ever restores the newest PROVEN copy — but the
|
|
3169
|
+
* user may know better ("the version from last week worked"). So they may pick
|
|
3170
|
+
* any copy in the list, and the plugin restores exactly that one. Three rules
|
|
3171
|
+
* keep that safe:
|
|
3172
|
+
*
|
|
3173
|
+
* 1. THE INPUT IS A NAME, NOT A PATH. It has to match a name this host
|
|
3174
|
+
* reports from `listBackupIdentities()`, so `../`, a drive letter or any
|
|
3175
|
+
* separator is refused before anything is read — a traversal never even
|
|
3176
|
+
* becomes a lookup.
|
|
3177
|
+
* 2. THE CURRENT STATE IS SAVED FIRST. A snapshot of the profile as it is
|
|
3178
|
+
* right now is written before anything is overwritten, so a manual
|
|
3179
|
+
* rollback is itself undoable. Without that snapshot nothing is restored.
|
|
3180
|
+
* 3. THE RESTART IS THE NORMAL ONE. The plugin never kills the host itself:
|
|
3181
|
+
* it writes the receipt and requests a restart, exactly like the pill.
|
|
3182
|
+
*
|
|
3183
|
+
* @param {{ name?: string }} [request]
|
|
3184
|
+
* @returns {Promise<{ ok: boolean, status: number, error?: string|null, receipt?: object }>}
|
|
3185
|
+
*/
|
|
3186
|
+
const manualRollback = async ({ name } = {}) => {
|
|
3187
|
+
const requested = typeof name === 'string' ? name.trim() : ''
|
|
3188
|
+
if (requested === '') {
|
|
3189
|
+
return { ok: false, status: 400, error: 'a backup name is required' }
|
|
3190
|
+
}
|
|
3191
|
+
// ① A PLAIN NAME, on purpose: the list is the allow-list.
|
|
3192
|
+
if (
|
|
3193
|
+
requested.includes('/')
|
|
3194
|
+
|| requested.includes('\\')
|
|
3195
|
+
|| requested.includes('..')
|
|
3196
|
+
|| path.isAbsolute(requested)
|
|
3197
|
+
|| requested !== path.basename(requested)
|
|
3198
|
+
) {
|
|
3199
|
+
log.warn('the manual rollback was refused: the name is not a plain backup name:', requested)
|
|
3200
|
+
return { ok: false, status: 400, error: 'the name is not a backup name' }
|
|
3201
|
+
}
|
|
3202
|
+
const backupsDir = backupsDirFor(stateDirOf())
|
|
3203
|
+
const target = listBackupIdentities(backupsDir).find((entry) => entry.name === requested)
|
|
3204
|
+
if (!target || typeof target.dir !== 'string' || target.dir === '') {
|
|
3205
|
+
log.warn('the manual rollback was refused: there is no such backup:', requested)
|
|
3206
|
+
return { ok: false, status: 404, error: 'there is no such backup' }
|
|
3207
|
+
}
|
|
3208
|
+
const profile = resolveProfileDirDetailed()
|
|
3209
|
+
const profileDir = typeof profile.dir === 'string' ? profile.dir : ''
|
|
3210
|
+
if (profileDir === '') {
|
|
3211
|
+
log.warn('the manual rollback was refused: the profile directory is unknown')
|
|
3212
|
+
return { ok: false, status: 409, error: 'the profile directory is unknown' }
|
|
3213
|
+
}
|
|
3214
|
+
// ② THE WAY BACK, taken first. A manual rollback that cannot save the
|
|
3215
|
+
// current state is not offered at all.
|
|
3216
|
+
let safety = null
|
|
3217
|
+
try {
|
|
3218
|
+
safety = await createBackup({
|
|
3219
|
+
profileDir,
|
|
3220
|
+
backupsDir,
|
|
3221
|
+
includePlugins: config.backupIncludePlugins !== false,
|
|
3222
|
+
maxBackups: config.maxBackups,
|
|
3223
|
+
kind: KIND_SNAPSHOT,
|
|
3224
|
+
reason: 'before-manual-rollback',
|
|
3225
|
+
version: ownVersion(),
|
|
3226
|
+
bootId,
|
|
3227
|
+
})
|
|
3228
|
+
} catch (error) {
|
|
3229
|
+
safety = { ok: false, error: String(error) }
|
|
3230
|
+
}
|
|
3231
|
+
if (!safety || safety.ok !== true) {
|
|
3232
|
+
const reason = safety && safety.error ? safety.error : 'unknown error'
|
|
3233
|
+
log.error('the manual rollback was refused: the current state could not be saved first:', reason)
|
|
3234
|
+
return { ok: false, status: 409, error: 'the current state could not be saved first: ' + reason }
|
|
3235
|
+
}
|
|
3236
|
+
// ONE restore at a time: the detached watchdog takes the same lock.
|
|
3237
|
+
const lock = claimRollbackLock(stateDirOf())
|
|
3238
|
+
if (!lock.ok) {
|
|
3239
|
+
log.warn('the manual rollback stands down: another rollback is already running:', lock.error)
|
|
3240
|
+
return { ok: false, status: 409, error: 'another rollback is already running' }
|
|
3241
|
+
}
|
|
3242
|
+
let restored = null
|
|
3243
|
+
try {
|
|
3244
|
+
// ③ THE COPY THE USER PICKED — not the newest, not the proven one.
|
|
3245
|
+
restored = restoreBackup({ backupDir: target.dir, profileDir })
|
|
3246
|
+
} catch (error) {
|
|
3247
|
+
restored = { ok: false, restored: [], error: String(error) }
|
|
3248
|
+
} finally {
|
|
3249
|
+
releaseRollbackLock(stateDirOf())
|
|
3250
|
+
}
|
|
3251
|
+
const ok = Boolean(restored && restored.ok === true)
|
|
3252
|
+
const receipt = {
|
|
3253
|
+
at: new Date().toISOString(),
|
|
3254
|
+
name: target.name,
|
|
3255
|
+
dir: target.dir,
|
|
3256
|
+
kind: target.kind,
|
|
3257
|
+
version: target.version ?? null,
|
|
3258
|
+
restored: restored && Array.isArray(restored.restored) ? restored.restored : [],
|
|
3259
|
+
ok,
|
|
3260
|
+
error: ok ? null : ((restored && restored.error) || 'the restore failed'),
|
|
3261
|
+
// The copy taken right before the restore: the manual rollback can be
|
|
3262
|
+
// undone with it, even if the picked one was older than the hardware.
|
|
3263
|
+
safetySnapshot: typeof safety.dir === 'string' ? safety.dir : null,
|
|
3264
|
+
restartRequested: false,
|
|
3265
|
+
}
|
|
3266
|
+
const writeReceipt = () => {
|
|
3267
|
+
try {
|
|
3268
|
+
fs.writeFileSync(
|
|
3269
|
+
path.join(stateDirOf(), MANUAL_ROLLBACK_RECEIPT),
|
|
3270
|
+
JSON.stringify(receipt, null, 2),
|
|
3271
|
+
'utf8',
|
|
3272
|
+
)
|
|
3273
|
+
} catch (error) {
|
|
3274
|
+
log.warn('could not write the manual rollback receipt:', String(error))
|
|
3275
|
+
}
|
|
3276
|
+
}
|
|
3277
|
+
// ④ THE RECEIPT, before anything else happens: the next boot reads it.
|
|
3278
|
+
writeReceipt()
|
|
3279
|
+
if (!ok) {
|
|
3280
|
+
log.error('the manual rollback could not restore the copy:', receipt.error)
|
|
3281
|
+
return { ok: false, status: 500, error: receipt.error, receipt }
|
|
3282
|
+
}
|
|
3283
|
+
// ⑤ A NORMAL RESTART (the same request the pill makes). The plugin does not
|
|
3284
|
+
// kill anything itself; the restored profile is what comes back up.
|
|
3285
|
+
requestRestart(state, {
|
|
3286
|
+
reason: 'manual rollback to ' + target.name,
|
|
3287
|
+
bySessionId: 'settings',
|
|
3288
|
+
resume: false,
|
|
3289
|
+
now: Date.now(),
|
|
3290
|
+
})
|
|
3291
|
+
markRestartStage('waiting')
|
|
3292
|
+
persist()
|
|
3293
|
+
startPolling()
|
|
3294
|
+
receipt.restartRequested = true
|
|
3295
|
+
writeReceipt()
|
|
3296
|
+
log.warn('the manual rollback was done; a restart was requested', {
|
|
3297
|
+
name: target.name,
|
|
3298
|
+
kind: target.kind,
|
|
3299
|
+
restored: receipt.restored,
|
|
3300
|
+
safetySnapshot: receipt.safetySnapshot,
|
|
3301
|
+
})
|
|
3302
|
+
return { ok: true, status: 200, receipt }
|
|
3303
|
+
}
|
|
3304
|
+
|
|
2864
3305
|
/**
|
|
2865
3306
|
* WHERE THE RESCUE SCRIPTS ARE.
|
|
2866
3307
|
*
|
|
@@ -3489,10 +3930,15 @@ const titleOfSummary = (item) => {
|
|
|
3489
3930
|
*/
|
|
3490
3931
|
const rollbackInProcess = ({ backup, profileDir, reason }) => {
|
|
3491
3932
|
const restored = []
|
|
3492
|
-
|
|
3493
|
-
|
|
3494
|
-
|
|
3495
|
-
|
|
3933
|
+
// ONLY a backup that a boot has already proven may be rolled back to. The
|
|
3934
|
+
// snapshot taken right before THIS restart may be the broken state itself
|
|
3935
|
+
// (the user edited a plugin and then restarted), so it is never the target.
|
|
3936
|
+
const knownGood = latestKnownGood(backupsDirFor(stateDirOf()))
|
|
3937
|
+
const targetDir = knownGood && typeof knownGood.dir === 'string' ? knownGood.dir : ''
|
|
3938
|
+
if (targetDir === '') {
|
|
3939
|
+
log.error('the restart failed and there is no known-good backup to roll back to:', reason)
|
|
3940
|
+
writeRestartReport({ rolledBack: false, reason: 'no-known-good: ' + reason, backupDir: null, restored })
|
|
3941
|
+
return { rolledBack: false, restored, error: 'no-known-good' }
|
|
3496
3942
|
}
|
|
3497
3943
|
if (typeof profileDir !== 'string' || profileDir === '') {
|
|
3498
3944
|
log.error('the restart failed and the profile directory is unknown:', reason)
|
|
@@ -3509,24 +3955,24 @@ const titleOfSummary = (item) => {
|
|
|
3509
3955
|
}
|
|
3510
3956
|
let result = null
|
|
3511
3957
|
try {
|
|
3512
|
-
result = restoreBackup({ backupDir:
|
|
3958
|
+
result = restoreBackup({ backupDir: targetDir, profileDir })
|
|
3513
3959
|
} catch (error) {
|
|
3514
3960
|
result = { ok: false, restored: [], error: String(error) }
|
|
3515
3961
|
} finally {
|
|
3516
3962
|
releaseRollbackLock(stateDirOf())
|
|
3517
3963
|
}
|
|
3518
3964
|
if (result && result.ok === true) {
|
|
3519
|
-
log.warn('the
|
|
3965
|
+
log.warn('the known-good backup was restored:', {
|
|
3520
3966
|
reason,
|
|
3521
3967
|
restored: result.restored,
|
|
3522
|
-
backupDir:
|
|
3968
|
+
backupDir: targetDir,
|
|
3523
3969
|
})
|
|
3524
|
-
writeRestartReport({ rolledBack: true, reason, backupDir:
|
|
3970
|
+
writeRestartReport({ rolledBack: true, reason, backupDir: targetDir, restored: result.restored })
|
|
3525
3971
|
return { rolledBack: true, restored: result.restored, error: null }
|
|
3526
3972
|
}
|
|
3527
3973
|
const error = (result && result.error) || 'the restore failed'
|
|
3528
|
-
log.error('the
|
|
3529
|
-
writeRestartReport({ rolledBack: false, reason, backupDir:
|
|
3974
|
+
log.error('the known-good backup could not be restored:', reason, error)
|
|
3975
|
+
writeRestartReport({ rolledBack: false, reason, backupDir: targetDir, restored })
|
|
3530
3976
|
return { rolledBack: false, restored, error }
|
|
3531
3977
|
}
|
|
3532
3978
|
|
|
@@ -3542,9 +3988,13 @@ const titleOfSummary = (item) => {
|
|
|
3542
3988
|
* @returns {{ ok: boolean, method: string, error: string|null, scriptPath: string|null }}
|
|
3543
3989
|
*/
|
|
3544
3990
|
const armRollbackWatcher = ({ backup, profileDir, gateScriptPath = null }) => {
|
|
3545
|
-
|
|
3546
|
-
|
|
3547
|
-
|
|
3991
|
+
// The watcher must restore a state a boot has already proven. The snapshot
|
|
3992
|
+
// of the restart it is guarding may be the broken state itself.
|
|
3993
|
+
const knownGood = latestKnownGood(backupsDirFor(stateDirOf()))
|
|
3994
|
+
const targetDir = knownGood && typeof knownGood.dir === 'string' ? knownGood.dir : ''
|
|
3995
|
+
if (targetDir === '') {
|
|
3996
|
+
log.warn('no known-good backup: the automatic rollback is not armed')
|
|
3997
|
+
return { ok: false, method: 'none', error: 'no-known-good', scriptPath: null }
|
|
3548
3998
|
}
|
|
3549
3999
|
if (typeof profileDir !== 'string' || profileDir === '') {
|
|
3550
4000
|
log.warn('the profile directory is unknown: the automatic rollback is not armed')
|
|
@@ -3561,7 +4011,7 @@ const titleOfSummary = (item) => {
|
|
|
3561
4011
|
portFile: accessFilePath(stateDir),
|
|
3562
4012
|
expectedBootId: bootId,
|
|
3563
4013
|
outcomeFile: path.join(stateDir, RESTART_OUTCOME_FILE),
|
|
3564
|
-
backupDir:
|
|
4014
|
+
backupDir: targetDir,
|
|
3565
4015
|
profileDir,
|
|
3566
4016
|
reportPath: path.join(stateDir, ROLLBACK_REPORT_NAME),
|
|
3567
4017
|
lockFile: path.join(stateDir, ROLLBACK_LOCK_NAME),
|
|
@@ -3648,6 +4098,11 @@ const titleOfSummary = (item) => {
|
|
|
3648
4098
|
blockedBy: verdict.blockedBy,
|
|
3649
4099
|
quietForMs: verdict.quietForMs,
|
|
3650
4100
|
})
|
|
4101
|
+
// Postponed BEFORE the copy: the stage goes back to waiting (with the
|
|
4102
|
+
// reason and a fresh timestamp), so no card survives the deferral.
|
|
4103
|
+
deferRestartStage(verdict.reason || 'conversation-active', {
|
|
4104
|
+
blocked: countBlockerTasks(verdict.blockedBy),
|
|
4105
|
+
})
|
|
3651
4106
|
persist()
|
|
3652
4107
|
startPolling()
|
|
3653
4108
|
return
|
|
@@ -3681,29 +4136,60 @@ const titleOfSummary = (item) => {
|
|
|
3681
4136
|
// and now they say so.
|
|
3682
4137
|
const stateDir = path.dirname(stateFile)
|
|
3683
4138
|
resetSnapshotCancel()
|
|
3684
|
-
|
|
4139
|
+
// HAS THIS REQUEST ALREADY PAID FOR ITS SNAPSHOT? (user report, 2026-10-30:
|
|
4140
|
+
// the copy finished, a conversation postponed the restart, and the retry
|
|
4141
|
+
// copied all 205 MB again.) Then it is reused as it is: same request, same
|
|
4142
|
+
// directory, same manifest.
|
|
4143
|
+
const paid = reusableSnapshot()
|
|
3685
4144
|
let cancelled = false
|
|
3686
|
-
let snapshot
|
|
3687
|
-
|
|
3688
|
-
snapshot
|
|
3689
|
-
|
|
3690
|
-
|
|
3691
|
-
log,
|
|
3692
|
-
profile: resolveProfileDirDetailed(),
|
|
3693
|
-
onProgress: (progress) => {
|
|
3694
|
-
recordSnapshotProgress(progress)
|
|
3695
|
-
},
|
|
3696
|
-
shouldCancel: () => snapshotCancelled(),
|
|
4145
|
+
let snapshot = null
|
|
4146
|
+
if (paid !== null) {
|
|
4147
|
+
log.warn('reusing the snapshot this restart already took; not copying the profile again', {
|
|
4148
|
+
dir: paid.dir,
|
|
4149
|
+
bytes: paid.bytes,
|
|
3697
4150
|
})
|
|
3698
|
-
|
|
3699
|
-
|
|
3700
|
-
|
|
3701
|
-
|
|
3702
|
-
|
|
3703
|
-
|
|
3704
|
-
|
|
3705
|
-
|
|
3706
|
-
|
|
4151
|
+
snapshot = {
|
|
4152
|
+
backup: {
|
|
4153
|
+
ok: true,
|
|
4154
|
+
dir: paid.dir,
|
|
4155
|
+
bytes: paid.bytes,
|
|
4156
|
+
totalBytes: paid.total,
|
|
4157
|
+
fileCount: paid.files,
|
|
4158
|
+
reused: true,
|
|
4159
|
+
},
|
|
4160
|
+
profileDir: paid.profileDir,
|
|
4161
|
+
profile: { dir: paid.profileDir, source: 'reused' },
|
|
4162
|
+
requested: true,
|
|
4163
|
+
durationMs: 0,
|
|
4164
|
+
error: null,
|
|
4165
|
+
reused: true,
|
|
4166
|
+
}
|
|
4167
|
+
} else {
|
|
4168
|
+
markRestartStage('snapshotting')
|
|
4169
|
+
try {
|
|
4170
|
+
snapshot = await snapshotFn({
|
|
4171
|
+
config,
|
|
4172
|
+
stateDir,
|
|
4173
|
+
log,
|
|
4174
|
+
profile: resolveProfileDirDetailed(),
|
|
4175
|
+
onProgress: (progress) => {
|
|
4176
|
+
recordSnapshotProgress(progress)
|
|
4177
|
+
},
|
|
4178
|
+
shouldCancel: () => snapshotCancelled(),
|
|
4179
|
+
})
|
|
4180
|
+
} finally {
|
|
4181
|
+
// The flag belongs to ONE snapshot; a stale `true` would cancel the
|
|
4182
|
+
// next restart before it began.
|
|
4183
|
+
const wasCancelled = snapshotCancelled()
|
|
4184
|
+
resetSnapshotCancel()
|
|
4185
|
+
// Two ways a copy can be called off: the flag this process set (the
|
|
4186
|
+
// cancel route) or the snapshot reporting it itself. Both mean the same
|
|
4187
|
+
// thing, and both must stop the restart.
|
|
4188
|
+
cancelled = wasCancelled || (snapshot !== null && typeof snapshot === 'object' && snapshot.cancelled === true)
|
|
4189
|
+
}
|
|
4190
|
+
// Remember it: if this restart is postponed now, the retry must not copy
|
|
4191
|
+
// the profile a second time.
|
|
4192
|
+
if (!cancelled) rememberSnapshot(snapshot)
|
|
3707
4193
|
}
|
|
3708
4194
|
if (cancelled) {
|
|
3709
4195
|
// The user called it off while the files were being copied: the
|
|
@@ -3718,6 +4204,34 @@ const titleOfSummary = (item) => {
|
|
|
3718
4204
|
stopPolling()
|
|
3719
4205
|
return
|
|
3720
4206
|
}
|
|
4207
|
+
// A COPY THAT IS DONE IS NOT A RESTART YET (user report, 2026-10-30): "the
|
|
4208
|
+
// backup finished, and then the card said 正在重启 for 25 s while nothing was
|
|
4209
|
+
// restarting". The gate is re-checked HERE, in the same tick, so a
|
|
4210
|
+
// conversation that started during the copy postpones the restart at once
|
|
4211
|
+
// instead of being discovered by the helper half a minute later.
|
|
4212
|
+
const afterCopy = gate()
|
|
4213
|
+
if (!afterCopy || afterCopy.allow !== true) {
|
|
4214
|
+
const why = afterCopy && typeof afterCopy.reason === 'string' && afterCopy.reason !== ''
|
|
4215
|
+
? afterCopy.reason
|
|
4216
|
+
: 'conversation-active'
|
|
4217
|
+
log.info('restart deferred right after the snapshot: a conversation is active again', {
|
|
4218
|
+
reason: why,
|
|
4219
|
+
blockedBy: afterCopy ? afterCopy.blockedBy : null,
|
|
4220
|
+
})
|
|
4221
|
+
deferRestartStage('conversation-active after snapshot: ' + why, {
|
|
4222
|
+
blocked: afterCopy && Array.isArray(afterCopy.blockedBy) ? countBlockerTasks(afterCopy.blockedBy) : 0,
|
|
4223
|
+
})
|
|
4224
|
+
setPhase(state, 'deferred')
|
|
4225
|
+
persist()
|
|
4226
|
+
startPolling()
|
|
4227
|
+
return
|
|
4228
|
+
}
|
|
4229
|
+
// THE INVARIANT (user report, 2026-10-29): 'snapshotting' exists ONLY while
|
|
4230
|
+
// files are being copied. The moment the copy returns — copied, cancelled
|
|
4231
|
+
// or thrown — the stage leaves it. Everything after this point (arming the
|
|
4232
|
+
// watchdog, the supervisor hand-off, the command) is "the restart is being
|
|
4233
|
+
// executed", and a deferral from here on goes back to 'waiting'.
|
|
4234
|
+
markRestartStage('restarting', { reason: 'snapshot done' })
|
|
3721
4235
|
// The snapshot is the rollback point of BOTH branches.
|
|
3722
4236
|
const backup = snapshot && snapshot.backup ? snapshot.backup : null
|
|
3723
4237
|
const profileDir = snapshot && snapshot.profileDir ? snapshot.profileDir : null
|
|
@@ -3730,9 +4244,14 @@ const titleOfSummary = (item) => {
|
|
|
3730
4244
|
// old one (v1) is NOT started next to it — they take the same lock, and
|
|
3731
4245
|
// the one that loses it gives up. If v2 lost, the profile would be
|
|
3732
4246
|
// restored but nothing would be running any more.
|
|
3733
|
-
|
|
4247
|
+
// The watchdog restores a state a boot has already proven, NOT the
|
|
4248
|
+
// snapshot of this restart (which may be the broken state itself).
|
|
4249
|
+
const knownGoodForWatchdog = backup && backup.ok === true && profileDir !== null
|
|
4250
|
+
? latestKnownGood(backupsDirFor(stateDirOf()))
|
|
4251
|
+
: null
|
|
4252
|
+
const watchdogArm = knownGoodForWatchdog && typeof knownGoodForWatchdog.dir === 'string' && profileDir !== null
|
|
3734
4253
|
? armWatchdogFn({
|
|
3735
|
-
snapshotDir:
|
|
4254
|
+
snapshotDir: knownGoodForWatchdog.dir,
|
|
3736
4255
|
profileDir,
|
|
3737
4256
|
relaunchCommand: useCustomCommand && typeof config.restartCommand === 'string' && config.restartCommand.trim() !== ''
|
|
3738
4257
|
? {
|
|
@@ -3852,6 +4371,10 @@ const titleOfSummary = (item) => {
|
|
|
3852
4371
|
// FROM HERE ON THE RESTART CANNOT BE CANCELLED, and the interface says
|
|
3853
4372
|
// so: the host is on its way out.
|
|
3854
4373
|
markRestartStage('restarting')
|
|
4374
|
+
// A command that takes a while must not keep claiming "restarting" once
|
|
4375
|
+
// a conversation is active again (the reason is remembered and the card
|
|
4376
|
+
// turns into "restart postponed").
|
|
4377
|
+
armRestartBusyWatch()
|
|
3855
4378
|
const outcome = await runCommandFn({
|
|
3856
4379
|
command,
|
|
3857
4380
|
timeoutMs: config.restartCommandTimeoutMs,
|
|
@@ -3891,6 +4414,13 @@ const titleOfSummary = (item) => {
|
|
|
3891
4414
|
})
|
|
3892
4415
|
if (confirmed && confirmed.ok === true) {
|
|
3893
4416
|
log.warn('the new instance answered its health route:', confirmed)
|
|
4417
|
+
// THE SNAPSHOT HAS PROVEN ITSELF: this restart came up, so the copy
|
|
4418
|
+
// taken before it is the one a failed boot may be rolled back to
|
|
4419
|
+
// (user decision, 2026-10-30).
|
|
4420
|
+
if (backup && typeof backup.dir === 'string' && backup.dir !== '') {
|
|
4421
|
+
const promoted = markKnownGood(backup.dir, { reason: 'boot-ok' })
|
|
4422
|
+
log.info('the snapshot of this restart is now known-good', { dir: backup.dir, promoted })
|
|
4423
|
+
}
|
|
3894
4424
|
writeRestartOutcome({ decided: true, ok: true, reason: 'the new instance answered' })
|
|
3895
4425
|
recordRestartVerdict({
|
|
3896
4426
|
result: RESTART_RESULTS.OK,
|
|
@@ -3990,12 +4520,16 @@ const titleOfSummary = (item) => {
|
|
|
3990
4520
|
})
|
|
3991
4521
|
log.warn('scheduled detached restart:', result)
|
|
3992
4522
|
if (!result || result.ok !== true) {
|
|
3993
|
-
|
|
4523
|
+
deferRestartStage((result && result.error) || 'detached restart refused')
|
|
3994
4524
|
rollbackRestart((result && result.error) || 'detached restart refused')
|
|
3995
4525
|
} else {
|
|
3996
4526
|
// The helper is scheduled: the host is going down, and from here on a
|
|
3997
4527
|
// cancel would be a promise this plugin cannot keep.
|
|
3998
4528
|
markRestartStage('restarting')
|
|
4529
|
+
// THE 25-SECOND LIE (user report, 2026-10-30): the helper waits for its
|
|
4530
|
+
// own idle re-check, so this is exactly the window in which a
|
|
4531
|
+
// conversation can start. Watch it every 350 ms.
|
|
4532
|
+
armRestartBusyWatch()
|
|
3999
4533
|
armGateWatchdog()
|
|
4000
4534
|
}
|
|
4001
4535
|
}
|
|
@@ -4367,6 +4901,34 @@ const titleOfSummary = (item) => {
|
|
|
4367
4901
|
// single description (text included) that the state route, the health route
|
|
4368
4902
|
// and the settings page share.
|
|
4369
4903
|
markRestartStage,
|
|
4904
|
+
/**
|
|
4905
|
+
* Reset a stage that cannot be true in THIS process (boot / plugin reload).
|
|
4906
|
+
*
|
|
4907
|
+
* WHY (user report, 2026-10-29): the stage lives in the checkpoint, so a
|
|
4908
|
+
* 'snapshotting' or 'restarting' written by an earlier attempt can be read
|
|
4909
|
+
* back by a process in which nothing is being copied and nothing is going
|
|
4910
|
+
* down. The interface would then draw a card for an action that is not
|
|
4911
|
+
* happening — the card that never went away. A stage that describes an
|
|
4912
|
+
* action must not outlive the process that performed it.
|
|
4913
|
+
*
|
|
4914
|
+
* @param {string} reason why it is being reset ("boot", "plugin reload", …)
|
|
4915
|
+
* @returns {object|null} the recorded stage, or null when nothing was stale
|
|
4916
|
+
*/
|
|
4917
|
+
resetStaleStage(reason) {
|
|
4918
|
+
const record = state.restartPhase
|
|
4919
|
+
const phase = record && typeof record === 'object' ? record.phase : null
|
|
4920
|
+
if (phase !== 'snapshotting' && phase !== 'restarting') return null
|
|
4921
|
+
let blocked = 0
|
|
4922
|
+
try {
|
|
4923
|
+
blocked = countBlockerTasks(gate().blockedBy)
|
|
4924
|
+
} catch {
|
|
4925
|
+
blocked = 0
|
|
4926
|
+
}
|
|
4927
|
+
return markRestartStage('waiting', {
|
|
4928
|
+
blocked,
|
|
4929
|
+
reason: reason ? 'boot: ' + reason : 'boot',
|
|
4930
|
+
})
|
|
4931
|
+
},
|
|
4370
4932
|
recordSnapshotProgress,
|
|
4371
4933
|
clearRestartStage,
|
|
4372
4934
|
restartStageInfo,
|
|
@@ -4389,6 +4951,8 @@ const titleOfSummary = (item) => {
|
|
|
4389
4951
|
blocking: info.blocking,
|
|
4390
4952
|
// The size of the pre-restart snapshot, on every stage (bytes).
|
|
4391
4953
|
snapshotBytes: info.snapshotBytes,
|
|
4954
|
+
// Why this stage ("deferred: …" = the restart went back to waiting).
|
|
4955
|
+
reason: info.reason,
|
|
4392
4956
|
progress: info.progress === null
|
|
4393
4957
|
? null
|
|
4394
4958
|
: { files: info.progress.files, bytes: info.progress.bytes, total: info.progress.total, percent: info.progress.percent, ageMs: info.progress.ageMs },
|
|
@@ -4467,6 +5031,10 @@ const titleOfSummary = (item) => {
|
|
|
4467
5031
|
* WHAT IS ON DISK RIGHT NOW: the backups of this host.
|
|
4468
5032
|
*/
|
|
4469
5033
|
backupsInfo,
|
|
5034
|
+
// RESTORE ONE COPY THE USER PICKED (they may know better than "the newest
|
|
5035
|
+
// proven one"): save the current state, restore the picked copy, request a
|
|
5036
|
+
// normal restart.
|
|
5037
|
+
manualRollback,
|
|
4470
5038
|
// Where the rescue scripts are, and whether the desktop shortcut exists.
|
|
4471
5039
|
rescueKitInfo,
|
|
4472
5040
|
// What this machine can do: can it create a one-off scheduled task? (The
|
|
@@ -4660,6 +5228,7 @@ const titleOfSummary = (item) => {
|
|
|
4660
5228
|
flushSessions,
|
|
4661
5229
|
releaseRequesters,
|
|
4662
5230
|
markDisposed: () => {
|
|
5231
|
+
stopRestartBusyWatch()
|
|
4663
5232
|
disposed = true
|
|
4664
5233
|
},
|
|
4665
5234
|
noteActivity,
|
|
@@ -5349,6 +5918,16 @@ export async function verifyRestartBoot(runtime, options = {}) {
|
|
|
5349
5918
|
if (record !== null) {
|
|
5350
5919
|
if (evaluated.ok) {
|
|
5351
5920
|
log.warn('the new instance verified itself:', { durationMs: record.durationMs, checks })
|
|
5921
|
+
// THIS SNAPSHOT HAS NOW PROVEN IT BOOTS: the restart it belongs to came
|
|
5922
|
+
// up, so it is the copy a failed boot may be rolled back to (user
|
|
5923
|
+
// decision, 2026-10-30). A FAILED boot never promotes anything.
|
|
5924
|
+
const snapshotDir = pending && pending.snapshot && typeof pending.snapshot.dir === 'string'
|
|
5925
|
+
? pending.snapshot.dir
|
|
5926
|
+
: ''
|
|
5927
|
+
if (snapshotDir !== '') {
|
|
5928
|
+
const promoted = markKnownGood(snapshotDir, { reason: 'boot-ok' })
|
|
5929
|
+
log.info('the snapshot of this restart is now known-good', { dir: snapshotDir, promoted })
|
|
5930
|
+
}
|
|
5352
5931
|
} else {
|
|
5353
5932
|
log.error('the new instance did NOT come up cleanly:', {
|
|
5354
5933
|
durationMs: record.durationMs,
|
|
@@ -6714,6 +7293,70 @@ function buildHandoffRoute(ctx, runtime, config) {
|
|
|
6714
7293
|
}
|
|
6715
7294
|
}
|
|
6716
7295
|
|
|
7296
|
+
/**
|
|
7297
|
+
* Build the POST `/rollback-to` route: restore ONE backup the user picked.
|
|
7298
|
+
*
|
|
7299
|
+
* It sits next to the health route and uses the same authentication rule
|
|
7300
|
+
* (loopback OR a request the connection service admits). Body:
|
|
7301
|
+
* `{ "name": "<backup directory name>" }`. The name has to be one this host
|
|
7302
|
+
* reports from the backup list — the route never takes a path, so `../` and
|
|
7303
|
+
* absolute paths are refused before anything is read. The answer carries the
|
|
7304
|
+
* receipt (what was restored, and the snapshot taken right before it):
|
|
7305
|
+
*
|
|
7306
|
+
* - 400 when no name / not a plain backup name was given,
|
|
7307
|
+
* - 404 when there is no such backup,
|
|
7308
|
+
* - 409 when the current state could not be saved first, or when another
|
|
7309
|
+
* rollback (the detached watchdog) holds the lock,
|
|
7310
|
+
* - 500 when the restore itself failed,
|
|
7311
|
+
* - 200 with `{ ok: true, receipt }` when the copy is back AND a normal
|
|
7312
|
+
* restart has been requested.
|
|
7313
|
+
*/
|
|
7314
|
+
function buildManualRollbackRoute(ctx, runtime, config) {
|
|
7315
|
+
const routePath = resolveManualRollbackPath(config)
|
|
7316
|
+
return {
|
|
7317
|
+
kind: 'exact',
|
|
7318
|
+
path: routePath,
|
|
7319
|
+
handler: (req, res) => {
|
|
7320
|
+
const remote = req && req.socket ? req.socket.remoteAddress : undefined
|
|
7321
|
+
const loopback = remote === undefined
|
|
7322
|
+
|| remote === '127.0.0.1'
|
|
7323
|
+
|| remote === '::1'
|
|
7324
|
+
|| remote === '::ffff:127.0.0.1'
|
|
7325
|
+
if (!loopback && !admittedByConnection(ctx, req)) {
|
|
7326
|
+
res.statusCode = 403
|
|
7327
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
7328
|
+
res.end('{"ok":false,"error":"loopback-only"}')
|
|
7329
|
+
return
|
|
7330
|
+
}
|
|
7331
|
+
if (!req || req.method !== 'POST') {
|
|
7332
|
+
res.statusCode = 405
|
|
7333
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
7334
|
+
res.setHeader('allow', 'POST')
|
|
7335
|
+
res.end('{"ok":false,"error":"method-not-allowed"}')
|
|
7336
|
+
return
|
|
7337
|
+
}
|
|
7338
|
+
void readJsonBody(req).then(async (payload) => {
|
|
7339
|
+
const body = payload && typeof payload === 'object' ? payload : {}
|
|
7340
|
+
const result = await runtime.manualRollback({ name: body.name })
|
|
7341
|
+
const status = result && Number.isFinite(result.status) ? result.status : 400
|
|
7342
|
+
res.statusCode = status
|
|
7343
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
7344
|
+
res.setHeader('cache-control', 'no-store')
|
|
7345
|
+
res.end(JSON.stringify({
|
|
7346
|
+
ok: result ? result.ok === true : false,
|
|
7347
|
+
error: result && result.error ? result.error : null,
|
|
7348
|
+
receipt: result && result.receipt ? result.receipt : null,
|
|
7349
|
+
restartRequested: Boolean(result && result.receipt && result.receipt.restartRequested),
|
|
7350
|
+
}))
|
|
7351
|
+
}).catch((error) => {
|
|
7352
|
+
res.statusCode = 400
|
|
7353
|
+
res.setHeader('content-type', 'application/json; charset=utf-8')
|
|
7354
|
+
res.end(JSON.stringify({ ok: false, error: describeError(error) }))
|
|
7355
|
+
})
|
|
7356
|
+
},
|
|
7357
|
+
}
|
|
7358
|
+
}
|
|
7359
|
+
|
|
6717
7360
|
/**
|
|
6718
7361
|
* Build the announcements route: the one-time announcement of a new feature.
|
|
6719
7362
|
*
|
|
@@ -7936,6 +8579,10 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
7936
8579
|
// The copy progress on its own as well, so a client that only wants
|
|
7937
8580
|
// the numbers does not have to walk the stage object.
|
|
7938
8581
|
progress: stage.progress,
|
|
8582
|
+
// THE FIRST SAFE COPY: what happened when this machine looked for a
|
|
8583
|
+
// known-good backup and found none (user decision, 2026-10-30).
|
|
8584
|
+
// Absent when nothing was ever tried, so an old client sees no change.
|
|
8585
|
+
...(live.firstInstallBackup ? { firstInstallBackup: live.firstInstallBackup } : {}),
|
|
7939
8586
|
updatedAt: live.updatedAt,
|
|
7940
8587
|
// The park route, so the client never has to guess the prefix, and
|
|
7941
8588
|
// the goals this plugin already disarmed (they are re-armed after
|
|
@@ -7945,6 +8592,9 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
7945
8592
|
// The route that withdraws ONE session's own restart request (the
|
|
7946
8593
|
// manual "cancel my restart" of the pill).
|
|
7947
8594
|
cancelUrl: resolveCancelPath(config),
|
|
8595
|
+
// THE ROUTE OF THE MANUAL ROLLBACK: POST a backup NAME and this
|
|
8596
|
+
// host saves the current state, restores that copy and restarts.
|
|
8597
|
+
rollbackUrl: resolveManualRollbackPath(config),
|
|
7948
8598
|
// The route that persists the one-time install choice, plus the
|
|
7949
8599
|
// value that is effective right now.
|
|
7950
8600
|
choiceUrl: resolveChoicePath(config),
|
|
@@ -8042,6 +8692,9 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
8042
8692
|
parkUrl: resolveParkPath(config),
|
|
8043
8693
|
recoverUrl: resolveRecoverPath(config),
|
|
8044
8694
|
cancelUrl: resolveCancelPath(config),
|
|
8695
|
+
// THE ROUTE OF THE MANUAL ROLLBACK: POST a backup NAME and this
|
|
8696
|
+
// host saves the current state, restores that copy and restarts.
|
|
8697
|
+
rollbackUrl: resolveManualRollbackPath(config),
|
|
8045
8698
|
choiceUrl: resolveChoicePath(config),
|
|
8046
8699
|
announcementsUrl: resolveAnnouncementsPath(config),
|
|
8047
8700
|
backupOnRestart: config.backupOnRestart !== false,
|
|
@@ -8133,6 +8786,10 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
8133
8786
|
// out there; it must be visible from outside the process, not only in
|
|
8134
8787
|
// a log line nobody reads.
|
|
8135
8788
|
watchdog: summarizeDisarm(runtime.watchdogDisarm),
|
|
8789
|
+
// THE FIRST SAFE COPY: what happened when this machine looked for a
|
|
8790
|
+
// known-good backup and found none (user decision, 2026-10-30). Absent
|
|
8791
|
+
// when nothing was ever tried.
|
|
8792
|
+
...(state.firstInstallBackup ? { firstInstallBackup: state.firstInstallBackup } : {}),
|
|
8136
8793
|
// THE RESTART STAGE, read from the health route as well: a script (or a
|
|
8137
8794
|
// user watching the console) can tell "waiting" from "backing up 120 MB
|
|
8138
8795
|
// of 156 MB" without opening the interface.
|
|
@@ -8171,6 +8828,7 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
8171
8828
|
const handoffRoute = buildHandoffRoute(ctx, runtime, config)
|
|
8172
8829
|
const choiceRoute = buildChoiceRoute(ctx, runtime, config)
|
|
8173
8830
|
const cancelRoute = buildCancelRoute(ctx, runtime, config)
|
|
8831
|
+
const manualRollbackRoute = buildManualRollbackRoute(ctx, runtime, config)
|
|
8174
8832
|
const announcementsRoute = buildAnnouncementsRoute(ctx, runtime, config)
|
|
8175
8833
|
const supervisorRoute = buildSupervisorRoute(ctx, runtime, config)
|
|
8176
8834
|
const rescueRoute = buildRescueRoute(ctx, runtime, config)
|
|
@@ -8197,6 +8855,7 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
8197
8855
|
{ name: 'handoff', route: handoffRoute },
|
|
8198
8856
|
{ name: 'choice', route: choiceRoute },
|
|
8199
8857
|
{ name: 'cancel', route: cancelRoute },
|
|
8858
|
+
{ name: 'manual-rollback', route: manualRollbackRoute },
|
|
8200
8859
|
{ name: 'announcements', route: announcementsRoute },
|
|
8201
8860
|
{ name: 'supervisor', route: supervisorRoute },
|
|
8202
8861
|
{ name: 'rescue', route: rescueRoute },
|
|
@@ -8418,7 +9077,23 @@ export function apply(ctx, rawConfig, options = {}) {
|
|
|
8418
9077
|
// says "back — carrying on". Written here (not after the recovery) so a boot
|
|
8419
9078
|
// that finds nothing to resume still shows the end of the story instead of a
|
|
8420
9079
|
// spinner left over from the old process.
|
|
8421
|
-
if (runtime.isNewBoot)
|
|
9080
|
+
if (runtime.isNewBoot) {
|
|
9081
|
+
runtime.markRestartStage('back')
|
|
9082
|
+
} else {
|
|
9083
|
+
// The SAME process is loading the plugin again: a stage left on disk by an
|
|
9084
|
+
// earlier attempt ('snapshotting' / 'restarting') describes an action that is
|
|
9085
|
+
// not running in this process, so it is reset instead of being inherited
|
|
9086
|
+
// (user report, 2026-10-29: a card that never went away).
|
|
9087
|
+
runtime.resetStaleStage('plugin reload')
|
|
9088
|
+
}
|
|
9089
|
+
|
|
9090
|
+
// THE FIRST SAFE COPY (user decision, 2026-10-30): a machine that has never
|
|
9091
|
+
// restarted has nothing a failed boot could roll back to, so a NEW host
|
|
9092
|
+
// process seeds one known-good copy. A reload seeds nothing, and the seed is
|
|
9093
|
+
// fire-and-forget: the boot path never waits for a 117 MB copy.
|
|
9094
|
+
void seedFirstInstallBackup(runtime, config).catch((error) => {
|
|
9095
|
+
log.warn('the first-install safe copy failed:', String(error))
|
|
9096
|
+
})
|
|
8422
9097
|
|
|
8423
9098
|
trackAgents(ctx, runtime, config)
|
|
8424
9099
|
|
|
@@ -8496,6 +9171,11 @@ export function apply(ctx, rawConfig, options = {}) {
|
|
|
8496
9171
|
sleep.controller.dispose()
|
|
8497
9172
|
persistSafely(runtime)
|
|
8498
9173
|
}, 'dsh-autorestart lifecycle')
|
|
9174
|
+
|
|
9175
|
+
// The runtime is handed back so a test can look at — and drive — the very
|
|
9176
|
+
// object the routes use (the cancel route is decided by ITS stage). Nothing in
|
|
9177
|
+
// the plugin system reads this value.
|
|
9178
|
+
return runtime
|
|
8499
9179
|
}
|
|
8500
9180
|
|
|
8501
9181
|
export {
|