dsh-autorestart 0.8.0 → 0.8.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.js CHANGED
@@ -49,9 +49,17 @@ import {
49
49
  setHandoff,
50
50
  setRecoveryOutcome,
51
51
  setPhase,
52
+ setRestartPhase,
53
+ setRestartProgress,
54
+ clearRestartPhase,
52
55
  stampResumedGoal,
53
56
  writeState,
54
57
  } from './lib/checkpoint.js'
58
+ import {
59
+ cancelRefusalReason,
60
+ describeRestartPhase,
61
+ RESTART_PHASES,
62
+ } from './lib/restart-phase.js'
55
63
  import {
56
64
  bootPlan,
57
65
  composeHandoff,
@@ -117,13 +125,18 @@ import {
117
125
  } from './lib/watchdog-tier.js'
118
126
  import {
119
127
  buildPending,
128
+ clearPending,
120
129
  clearReady,
121
- readReady,
130
+ readPending,
122
131
  readReport,
123
132
  writePending,
124
- writeReady,
125
133
  writeReport,
126
134
  } from './lib/watchdog.js'
135
+ import {
136
+ disarmWatchdog as runDisarmWatchdog,
137
+ markReady,
138
+ readDisarmRecord,
139
+ } from './lib/watchdog-disarm.js'
127
140
  import {
128
141
  ARBITERS,
129
142
  planWatchdogArbiter,
@@ -374,7 +387,7 @@ function snapGraceChoice(value, fallback = DEFAULT_RESTART_GRACE_MS) {
374
387
  * `pluginVersion()`), but a bundled installation may not ship that file — so
375
388
  * the constant below is the last resort. A test keeps the two in sync.
376
389
  */
377
- const PLUGIN_VERSION_FALLBACK = '0.8.0'
390
+ const PLUGIN_VERSION_FALLBACK = '0.8.2'
378
391
 
379
392
  /**
380
393
  * The version of the graceful-shutdown FEATURE, not the version of the package.
@@ -1398,12 +1411,16 @@ function backupsDirFor(dir) {
1398
1411
  * error: string|null,
1399
1412
  * }}
1400
1413
  */
1401
- export function takePreRestartSnapshot({
1414
+ export async function takePreRestartSnapshot({
1402
1415
  config = {},
1403
1416
  stateDir = '',
1404
1417
  log = console,
1405
1418
  profile = null,
1406
1419
  createBackup: createBackupFn = createBackup,
1420
+ // The REAL progress of the copy, and the way out of it: both are handed to
1421
+ // the backup itself, so the numbers on screen are the numbers on disk.
1422
+ onProgress = null,
1423
+ shouldCancel = null,
1407
1424
  } = {}) {
1408
1425
  const startedAt = Date.now()
1409
1426
  const info = profile && typeof profile === 'object'
@@ -1427,17 +1444,33 @@ export function takePreRestartSnapshot({
1427
1444
  }
1428
1445
  let backup = null
1429
1446
  try {
1430
- backup = createBackupFn({
1447
+ backup = await createBackupFn({
1431
1448
  profileDir,
1432
1449
  backupsDir: backupsDirFor(stateDir),
1433
1450
  includePlugins: config.backupIncludePlugins !== false,
1434
1451
  maxBackups: config.maxBackups,
1452
+ onProgress: typeof onProgress === 'function' ? onProgress : null,
1453
+ shouldCancel: typeof shouldCancel === 'function' ? shouldCancel : null,
1435
1454
  })
1436
1455
  } catch (error) {
1437
1456
  // A backup that throws must not take the restart down with it.
1438
1457
  backup = { ok: false, error: String(error) }
1439
1458
  }
1440
1459
  const durationMs = Math.max(0, Date.now() - startedAt)
1460
+ if (backup && backup.cancelled === true) {
1461
+ // The user called it off: the half-written snapshot was already removed by
1462
+ // the copy loop, and there is nothing to report as a rollback point.
1463
+ log.warn('the pre-restart backup was cancelled by the user', { durationMs })
1464
+ return {
1465
+ backup,
1466
+ profileDir,
1467
+ profile: info,
1468
+ requested: true,
1469
+ durationMs,
1470
+ cancelled: true,
1471
+ error: null,
1472
+ }
1473
+ }
1441
1474
  if (backup && backup.ok === true) {
1442
1475
  log.warn('pre-restart backup written', {
1443
1476
  dir: backup.dir,
@@ -1645,6 +1678,17 @@ export function createRuntime(ctx, config, options = {}) {
1645
1678
  let recoverTimer
1646
1679
  let disposed = false
1647
1680
  let lastPersistError
1681
+ /**
1682
+ * THE LAST DISARM of the watchdog: the record of "the host is up, so nothing
1683
+ * may roll it back" (see lib/watchdog-disarm.js). It is kept on `runtime` so
1684
+ * the settings page and the diagnostics can show a FAILED disarm instead of
1685
+ * the silence that caused the 0.8.2 incident.
1686
+ */
1687
+ let watchdogDisarm = readDisarmRecord(path.dirname(stateFile))
1688
+ /** The disarm that is currently running (one at a time; null when idle). */
1689
+ let disarmInFlight = null
1690
+ /** A disarm was asked for while one was running: run once more afterwards. */
1691
+ let disarmPendingAgain = false
1648
1692
  /**
1649
1693
  * The tool registrations that failed. A failed registration is recorded and
1650
1694
  * logged, never thrown: the plugin still loads and the other tools still work.
@@ -2605,6 +2649,93 @@ const titleOfSummary = (item) => {
2605
2649
  */
2606
2650
  const stateDirOf = () => path.dirname(stateFile)
2607
2651
 
2652
+ /**
2653
+ * THE STAGE LINE of a restart ("waiting / backing up / restarting / back").
2654
+ *
2655
+ * It is written ONCE per stage change and persisted IMMEDIATELY, because the
2656
+ * whole point of it is the first second after the click: a stage that reaches
2657
+ * the interface only after the 156 MB snapshot would tell the user nothing
2658
+ * about the ten seconds they spent wondering whether the button worked.
2659
+ *
2660
+ * @param {string} phase one of RESTART_PHASES
2661
+ * @param {{ blocked?: number, now?: number }} [options]
2662
+ * @returns {object|null} the recorded stage
2663
+ */
2664
+ const markRestartStage = (phase, { blocked = 0, now = Date.now() } = {}) => {
2665
+ if (!RESTART_PHASES.includes(phase)) return null
2666
+ const record = setRestartPhase(state, { phase, blocked, now })
2667
+ persist()
2668
+ return record
2669
+ }
2670
+
2671
+ /**
2672
+ * Record the REAL progress of the running snapshot (files and bytes).
2673
+ *
2674
+ * The copy loop calls this between files; the checkpoint is written with a
2675
+ * small throttle, because a state write per copied file would slow the copy
2676
+ * down for no gain. The in-memory record is always current, so a `state=1`
2677
+ * poll that arrives between two writes still sees the newest numbers.
2678
+ *
2679
+ * @param {{ files?: number, bytes?: number, total?: number, filesTotal?: number, now?: number }} progress
2680
+ */
2681
+ let lastProgressPersist = 0
2682
+ const recordSnapshotProgress = (progress) => {
2683
+ const now = Date.now()
2684
+ const record = setRestartProgress(state, { ...progress, now })
2685
+ if (now - lastProgressPersist >= 250) {
2686
+ lastProgressPersist = now
2687
+ persist()
2688
+ }
2689
+ return record
2690
+ }
2691
+
2692
+ /** Forget the stage (cancel, or a restart that could not be started). */
2693
+ const clearRestartStage = () => {
2694
+ if (clearRestartPhase(state)) persist()
2695
+ }
2696
+
2697
+ /**
2698
+ * The stage as every surface shows it (state route, health route, client).
2699
+ *
2700
+ * @returns {object} the described stage (always an object, never null)
2701
+ */
2702
+ const restartStageInfo = () => {
2703
+ const record = state.restartPhase
2704
+ const blocked = (() => {
2705
+ try {
2706
+ return countBlockerTasks(gate().blockedBy)
2707
+ } catch {
2708
+ return 0
2709
+ }
2710
+ })()
2711
+ return describeRestartPhase({
2712
+ phase: record && typeof record === 'object' ? record.phase : 'idle',
2713
+ startedAt: record && typeof record === 'object' ? record.startedAt : null,
2714
+ at: record && typeof record === 'object' ? record.at : null,
2715
+ progress: record && typeof record === 'object' ? record.progress : null,
2716
+ blocking: record && typeof record === 'object' && Number.isFinite(record.blocked) ? record.blocked : blocked,
2717
+ now: Date.now(),
2718
+ })
2719
+ }
2720
+
2721
+ /**
2722
+ * A cancel request that arrived while the snapshot was being written.
2723
+ *
2724
+ * The backup loop asks this between files: the copy stops, its half-written
2725
+ * directory is removed, and the host never restarts. A flag (not a thrown
2726
+ * error) because the loop is synchronous and must finish its bookkeeping.
2727
+ */
2728
+ let snapshotCancelRequested = false
2729
+ const requestSnapshotCancel = () => {
2730
+ snapshotCancelRequested = true
2731
+ return true
2732
+ }
2733
+ const snapshotCancelled = () => snapshotCancelRequested === true
2734
+ const resetSnapshotCancel = () => {
2735
+ snapshotCancelRequested = false
2736
+ }
2737
+
2738
+
2608
2739
  /**
2609
2740
  * The durable record of the last restart: the snapshot that was taken, the
2610
2741
  * verdict of the new instance and its four boot checks.
@@ -2616,17 +2747,14 @@ const titleOfSummary = (item) => {
2616
2747
  /**
2617
2748
  * The version of THIS plugin (the one the user runs).
2618
2749
  *
2750
+ * It delegates to the module-level reader ON PURPOSE: a local copy used to
2751
+ * look for `../package.json`, which is one directory ABOVE the plugin, so
2752
+ * every answer was `unknown` — and the health route reported that instead of
2753
+ * the version that is really loaded.
2754
+ *
2619
2755
  * @returns {string}
2620
2756
  */
2621
- const pluginVersion = () => {
2622
- try {
2623
- const raw = fs.readFileSync(new URL('../package.json', import.meta.url), 'utf8')
2624
- const parsed = JSON.parse(raw)
2625
- return typeof parsed.version === 'string' && parsed.version !== '' ? parsed.version : 'unknown'
2626
- } catch {
2627
- return 'unknown'
2628
- }
2629
- }
2757
+ const ownVersion = () => pluginVersion()
2630
2758
 
2631
2759
  /**
2632
2760
  * Write (or update) the durable record of the restart.
@@ -2960,7 +3088,7 @@ const titleOfSummary = (item) => {
2960
3088
  * @returns {string}
2961
3089
  */
2962
3090
  const diagnosticsText = () => buildDiagnostics({
2963
- version: pluginVersion(),
3091
+ version: ownVersion(),
2964
3092
  platform: env.platform,
2965
3093
  hostKind: hostKindOf().kind,
2966
3094
  isDesktopApp: env.isDesktopApp === true,
@@ -2970,6 +3098,9 @@ const titleOfSummary = (item) => {
2970
3098
  lastRestart: lastRestartVerdict(),
2971
3099
  stateDir: stateDirOf(),
2972
3100
  watchdog: watchdogInfo(),
3101
+ // DID THE DISARM WORK? A failed disarm leaves an armed watchdog behind,
3102
+ // and that is the first thing a bug report has to show.
3103
+ watchdogDisarm: summarizeDisarm(watchdogDisarm),
2973
3104
  })
2974
3105
 
2975
3106
  /**
@@ -3540,13 +3671,50 @@ const titleOfSummary = (item) => {
3540
3671
  // app relaunch and the custom restart command share this single entry, so
3541
3672
  // a hosted host that is restarted by a command is just as recoverable as
3542
3673
  // the desktop app.
3674
+ //
3675
+ // THE STAGE IS ON SCREEN BEFORE THE COPY STARTS, and the copy reports the
3676
+ // files and bytes it really writes. That is the fix of "I clicked restart
3677
+ // and nothing happened for ten seconds": the ten seconds ARE the snapshot,
3678
+ // and now they say so.
3543
3679
  const stateDir = path.dirname(stateFile)
3544
- const snapshot = snapshotFn({
3545
- config,
3546
- stateDir,
3547
- log,
3548
- profile: resolveProfileDirDetailed(),
3549
- })
3680
+ resetSnapshotCancel()
3681
+ markRestartStage('snapshotting')
3682
+ let cancelled = false
3683
+ let snapshot
3684
+ try {
3685
+ snapshot = await snapshotFn({
3686
+ config,
3687
+ stateDir,
3688
+ log,
3689
+ profile: resolveProfileDirDetailed(),
3690
+ onProgress: (progress) => {
3691
+ recordSnapshotProgress(progress)
3692
+ },
3693
+ shouldCancel: () => snapshotCancelled(),
3694
+ })
3695
+ } finally {
3696
+ // The flag belongs to ONE snapshot; a stale `true` would cancel the
3697
+ // next restart before it began.
3698
+ const wasCancelled = snapshotCancelled()
3699
+ resetSnapshotCancel()
3700
+ // Two ways a copy can be called off: the flag this process set (the
3701
+ // cancel route) or the snapshot reporting it itself. Both mean the same
3702
+ // thing, and both must stop the restart.
3703
+ cancelled = wasCancelled || (snapshot !== null && typeof snapshot === 'object' && snapshot.cancelled === true)
3704
+ }
3705
+ if (cancelled) {
3706
+ // The user called it off while the files were being copied: the
3707
+ // half-written snapshot is already gone (createBackup removes it), the
3708
+ // host does not restart, and the request is WITHDRAWN — a restart the
3709
+ // user cancelled must not be retried five seconds later.
3710
+ log.warn('the restart was cancelled while the snapshot was being written')
3711
+ clearRestartStage()
3712
+ releaseRequesters()
3713
+ cancelRestart(state, { now: Date.now() })
3714
+ persist()
3715
+ stopPolling()
3716
+ return
3717
+ }
3550
3718
  // The snapshot is the rollback point of BOTH branches.
3551
3719
  const backup = snapshot && snapshot.backup ? snapshot.backup : null
3552
3720
  const profileDir = snapshot && snapshot.profileDir ? snapshot.profileDir : null
@@ -3677,6 +3845,10 @@ const titleOfSummary = (item) => {
3677
3845
  // looked at "the spawn did not throw", so a command that does not
3678
3846
  // exist or that exits with an error was reported as a successful
3679
3847
  // restart.
3848
+ //
3849
+ // FROM HERE ON THE RESTART CANNOT BE CANCELLED, and the interface says
3850
+ // so: the host is on its way out.
3851
+ markRestartStage('restarting')
3680
3852
  const outcome = await runCommandFn({
3681
3853
  command,
3682
3854
  timeoutMs: config.restartCommandTimeoutMs,
@@ -3814,8 +3986,15 @@ const titleOfSummary = (item) => {
3814
3986
  watchScript,
3815
3987
  })
3816
3988
  log.warn('scheduled detached restart:', result)
3817
- if (!result || result.ok !== true) rollbackRestart((result && result.error) || 'detached restart refused')
3818
- else armGateWatchdog()
3989
+ if (!result || result.ok !== true) {
3990
+ markRestartStage('waiting', { blocked: 0 })
3991
+ rollbackRestart((result && result.error) || 'detached restart refused')
3992
+ } else {
3993
+ // The helper is scheduled: the host is going down, and from here on a
3994
+ // cancel would be a promise this plugin cannot keep.
3995
+ markRestartStage('restarting')
3996
+ armGateWatchdog()
3997
+ }
3819
3998
  }
3820
3999
  } catch (error) {
3821
4000
  rollbackRestart(String(error))
@@ -4059,6 +4238,157 @@ const titleOfSummary = (item) => {
4059
4238
  get bootId() {
4060
4239
  return bootId
4061
4240
  },
4241
+ /**
4242
+ * THE STATE DIRECTORY of this boot — the checkpoint, the watchdog markers
4243
+ * and the scheduling-item leftovers all live in it.
4244
+ *
4245
+ * It is exposed ON PURPOSE: the health route is a module-level function
4246
+ * and used to read the directory from a variable that only existed inside
4247
+ * this factory. The lookup threw `ReferenceError: stateDirOf is not
4248
+ * defined` on every request, an empty `catch {}` ate it, and a healthy
4249
+ * host left an armed watchdog behind (the 0.8.2 incident). Anything a
4250
+ * route needs must be reachable through `runtime`, never through a
4251
+ * closure it cannot see.
4252
+ */
4253
+ get stateDir() {
4254
+ return path.dirname(stateFile)
4255
+ },
4256
+ /**
4257
+ * The version of the plugin that is actually LOADED in this process.
4258
+ *
4259
+ * The health route reports it so a real-machine check can tell "the new
4260
+ * code is running" from "the old code is still loaded".
4261
+ */
4262
+ get pluginVersion() {
4263
+ return ownVersion()
4264
+ },
4265
+ /**
4266
+ * DISARM THE WATCHDOG: "this host is up, so nothing may roll it back".
4267
+ *
4268
+ * The ready marker is written on the FIRST tick of the first attempt (so a
4269
+ * caller inside an HTTP route has it on disk before it answers), the
4270
+ * pending marker and the scheduling item are cleaned with retries, and the
4271
+ * record is kept on `runtime` and written next to the markers.
4272
+ *
4273
+ * One run at a time: a second call while one is running returns the same
4274
+ * promise instead of starting a second `schtasks` dance.
4275
+ *
4276
+ * @param {string} [source] where the call came from ('boot', 'health-route', …)
4277
+ * @returns {Promise<object>} the record
4278
+ */
4279
+ disarmWatchdog(source = 'boot') {
4280
+ if (disarmInFlight !== null) {
4281
+ // One run at a time (two `schtasks /Delete` calls fight each other),
4282
+ // but a request that arrives DURING a run is remembered: a pending
4283
+ // marker written in the meantime must still be cleaned, not lost.
4284
+ disarmPendingAgain = true
4285
+ return disarmInFlight
4286
+ }
4287
+ const one = (where) => {
4288
+ const port = Number(runtime.webPort)
4289
+ return runDisarmWatchdog({
4290
+ stateDir: stateDirOf(),
4291
+ bootId,
4292
+ port: Number.isFinite(port) && port > 0 ? port : null,
4293
+ source: where,
4294
+ log,
4295
+ })
4296
+ .then((record) => {
4297
+ watchdogDisarm = record
4298
+ if (record.ok === true) {
4299
+ log.info('the watchdog is disarmed (' + where + ')', {
4300
+ attempts: record.attempts,
4301
+ tier: record.tier,
4302
+ })
4303
+ }
4304
+ return record
4305
+ })
4306
+ .catch((error) => {
4307
+ // The disarm must never reject: the health route and the boot path
4308
+ // both call it without a `catch`.
4309
+ watchdogDisarm = {
4310
+ at: new Date().toISOString(),
4311
+ source: String(where),
4312
+ bootId,
4313
+ attempts: 0,
4314
+ ok: false,
4315
+ ready: false,
4316
+ pending: null,
4317
+ tier: null,
4318
+ scheduleRemoved: null,
4319
+ scheduleError: null,
4320
+ errors: [{
4321
+ attempt: 0,
4322
+ step: 'disarm',
4323
+ name: error && error.name ? String(error.name) : 'Error',
4324
+ message: error && error.message ? String(error.message) : String(error),
4325
+ }],
4326
+ }
4327
+ log.warn('the watchdog could not be disarmed:', String(error))
4328
+ return watchdogDisarm
4329
+ })
4330
+ }
4331
+ const drive = async () => {
4332
+ let record = await one(source)
4333
+ while (disarmPendingAgain === true) {
4334
+ disarmPendingAgain = false
4335
+ record = await one(String(source) + '+again')
4336
+ }
4337
+ return record
4338
+ }
4339
+ const running = drive().then(
4340
+ (record) => {
4341
+ disarmInFlight = null
4342
+ disarmPendingAgain = false
4343
+ return record
4344
+ },
4345
+ (error) => {
4346
+ disarmInFlight = null
4347
+ disarmPendingAgain = false
4348
+ throw error
4349
+ },
4350
+ )
4351
+ disarmInFlight = running
4352
+ return running
4353
+ },
4354
+ /**
4355
+ * The record of the last disarm (null when this host never disarmed one).
4356
+ */
4357
+ get watchdogDisarm() {
4358
+ return watchdogDisarm
4359
+ },
4360
+ // THE STAGE OF A RESTART, as every surface reads it. `markRestartStage`
4361
+ // writes it and persists immediately (that is what makes the first stage
4362
+ // appear within the first second after the click), `recordSnapshotProgress`
4363
+ // carries the REAL bytes of the running copy, and `restartStageInfo` is the
4364
+ // single description (text included) that the state route, the health route
4365
+ // and the settings page share.
4366
+ markRestartStage,
4367
+ recordSnapshotProgress,
4368
+ clearRestartStage,
4369
+ restartStageInfo,
4370
+ requestSnapshotCancel,
4371
+ snapshotCancelled,
4372
+ resetSnapshotCancel,
4373
+ /**
4374
+ * The stage of a restart, cut down for the health route (no long texts).
4375
+ *
4376
+ * @returns {object} `{ phase, at, progress, slow, cancellable }`
4377
+ */
4378
+ restartStageBrief() {
4379
+ const info = restartStageInfo()
4380
+ return {
4381
+ phase: info.phase,
4382
+ at: info.at,
4383
+ elapsedMs: info.elapsedMs,
4384
+ progress: info.progress === null
4385
+ ? null
4386
+ : { files: info.progress.files, bytes: info.progress.bytes, total: info.progress.total, percent: info.progress.percent, ageMs: info.progress.ageMs },
4387
+ slow: info.slow,
4388
+ cancellable: info.cancellable,
4389
+ cancelReason: info.cancelReason,
4390
+ }
4391
+ },
4062
4392
  /**
4063
4393
  * Where the effective `backupIncludePlugins` value came from:
4064
4394
  * `config` (the profile row), `fallback-file` (the degraded path) or
@@ -5149,6 +5479,10 @@ function registerTools(ctx, runtime, config) {
5149
5479
  if (typeof args.defer_seconds === 'number' && Number.isFinite(args.defer_seconds) && args.defer_seconds > 0) {
5150
5480
  state.restart.requestedAt = now + Math.min(args.defer_seconds, 3600) * 1000
5151
5481
  }
5482
+ // THE FIRST STAGE, RIGHT NOW — not when the backup starts. The interface
5483
+ // polls this out of the state route, so the click produces a visible
5484
+ // answer in well under a second.
5485
+ runtime.markRestartStage('waiting', { blocked: countBlockerTasks(runtime.gate().blockedBy) })
5152
5486
  runtime.persist()
5153
5487
  runtime.startPolling()
5154
5488
  log.info('restart requested', { generation: state.generation, sessionId, resume, parkGoals })
@@ -6077,6 +6411,9 @@ function buildParkRoute(ctx, runtime, config) {
6077
6411
  resume: true,
6078
6412
  now: Date.now(),
6079
6413
  })
6414
+ // The first stage, in the same breath as the request: the answer to
6415
+ // this POST is already "waiting for the sessions", never silence.
6416
+ runtime.markRestartStage('waiting', { blocked: blockingSessionIds(runtime).length })
6080
6417
  runtime.persist()
6081
6418
  runtime.startPolling()
6082
6419
  }
@@ -6172,6 +6509,28 @@ function buildCancelRoute(ctx, runtime, config) {
6172
6509
  })
6173
6510
  return
6174
6511
  }
6512
+ // THE STAGE DECIDES WHETHER A CANCEL IS HONEST.
6513
+ //
6514
+ // While the snapshot is being written the copy is stopped between two
6515
+ // files and its half-written directory is removed — the host never
6516
+ // restarts. Once the host is on its way out (`restarting`) a cancel
6517
+ // would be a promise this plugin cannot keep, so it is REFUSED with the
6518
+ // reason, and the button is greyed out on the same answer.
6519
+ const stage = runtime.restartStageInfo()
6520
+ if (stage.cancellable !== true) {
6521
+ const reason = cancelRefusalReason(stage.phase) || 'the host is already restarting'
6522
+ runtime.log.warn(`the cancel route refused the request of session ${sessionId}: ${reason}`)
6523
+ answer(409, {
6524
+ ok: false,
6525
+ cancelled: false,
6526
+ reason: 'not-cancellable',
6527
+ stage: stage.phase,
6528
+ error: reason,
6529
+ remainingRequesters: 0,
6530
+ sessionId,
6531
+ })
6532
+ return
6533
+ }
6175
6534
  const outcome = cancelOwnRestart(runtime.state, { sessionId, now: Date.now() })
6176
6535
  if (!outcome.removed) {
6177
6536
  // Nothing was changed: the caller is not a requester (or the
@@ -6194,7 +6553,14 @@ function buildCancelRoute(ctx, runtime, config) {
6194
6553
  }
6195
6554
  // Release ONLY this session's continuation request.
6196
6555
  runtime.releaseRequesters([sessionId])
6197
- if (outcome.cancelled) runtime.stopPolling()
6556
+ if (outcome.cancelled) {
6557
+ // The whole restart is off. If the snapshot is being written right
6558
+ // now, the copy stops at the next file and removes its half-written
6559
+ // directory (the host never restarts); otherwise the stage just goes.
6560
+ if (stage.phase === 'snapshotting') runtime.requestSnapshotCancel()
6561
+ runtime.clearRestartStage()
6562
+ runtime.stopPolling()
6563
+ }
6198
6564
  runtime.persist()
6199
6565
  runtime.log.info(
6200
6566
  outcome.cancelled
@@ -7377,6 +7743,39 @@ function buildSleepRoute(ctx, runtime, config, sleep) {
7377
7743
  }
7378
7744
 
7379
7745
  /** Expose a tiny read-only health route for humans and verification scripts. */
7746
+ /**
7747
+ * The disarm record, cut down to what a reader outside the process needs.
7748
+ *
7749
+ * The health route answers with it, so a real-machine check can tell "the
7750
+ * watchdog was stood down" from "it is still armed" without reading a log.
7751
+ *
7752
+ * @param {object|null} record
7753
+ * @returns {object|null}
7754
+ */
7755
+ function summarizeDisarm(record) {
7756
+ if (!record || typeof record !== 'object') return null
7757
+ const errors = Array.isArray(record.errors) ? record.errors : []
7758
+ return {
7759
+ ok: record.ok === true,
7760
+ at: typeof record.at === 'string' ? record.at : null,
7761
+ source: typeof record.source === 'string' ? record.source : null,
7762
+ attempts: Number.isFinite(Number(record.attempts)) ? Number(record.attempts) : 0,
7763
+ ready: record.ready === true,
7764
+ pending: record.pending === true,
7765
+ tier: typeof record.tier === 'string' ? record.tier : null,
7766
+ scheduleRemoved: record.scheduleRemoved === true ? true : (record.scheduleRemoved === false ? false : null),
7767
+ // Only the first failure: the route answer must stay small, and one named
7768
+ // error is what makes the failure diagnosable.
7769
+ error: errors.length > 0
7770
+ ? {
7771
+ step: typeof errors[0].step === 'string' ? errors[0].step : null,
7772
+ name: typeof errors[0].name === 'string' ? errors[0].name : null,
7773
+ message: typeof errors[0].message === 'string' ? errors[0].message : null,
7774
+ }
7775
+ : null,
7776
+ }
7777
+ }
7778
+
7380
7779
  function registerHealthRoute(ctx, runtime, config, sleep) {
7381
7780
  const routePath = typeof config.healthPath === 'string' && config.healthPath.startsWith('/')
7382
7781
  ? config.healthPath
@@ -7471,6 +7870,10 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
7471
7870
  }
7472
7871
  }
7473
7872
  const verdict = runtime.gate()
7873
+ // The restart stage (waiting / snapshotting / restarting / back), the
7874
+ // real snapshot progress and the "is a cancel still honest" answer —
7875
+ // computed ONCE per answer so every field of the payload agrees.
7876
+ const stage = runtime.restartStageInfo()
7474
7877
  // Best effort: label the blocking sessions with their titles when the
7475
7878
  // session controller knows them. The title is optional, so an entry
7476
7879
  // without one is sent exactly as the gate produced it. The goal
@@ -7506,6 +7909,14 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
7506
7909
  // long the restart has been waiting, why, and who holds it — the
7507
7910
  // request itself is never dropped.
7508
7911
  longWait: runtime.longWaitInfo(),
7912
+ // THE STAGE OF THE RESTART, for the pill and the settings page:
7913
+ // which of the four stages is on screen, the real copy progress of
7914
+ // the snapshot, whether it is going slower than expected, and
7915
+ // whether a cancel would still be honest (`cancellable`).
7916
+ restartPhase: stage,
7917
+ // The copy progress on its own as well, so a client that only wants
7918
+ // the numbers does not have to walk the stage object.
7919
+ progress: stage.progress,
7509
7920
  updatedAt: live.updatedAt,
7510
7921
  // The park route, so the client never has to guess the prefix, and
7511
7922
  // the goals this plugin already disarmed (they are re-armed after
@@ -7630,27 +8041,64 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
7630
8041
  // THE READY MARKER: this host answers its own health route, so it is
7631
8042
  // up. The watchdog (which waits for the relaunched host) sees the
7632
8043
  // marker and stands down — no rollback of a host that is fine.
8044
+ //
8045
+ // THE STATE DIRECTORY COMES FROM `runtime`, NEVER FROM A CLOSURE:
8046
+ // 0.8.2 read it through a variable that only existed inside the runtime
8047
+ // factory, so every request threw `ReferenceError: stateDirOf is not
8048
+ // defined`, the `catch {}` swallowed it, and a healthy host left the
8049
+ // pending marker AND the scheduled task behind (the watchdog would have
8050
+ // rolled a healthy profile back). The failure is recorded now, and the
8051
+ // marker is written first because the marker alone is what stops a
8052
+ // rollback.
7633
8053
  try {
7634
- const marker = readReady(stateDirOf())
7635
- if (!marker || marker.bootId !== runtime.bootId) {
7636
- writeReady(stateDirOf(), { bootId: runtime.bootId, port: runtime.port ?? null })
7637
- // The host is up: the watchdog has nothing to do. Its scheduling
7638
- // item and its pending marker go NOW — nothing may be left behind.
7639
- if (readPending(stateDirOf()) !== null) {
7640
- clearPending(stateDirOf())
7641
- try {
7642
- removeScheduledTask({ taskName: 'dsh-autorestart-watchdog' })
7643
- } catch {
7644
- // The watchdog removes it itself when it runs.
8054
+ const stateDir = typeof runtime.stateDir === 'string' ? runtime.stateDir : ''
8055
+ if (stateDir !== '') {
8056
+ const marked = markReady({
8057
+ stateDir,
8058
+ bootId: runtime.bootId,
8059
+ port: runtime.webPort ?? null,
8060
+ })
8061
+ if (marked.ok !== true) {
8062
+ runtime.watchdogReadyError = {
8063
+ at: Date.now(),
8064
+ message: marked.detail,
8065
+ name: marked.error && marked.error.name ? String(marked.error.name) : null,
7645
8066
  }
7646
8067
  }
8068
+ // The marker alone stands the watchdog down; the pending marker and
8069
+ // the scheduling item are cleaned with retries in the background, so
8070
+ // `schtasks` never delays this answer.
8071
+ runtime.disarmWatchdog('health-route')
8072
+ }
8073
+ } catch (error) {
8074
+ // NEVER silent: a swallowed error here leaves an armed watchdog behind.
8075
+ // It is recorded on `runtime` (this scope has no logger) so the settings
8076
+ // page and the diagnostics can show it instead of pretending all is well.
8077
+ runtime.watchdogReadyError = {
8078
+ at: Date.now(),
8079
+ name: error && error.name ? String(error.name) : 'Error',
8080
+ message: String(error && error.message ? error.message : error),
8081
+ }
8082
+ // One retry with the bare minimum: the marker alone is what the
8083
+ // watchdog actually waits for.
8084
+ try {
8085
+ markReady({
8086
+ stateDir: typeof runtime.stateDir === 'string' ? runtime.stateDir : '',
8087
+ bootId: runtime.bootId,
8088
+ port: runtime.webPort ?? null,
8089
+ })
8090
+ runtime.watchdogReadyError.retry = 'ok'
8091
+ } catch (again) {
8092
+ runtime.watchdogReadyError.retry = String(again && again.message ? again.message : again)
7647
8093
  }
7648
- } catch {
7649
- // The marker is a convenience: the watchdog also asks the health route.
7650
8094
  }
7651
8095
  const body = JSON.stringify({
7652
8096
  ok: true,
7653
8097
  plugin: name,
8098
+ // The version of the code that is LOADED (not the fallback constant):
8099
+ // a real-machine check can tell "the new build is running" from "the
8100
+ // old one is still in memory".
8101
+ version: runtime.pluginVersion ?? PLUGIN_VERSION_FALLBACK,
7654
8102
  bootId: runtime.bootId,
7655
8103
  pid: process.pid,
7656
8104
  phase: state.phase,
@@ -7662,6 +8110,15 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
7662
8110
  manualRestartBlocks: runtime.manualRestartBlocks ?? 0,
7663
8111
  indexTaps: runtime.indexTaps ?? 0,
7664
8112
  configSchema: Config !== undefined,
8113
+ // DID THE DISARM WORK? A failed disarm means an armed watchdog is still
8114
+ // out there; it must be visible from outside the process, not only in
8115
+ // a log line nobody reads.
8116
+ watchdog: summarizeDisarm(runtime.watchdogDisarm),
8117
+ // THE RESTART STAGE, read from the health route as well: a script (or a
8118
+ // user watching the console) can tell "waiting" from "backing up 120 MB
8119
+ // of 156 MB" without opening the interface.
8120
+ restartPhase: runtime.restartStageBrief(),
8121
+ progress: runtime.restartStageBrief().progress,
7665
8122
  schemastery: Config === undefined
7666
8123
  ? (schemastery === undefined ? 'unresolved' : 'construction-failed')
7667
8124
  : (schemasterySource === 'bare-specifier' ? 'bare' : 'asar-fallback'),
@@ -7938,6 +8395,12 @@ export function apply(ctx, rawConfig, options = {}) {
7938
8395
  // Record this boot in the checkpoint even when there is nothing to recover.
7939
8396
  persistSafely(runtime)
7940
8397
 
8398
+ // THE LAST STAGE OF THE RESTART: this host IS the new one, so the interface
8399
+ // says "back — carrying on". Written here (not after the recovery) so a boot
8400
+ // that finds nothing to resume still shows the end of the story instead of a
8401
+ // spinner left over from the old process.
8402
+ if (runtime.isNewBoot) runtime.markRestartStage('back')
8403
+
7941
8404
  trackAgents(ctx, runtime, config)
7942
8405
 
7943
8406
  // The boot recovery pass is the plugin's primary boot duty, so it is scheduled
@@ -7955,8 +8418,7 @@ export function apply(ctx, rawConfig, options = {}) {
7955
8418
  })
7956
8419
  sleep.restore()
7957
8420
  }
7958
- if (runtime.isNewBoot && config.resumeOnBoot) {
7959
- runtime.recoverTimer = ctx.timeout(() => {
8421
+ if (runtime.isNewBoot && config.resumeOnBoot) { runtime.recoverTimer = ctx.timeout(() => {
7960
8422
  recoverAfterRestart(ctx, runtime, config)
7961
8423
  .then(() => {
7962
8424
  runtime.recovery = { ran: true, ok: true, detail: 'the boot recovery finished' }
@@ -7995,6 +8457,19 @@ export function apply(ctx, rawConfig, options = {}) {
7995
8457
  registerSystemPrompt(ctx, runtime)
7996
8458
  registerHealthRoute(ctx, runtime, config, sleep)
7997
8459
 
8460
+ // DISARM THE WATCHDOG AT BOOT — WITHOUT WAITING FOR ANYBODY TO ASK.
8461
+ //
8462
+ // This is the fix of the 0.8.2 incident: the ready marker and the cleanup of
8463
+ // the pending marker / scheduled task hung on the health route being hit.
8464
+ // The plugin is fully loaded at this point (every route is registered, the
8465
+ // boot recovery is scheduled), so this instance is "up" in the only sense
8466
+ // the watchdog cares about: it must not be rolled back.
8467
+ //
8468
+ // The call never rejects (see runtime.disarmWatchdog) and never blocks the
8469
+ // load: the marker is written synchronously on its first tick, the
8470
+ // `schtasks` cleanup runs with retries behind it.
8471
+ runtime.disarmWatchdog('boot')
8472
+
7998
8473
  ctx.effect(() => () => {
7999
8474
  runtime.markDisposed()
8000
8475
  runtime.stopPolling()