@andrian.yablonskyy/thub-coordinator 1.0.19 → 1.0.21

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@andrian.yablonskyy/thub-coordinator",
-  "version": "1.0.19",
+  "version": "1.0.21",
   "description": "TestHub Coordinator — job queue, resource registry, scheduler, heartbeat monitor, log/artifact store and web dashboard",
   "main": "src/server.js",
   "engines": {

package/public/css/thub.css

@@ -4,3 +4,27 @@
   white-space: pre-wrap;
   word-break: break-all;
 }
+
+/* Toast auto-close countdown (public/js/toasts.js): a thin bar along the
+   bottom that shrinks over the toast's delay; the toast closes when it
+   runs out. Hovering or focusing the toast pauses it. currentColor keeps
+   it readable on every text-bg-* variant. */
+.thub-toast-progress {
+  position: absolute;
+  inset: auto 0 0 0;
+  height: 3px;
+  background: currentColor;
+  opacity: 0.45;
+  transform-origin: left center;
+  animation: thub-toast-countdown var(--thub-toast-delay, 10s) linear forwards;
+}
+
+.toast:hover .thub-toast-progress,
+.toast:focus-within .thub-toast-progress {
+  animation-play-state: paused;
+}
+
+@keyframes thub-toast-countdown {
+  from { transform: scaleX(1); }
+  to { transform: scaleX(0); }
+}

package/public/js/log-viewer.js

@@ -21,13 +21,34 @@
   const jobId = pre.dataset.jobId,
     jobActive = pre.dataset.jobActive === 'true',
     filter = document.getElementById('stream-filter'),
-    lines = [];
+    lines = [],
+    // Same dd/mm/yyyy HH:MM:SS, 24-hour format and profile time zone as
+    // every server-rendered date (thub-common formatDateTime).
+    timeFormat = new Intl.DateTimeFormat('en-GB', {
+      timeZone: pre.dataset.timeZone || undefined,
+      day: '2-digit',
+      month: '2-digit',
+      year: 'numeric',
+      hour: '2-digit',
+      minute: '2-digit',
+      second: '2-digit',
+      hourCycle: 'h23'
+    });
+
+  function fmtTime(iso){
+    const d = new Date(iso);
+    if (Number.isNaN(d.getTime())){
+      return iso;
+    }
+    const p = Object.fromEntries(timeFormat.formatToParts(d).map(({ type, value }) => [type, value]));
+    return `${p.day}/${p.month}/${p.year} ${p.hour}:${p.minute}:${p.second}`;
+  }
 
   function render(){
     const wanted = filter.value;
     pre.textContent = lines
       .filter((l) => !wanted || l.stream === wanted)
-      .map((l) => `[${l.ts}] [${l.stream}] ${l.line}`)
+      .map((l) => `[${fmtTime(l.ts)}] [${l.stream}] ${l.line}`)
       .join('\n');
     pre.scrollTop = pre.scrollHeight;
   }

package/public/js/toasts.js

@@ -22,12 +22,28 @@
   document.querySelectorAll('[data-thub-toast]').forEach((el) => {
     const type = el.dataset.thubToast in DELAY_MS ? el.dataset.thubToast : 'info',
       delay = el.dataset.sticky ? null : DELAY_MS[type],
-      toast = new bootstrap.Toast(el, { autohide: delay !== null, delay: delay ?? 0 });
+      // Our own countdown instead of Bootstrap's autohide: that one restarts
+      // the full delay after every hover, which a progress bar can't show.
+      toast = new bootstrap.Toast(el, { autohide: false });
     // "Self-destroying": gone from the page once hidden, not just invisible.
     el.addEventListener('hidden.bs.toast', () => {
       toast.dispose();
       el.remove();
     });
+
+    // The bar is the timer: a CSS animation over `delay` (paused while the
+    // toast is hovered or focused, see thub.css) — the toast closes when
+    // it ends. Added here, not in the markup, so without this script no
+    // bar suggests a close that would never happen.
+    if (delay !== null){
+      const bar = document.createElement('div');
+      bar.className = 'thub-toast-progress';
+      bar.setAttribute('aria-hidden', 'true');
+      bar.style.setProperty('--thub-toast-delay', `${delay}ms`);
+      bar.addEventListener('animationend', () => toast.hide(), { once: true });
+      el.classList.add('position-relative', 'overflow-hidden');
+      el.appendChild(bar);
+    }
     toast.show();
   });
 })();

package/public/js/update-watch.js

@@ -0,0 +1,54 @@
+/**
+ * @file        packages/coordinator/public/js/update-watch.js
+ * @description Dashboard: while a Coordinator self-update is pending, polls its status and reloads the page when done
+ *
+ * @author      Andrian Yablonskyy
+ * @copyright   Copyright (c) 2026 Andrian Yablonskyy. All rights reserved.
+ *
+ * This file is part of TestHub and is proprietary and confidential.
+ * Unauthorized copying, modification, distribution, or use of this file,
+ * via any medium, is strictly prohibited without prior written permission
+ * from AdSystem.PRO.
+ */
+
+'use strict';
+
+(function (){
+  const marker = document.getElementById('coordinator-update-pending');
+  if (!marker){
+    return;
+  }
+  const currentVersion = marker.dataset.currentVersion,
+    POLL_MS = 3000;
+
+  // Reload once the Coordinator runs another version (the update is done),
+  // or no longer has one pending (it failed — the reloaded page says why).
+  // While it restarts, requests fail or hit the proxy's error page: keep
+  // polling. A reply that isn't our JSON (e.g. the login page, sessions
+  // having been reset by the restart) also means it's back — reload.
+  async function poll(){
+    let res;
+    try {
+      res = await fetch('/updates/status', { cache: 'no-store', credentials: 'same-origin', headers: { Accept: 'application/json' } });
+    }
+    catch {
+      return setTimeout(poll, POLL_MS); // down mid-restart
+    }
+    if (!res.ok){
+      return setTimeout(poll, POLL_MS); // e.g. 502 from the reverse proxy
+    }
+    let status;
+    try {
+      status = await res.json();
+    }
+    catch {
+      return location.reload();
+    }
+    if (status.coordinatorVersion !== currentVersion || !status.pending){
+      return location.reload();
+    }
+    setTimeout(poll, POLL_MS);
+  }
+
+  setTimeout(poll, POLL_MS);
+})();

package/scripts/install-systemd-unit.js

@@ -48,6 +48,7 @@ function renderUpdateUnits(user, paths){
     unitPath = (name) => path.join(__dirname, '..', 'systemd', name);
   return {
     [UPDATE_PATH_UNIT]: fs.readFileSync(unitPath(UPDATE_PATH_UNIT), 'utf8')
+      .replace(/^PathExists=.*$/m, `PathExists=${paths.updateRequestFile}`)
       .replace(/^PathModified=.*$/m, `PathModified=${paths.updateRequestFile}`),
     // npm is a `#!/usr/bin/env node` script, so this node goes first on PATH.
     [UPDATE_SERVICE_UNIT]: fs.readFileSync(unitPath(UPDATE_SERVICE_UNIT), 'utf8')

package/scripts/self-update-helper.js

@@ -15,9 +15,33 @@
 'use strict';
 
 const fs = require('node:fs'),
+  path = require('node:path'),
+  { execFileSync } = require('node:child_process'),
   { PACKAGES, isValidVersion, compareVersions, npmInstallGlobal } = require('@andrian.yablonskyy/thub-common'),
   { version: installedVersion } = require('../package.json');
 
+// Progress for the dashboard (services/updates.js reads it), next to the
+// request: {version, state, message, at} with state installing |
+// restarting | failed. Without it the dashboard can't tell a failed or
+// never-started update from one still running, and waits forever.
+let statusFile = null;
+function report(version, state, message = null){
+  if (!statusFile){
+    return;
+  }
+  try {
+    fs.writeFileSync(statusFile, JSON.stringify({ version, state, message, at: new Date().toISOString() }) + '\n', { mode: 0o644 });
+  }
+  catch (err){
+    console.error(`thub-coordinator-update: could not write ${statusFile}: ${err.message}`);
+  }
+}
+
+// The request only ever comes from a Coordinator older than `version`.
+function restartCoordinator(){
+  execFileSync('systemctl', ['restart', 'thub-coordinator.service'], { stdio: 'inherit' });
+}
+
 // argv: <request file> <user the Coordinator runs as>. The request file
 // is written by the (unprivileged) Coordinator, so only its `version` is
 // used, and only if it's a plain semver — the package is fixed.
@@ -27,6 +51,7 @@ function main(){
     console.error('usage: self-update-helper.js <request file> <user>');
     return 2;
   }
+  statusFile = path.join(path.dirname(requestFile), 'update-status.json');
 
   let request;
   try {
@@ -46,20 +71,41 @@ function main(){
   const target = request?.version;
   if (!isValidVersion(target)){
     console.error('thub-coordinator-update: ignoring request with an invalid version');
+    report(String(target), 'failed', 'The update request had an invalid version.');
     return 1;
   }
+  // Already installed on disk (e.g. a manual `npm i -g` whose restart never
+  // happened) while the running Coordinator is older — it asked, after all.
+  // Just restart it onto what's installed instead of leaving it waiting.
   if (compareVersions(installedVersion, target) >= 0){
-    console.log(`thub-coordinator-update: already on v${installedVersion} (requested v${target})`);
-    return 0;
+    console.log(`thub-coordinator-update: v${installedVersion} is already installed (requested v${target}) — restarting the Coordinator onto it`);
+    report(target, 'restarting');
+    try {
+      restartCoordinator();
+      return 0;
+    }
+    catch (err){
+      report(target, 'failed', `v${installedVersion} is installed, but restarting the Coordinator failed: ${err.message}`);
+      return 1;
+    }
   }
 
   console.log(`thub-coordinator-update: v${installedVersion} -> v${target} (requested by ${request.requestedBy || 'the dashboard'})`);
+  report(target, 'installing');
   // SUDO_USER makes the postinstall scripts target the Coordinator's user,
   // as a `sudo npm i -g` by that user would (install-target.js); the
   // postinstall then restarts thub-coordinator on the new version.
-  const status = npmInstallGlobal(PACKAGES.coordinator, target, { env: { ...process.env, SUDO_USER: user } });
+  let status;
+  try {
+    status = npmInstallGlobal(PACKAGES.coordinator, target, { env: { ...process.env, SUDO_USER: user } });
+  }
+  catch (err){
+    report(target, 'failed', `Could not run npm: ${err.message}`);
+    return 1;
+  }
   if (status !== 0){
     console.error(`thub-coordinator-update: npm i -g failed (exit ${status})`);
+    report(target, 'failed', `npm i -g failed (exit ${status}) — see: journalctl -u thub-coordinator-update`);
   }
   return status;
 }

package/src/services/updates.js

@@ -15,21 +15,82 @@
 
 const fs = require('node:fs'),
   path = require('node:path'),
-  { PACKAGES, fetchLatestVersion, defaultRegistry, isNewer } = require('@andrian.yablonskyy/thub-common'),
+  { PACKAGES, fetchLatestVersion, defaultRegistry, isNewer, compareVersions } = require('@andrian.yablonskyy/thub-common'),
   { version: coordinatorVersion } = require('../../package.json');
 
 // Installed by scripts/install-systemd-unit.js; runs the actual `npm i -g`
 // as root when the dashboard writes an update request (README §10.2).
-const UPDATE_PATH_UNIT = '/etc/systemd/system/thub-coordinator-update.path';
+const UPDATE_PATH_UNIT = '/etc/systemd/system/thub-coordinator-update.path',
+  // No sign of the root helper by then: the request wasn't picked up.
+  PICKUP_TIMEOUT_MS = 60_000,
+  // Longer than thub-coordinator-update.service's TimeoutStartSec (15 min).
+  INSTALL_TIMEOUT_MS = 20 * 60_000;
 
 // Latest published versions, refreshed every `updates.checkIntervalMin`
 // and on demand from the dashboard. Kept in memory only — a restart just
 // checks again.
-function createUpdatesService({ config }){
+// `pathUnit`, `fetchLatest` and `now` are overridable for tests only.
+function createUpdatesService({ config, pathUnit = UPDATE_PATH_UNIT, fetchLatest = fetchLatestVersion, now = Date.now }){
   const registry = config.updates.registry || defaultRegistry(),
-    state = { latest: {}, checkedAt: null, error: null, coordinatorPending: null };
+    state = { latest: {}, checkedAt: null, error: null, coordinatorPending: null, coordinatorRequestedAt: null, coordinatorUpdateError: null },
+    requestFile = path.join(config.dataDir, 'update-request.json'),
+    // Written by the root helper (scripts/self-update-helper.js).
+    statusFile = path.join(config.dataDir, 'update-status.json');
   let inFlight = null;
 
+  function readHelperStatus(){
+    try {
+      return JSON.parse(fs.readFileSync(statusFile, 'utf8'));
+    }
+    catch {
+      return null;
+    }
+  }
+
+  function failPending(message){
+    state.coordinatorUpdateError = { version: state.coordinatorPending, message, at: new Date().toISOString() };
+    state.coordinatorPending = null;
+    state.coordinatorRequestedAt = null;
+    console.error(`Coordinator self-update failed: ${message}`);
+  }
+
+  // A pending update ends with this process being restarted onto the new
+  // version. Anything else — the helper reporting failure, never picking
+  // the request up, or taking far too long — must end the wait too, or the
+  // dashboard shows "Updating…" forever.
+  function refreshPending(){
+    if (!state.coordinatorPending){
+      return;
+    }
+    const requestedAt = Date.parse(state.coordinatorRequestedAt),
+      helper = readHelperStatus(),
+      fresh = helper && Date.parse(helper.at) >= requestedAt - 1000;
+    if (fresh && helper.state === 'failed'){
+      failPending(helper.message || 'The update helper reported a failure — see: journalctl -u thub-coordinator-update');
+    }
+    else if (!fresh && now() - requestedAt > PICKUP_TIMEOUT_MS){
+      failPending(
+        fs.existsSync(requestFile)
+          ? `The update request wasn't picked up within ${PICKUP_TIMEOUT_MS / 1000} s — is the watcher running? ` +
+            'sudo systemctl enable --now thub-coordinator-update.path'
+          : 'The update helper started but never reported progress — see: journalctl -u thub-coordinator-update'
+      );
+    }
+    else if (fresh && now() - Date.parse(helper.at) > INSTALL_TIMEOUT_MS){
+      failPending(`The update has been "${helper.state}" for over ${INSTALL_TIMEOUT_MS / 60_000} min — see: journalctl -u thub-coordinator-update`);
+    }
+  }
+
+  // On startup: a helper status for this version (or older) means the
+  // update that restarted us is done — clear it.
+  function settleHelperStatus(){
+    const helper = readHelperStatus();
+    if (helper && helper.state !== 'failed' && compareVersions(coordinatorVersion, String(helper.version)) >= 0){
+      fs.rmSync(statusFile, { force: true });
+      console.log(`Coordinator self-update to v${helper.version} completed`);
+    }
+  }
+
   async function checkNow(){
     // Concurrent callers (timer + a dashboard click) share one check.
     inFlight = inFlight || (async () => {
@@ -37,7 +98,7 @@ function createUpdatesService({ config }){
         errors = [];
       await Promise.all(Object.entries(PACKAGES).map(async ([app, pkg]) => {
         try {
-          latest[app] = await fetchLatestVersion(pkg, { registry });
+          latest[app] = await fetchLatest(pkg, { registry });
         }
         catch (err){
           errors.push(err.message);
@@ -55,6 +116,7 @@ function createUpdatesService({ config }){
   }
 
   function status(){
+    refreshPending();
     return {
       ...state,
       coordinatorVersion,
@@ -70,17 +132,19 @@ function createUpdatesService({ config }){
     if (!target){
       throw Object.assign(new Error(`Already on the latest version (v${coordinatorVersion}).`), { status: 409 });
     }
-    if (!fs.existsSync(UPDATE_PATH_UNIT)){
+    if (!fs.existsSync(pathUnit)){
       throw Object.assign(
-        new Error(`Self-update isn't installed on this host (${UPDATE_PATH_UNIT}) — update by hand: thub-admin self-update`),
+        new Error(`Self-update isn't installed on this host (${pathUnit}) — update by hand: thub-admin self-update`),
         { status: 501 }
       );
     }
-    fs.writeFileSync(
-      path.join(config.dataDir, 'update-request.json'),
-      JSON.stringify({ version: target, requestedBy, requestedAt: new Date().toISOString() }) + '\n'
-    );
+    const requestedAt = new Date(now()).toISOString();
+    // A previous attempt's result mustn't be mistaken for this one's.
+    fs.rmSync(statusFile, { force: true });
+    fs.writeFileSync(requestFile, JSON.stringify({ version: target, requestedBy, requestedAt }) + '\n');
     state.coordinatorPending = target;
+    state.coordinatorRequestedAt = requestedAt;
+    state.coordinatorUpdateError = null;
     return target;
   }
 
@@ -97,6 +161,7 @@ function createUpdatesService({ config }){
   }
 
   function start(){
+    settleHelperStatus();
     const minutes = Number(config.updates.checkIntervalMin);
     if (!(minutes > 0)){
       return;

package/src/web/routes.js

@@ -20,7 +20,7 @@ const fs = require('node:fs'),
   multer = require('multer'),
   { requireAdminSession, requireAdminRole } = require('../auth'),
   { attachJobStream } = require('../api/sse'),
-  { RESOURCE_STATES, JOB_STATES, ACTIVE_JOB_STATES, isNewer, compareVersions } = require('@andrian.yablonskyy/thub-common'),
+  { RESOURCE_STATES, JOB_STATES, ACTIVE_JOB_STATES, isNewer, compareVersions, formatDateTime } = require('@andrian.yablonskyy/thub-common'),
   listPrefs = require('../services/list-prefs');
 
 // Resources page sort keys (list-prefs.js) -> comparators. Empty values
@@ -103,6 +103,18 @@ function createWebRouter({ services, config }){
     res.locals.messages = req.session?.flash || [];
     // Latest published versions (README §10.2) — navbar, Agents, Resources.
     res.locals.updates = services.updates.status();
+    // A failed Coordinator self-update: shown to each admin once, as an
+    // error toast (never auto-closes) — on a page view, not the polling
+    // endpoint or a POST whose redirect would drop it.
+    const updateError = res.locals.updates.coordinatorUpdateError;
+    if (updateError && user?.role === 'admin' && req.method === 'GET' && req.path !== '/updates/status' &&
+      req.session.seenUpdateError !== updateError.at){
+      req.session.seenUpdateError = updateError.at;
+      res.locals.messages = [
+        ...res.locals.messages,
+        { type: 'danger', text: `Coordinator update to v${updateError.version} failed: ${updateError.message}` }
+      ];
+    }
     res.locals.isNewer = isNewer;
     res.locals.currentPath = req.originalUrl;
     if (req.session){
@@ -121,8 +133,10 @@ function createWebRouter({ services, config }){
         m = Math.floor((s % 3600) / 60);
       return d ? `${d}d ${h}h` : h ? `${h}h ${m}m` : m ? `${m}m ${s % 60}s` : `${s}s`;
     };
-    res.locals.fmtDate = (iso, fallback = '—') =>
-      iso ? new Date(iso).toLocaleString('en-US', { timeZone: tz }) : fallback;
+    // dd/mm/yyyy HH:MM:SS, 24-hour (thub-common formatDateTime) — the one
+    // format for every date/time label.
+    res.locals.fmtDate = (iso, fallback = '—') => formatDateTime(iso, { timeZone: tz, fallback });
+    res.locals.userTimeZone = tz;
 
     // Idle timeout (§10.1): re-applied on every request (not just login)
     // so a mid-session profile change takes effect immediately, and so
@@ -427,9 +441,21 @@ function createWebRouter({ services, config }){
   router.post('/updates/coordinator', requireAdminRole, updateAction(async (req) => {
     const version = services.updates.requestCoordinatorUpdate(req.session.user.username);
     console.log(`Coordinator self-update to v${version} requested by ${req.session.user.username}`);
-    return `Updating the Coordinator to v${version} — the dashboard restarts in a minute or two; reload the page then.`;
+    return `Updating the Coordinator to v${version} — this page reloads by itself once it's back up.`;
   }));
 
+  // Polled by public/js/update-watch.js while a Coordinator update is
+  // pending: a new `coordinatorVersion` (the restart happened) or a cleared
+  // `pending` (it failed) tells the page to reload.
+  router.get('/updates/status', (req, res) => {
+    const s = services.updates.status();
+    res.set('Cache-Control', 'no-store').json({
+      coordinatorVersion: s.coordinatorVersion,
+      pending: s.coordinatorPending,
+      error: s.coordinatorUpdateError
+    });
+  });
+
   router.post('/resources/update-all', requireAdminRole, updateAction(async () => {
     const version = await services.updates.targetVersion('client'),
       n = services.registry.requestUpdateAll(version);

package/systemd/thub-coordinator-update.path

@@ -1,11 +1,14 @@
 [Unit]
 Description=TestHub Coordinator self-update request watcher
 
-# PathModified= is rewritten by scripts/install-systemd-unit.js to the
-# update-request.json in the Coordinator's dataDir. The dashboard's
+# PathExists=/PathModified= are rewritten by scripts/install-systemd-unit.js
+# to the update-request.json in the Coordinator's dataDir. The dashboard's
 # "Update app" button writes it; this starts thub-coordinator-update.service
-# (as root) to install the requested version.
+# (as root) to install the requested version. PathExists= also catches a
+# request written while this watch wasn't active (e.g. mid-reinstall) — the
+# helper deletes the file, so it doesn't re-trigger.
 [Path]
+PathExists=/var/lib/thub/.thub/update-request.json
 PathModified=/var/lib/thub/.thub/update-request.json
 Unit=thub-coordinator-update.service
 

package/test/updates.test.js

@@ -0,0 +1,106 @@
+/**
+ * @file        packages/coordinator/test/updates.test.js
+ * @description Tests: Coordinator self-update request lifecycle — the dashboard never waits forever on a stuck update
+ *
+ * @author      Andrian Yablonskyy
+ * @copyright   Copyright (c) 2026 Andrian Yablonskyy. All rights reserved.
+ *
+ * This file is part of TestHub and is proprietary and confidential.
+ * Unauthorized copying, modification, distribution, or use of this file,
+ * via any medium, is strictly prohibited without prior written permission
+ * from AdSystem.PRO.
+ */
+
+'use strict';
+
+const test = require('node:test'),
+  assert = require('node:assert/strict'),
+  fs = require('node:fs'),
+  os = require('node:os'),
+  path = require('node:path'),
+  { createUpdatesService } = require('../src/services/updates'),
+  { version: installed } = require('../package.json');
+
+const NEWER = installed.replace(/\d+$/, (n) => String(Number(n) + 1));
+
+async function setup(){
+  const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'thub-updates-')),
+    pathUnit = path.join(dataDir, 'fake.path'),
+    clock = { t: Date.parse('2026-09-28T12:00:00Z') };
+  fs.writeFileSync(pathUnit, '');
+  const updates = createUpdatesService({
+    config: { dataDir, updates: {} },
+    pathUnit,
+    fetchLatest: async () => NEWER,
+    now: () => clock.t
+  });
+  await updates.checkNow();
+  const helperSays = (state, message) => fs.writeFileSync(
+    path.join(dataDir, 'update-status.json'),
+    JSON.stringify({ version: NEWER, state, message, at: new Date(clock.t).toISOString() })
+  );
+  return { updates, dataDir, clock, helperSays };
+}
+
+test('a request is written for the root helper and shows as pending', async () => {
+  const { updates, dataDir } = await setup();
+  assert.equal(updates.requestCoordinatorUpdate('alice'), NEWER);
+  assert.equal(JSON.parse(fs.readFileSync(path.join(dataDir, 'update-request.json'), 'utf8')).version, NEWER);
+  assert.equal(updates.status().coordinatorPending, NEWER);
+});
+
+test('a request nobody picks up stops pending after the pickup timeout, with the reason', async () => {
+  const { updates, clock } = await setup();
+  updates.requestCoordinatorUpdate('alice');
+  clock.t += 30_000;
+  assert.equal(updates.status().coordinatorPending, NEWER); // still within the timeout
+  clock.t += 31_000;
+  const s = updates.status();
+  assert.equal(s.coordinatorPending, null);
+  assert.match(s.coordinatorUpdateError.message, /wasn't picked up.*thub-coordinator-update\.path/);
+});
+
+test('a helper failure ends the wait at once with its message', async () => {
+  const { updates, clock, helperSays } = await setup();
+  updates.requestCoordinatorUpdate('alice');
+  clock.t += 2000;
+  helperSays('installing');
+  assert.equal(updates.status().coordinatorPending, NEWER);
+  helperSays('failed', 'npm i -g failed (exit 1)');
+  const s = updates.status();
+  assert.equal(s.coordinatorPending, null);
+  assert.deepEqual([s.coordinatorUpdateError.version, s.coordinatorUpdateError.message], [NEWER, 'npm i -g failed (exit 1)']);
+});
+
+test('an install that runs far too long times out; a normal one keeps pending', async () => {
+  const { updates, clock, helperSays } = await setup();
+  updates.requestCoordinatorUpdate('alice');
+  helperSays('installing');
+  clock.t += 5 * 60_000;
+  assert.equal(updates.status().coordinatorPending, NEWER);
+  clock.t += 16 * 60_000;
+  assert.match(updates.status().coordinatorUpdateError.message, /"installing" for over 20 min/);
+});
+
+test('a stale failure from an earlier attempt is not mistaken for the new one', async () => {
+  const { updates, clock, helperSays } = await setup();
+  helperSays('failed', 'old failure');
+  clock.t += 60_000;
+  updates.requestCoordinatorUpdate('alice'); // removes the old status
+  assert.equal(updates.status().coordinatorPending, NEWER);
+  assert.equal(updates.status().coordinatorUpdateError, null);
+});
+
+test('on startup, a finished update\'s status is cleared; a failure is kept for the dashboard', async () => {
+  const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'thub-updates-')),
+    statusFile = path.join(dataDir, 'update-status.json'),
+    make = () => createUpdatesService({ config: { dataDir, updates: {} }, fetchLatest: async () => installed });
+
+  fs.writeFileSync(statusFile, JSON.stringify({ version: installed, state: 'installing', at: new Date().toISOString() }));
+  make().start();
+  assert.equal(fs.existsSync(statusFile), false);
+
+  fs.writeFileSync(statusFile, JSON.stringify({ version: NEWER, state: 'failed', at: new Date().toISOString() }));
+  make().start();
+  assert.equal(fs.existsSync(statusFile), true);
+});

package/views/layout.pug

@@ -52,7 +52,14 @@ html(lang="en" data-bs-theme=(user && user.theme) || "auto")
             if user && user.role === 'admin' && updates.coordinatorUpdate
               li.nav-item.d-flex.align-items-center.me-3
                 if updates.coordinatorPending
-                  button.btn.btn-sm.btn-warning(type="button" disabled)
+                  //- public/js/update-watch.js polls /updates/status and
+                  //- reloads the page once the new version is up (or the
+                  //- update failed, to show why).
+                  button.btn.btn-sm.btn-warning#coordinator-update-pending(
+                    type="button"
+                    disabled
+                    data-current-version=coordinatorVersion
+                  )
                     span.spinner-border.spinner-border-sm.me-1(aria-hidden="true")
                     = `Updating app to ${updates.coordinatorPending}…`
                 else
@@ -104,12 +111,16 @@ html(lang="en" data-bs-theme=(user && user.theme) || "auto")
                   )
                     i.bi.bi-box-arrow-right
     //- Flash messages as toasts (public/js/toasts.js): errors stay until
-    //- closed, warnings close after 30 s, info/success after 10 s. Rendered
+    //- closed, warnings close after 30 s, info/success after 10 s, with a
+    //- countdown bar along the bottom (paused on hover). Rendered
     //- visible (.show), so they're readable even if the script fails.
     if messages && messages.length
       .toast-container.position-fixed.end-0.p-3(style="top: 3.5rem; z-index: 1090;")
         each m in messages
-          - const type = ['danger', 'warning', 'info', 'success'].includes(m.type) ? m.type : 'info'
+          //- Error-level aliases map to danger (never auto-closes); anything
+          //- else unknown is shown as info.
+          - const alias = { error: 'danger', critical: 'danger', fatal: 'danger' }[m.type] || m.type
+          - const type = ['danger', 'warning', 'info', 'success'].includes(alias) ? alias : 'info'
           - const icon = { danger: 'x-octagon-fill', warning: 'exclamation-triangle-fill', info: 'info-circle-fill', success: 'check-circle-fill' }[type]
           .toast.show.border-0(
             class=`text-bg-${type}`
@@ -136,4 +147,6 @@ html(lang="en" data-bs-theme=(user && user.theme) || "auto")
     script(src="/js/tooltips.js")
     script(src="/js/toasts.js")
     script(src="/js/copy-to-clipboard.js")
+    if user && user.role === 'admin' && updates.coordinatorPending
+      script(src="/js/update-watch.js")
     block scripts

package/views/mixins/log-viewer.pug

@@ -10,5 +10,6 @@ mixin logViewer(jobId, jobActive)
     style="height: 420px; overflow-y: auto;"
     data-job-id=jobId
     data-job-active=jobActive ? 'true' : 'false'
+    data-time-zone=userTimeZone
   )
   script(src="/js/log-viewer.js")

package/views/mixins/resource.pug

@@ -2,7 +2,9 @@
 //- Overview and Resources pages. `returnTo` is the page the action forms
 //- redirect back to.
 
-mixin resourceActions(r, returnTo)
+//- `opts.update: false` leaves out the Update button — the resource card
+//- shows it on its Client version line instead.
+mixin resourceActions(r, returnTo, opts = {})
   form.d-inline(method="post" action=`/resources/${r.id}/maintenance`)
     input(type="hidden" name="enabled" value=r.status === 'MAINTENANCE' ? '0' : '1')
     input(type="hidden" name="returnTo" value=returnTo)
@@ -31,7 +33,8 @@ mixin resourceActions(r, returnTo)
       data-bs-title="Invalidate the current resource token and issue a new one — the Client won't reconnect until it has it"
     )
       i.bi.bi-arrow-repeat
-  +updateButton(`/resources/${r.id}/update`, r.update_to, returnTo)
+  if opts.update !== false
+    +updateButton(`/resources/${r.id}/update`, r.update_to, returnTo)
   +removeResourceButton(r, returnTo)
 
 //- Remove: a plain confirm when the resource is idle; with a running job,
@@ -168,7 +171,8 @@ mixin capabilitiesList(caps)
 //- Host uptime and the Client's current task/status with its duration,
 //- both from the last heartbeat. Measured up to now — or, for a Client
 //- that stopped heartbeating, up to its last heartbeat.
-mixin uptimeRows(r)
+//- `returnTo`: where the Task / status line's Cancel button redirects.
+mixin uptimeRows(r, returnTo)
   - const stale = r.status === 'OUT_OF_SERVICE' && r.last_heartbeat_at
   - const ref = stale ? new Date(r.last_heartbeat_at).getTime() : Date.now()
   - const activityLabel = { idle: 'Idle', job: 'Running job', locked: 'Locked locally', 'update-hold': 'Holding for self-update' }
@@ -193,27 +197,30 @@ mixin uptimeRows(r)
         span.text-body-secondary.small.ms-1 (at last heartbeat)
     else
       span.text-body-secondary —
+    if user.role === 'admin'
+      +cancelActivityButton(r, returnTo)
 
 //- Cancel the Client's current activity (its job, a local lock or a
-//- self-update hold); disabled while it's idle.
+//- self-update hold) — icon only, on the card's Task / status line, and
+//- only while there's something to cancel.
 mixin cancelActivityButton(r, returnTo)
   - const state = r.activity && r.activity.state
   - const what = r.activeJob ? `job ${r.activeJob.id}` : state === 'locked' ? 'the local lock' : state === 'update-hold' ? 'the self-update' : null
   if what
-    form.d-inline.ms-1(
+    form.d-inline.ms-2(
       method="post"
       action=`/resources/${r.id}/cancel-activity`
-      onsubmit=`return confirm('Cancel ${what} on ${r.name}?')`
+      data-confirm=`Cancel ${what} on ${r.name}?`
+      onsubmit="return confirm(this.dataset.confirm)"
     )
       input(type="hidden" name="returnTo" value=returnTo)
-      button.btn.btn-sm.btn-outline-danger(type="submit" data-bs-toggle="tooltip" data-bs-title=`Cancel ${what}`)
-        i.bi.bi-x-octagon.me-1
-        | Cancel
-  else
-    span.d-inline-block.ms-1(tabindex="0" data-bs-toggle="tooltip" data-bs-title="Nothing to cancel — the Client is idle")
-      button.btn.btn-sm.btn-outline-danger(type="button" disabled style="pointer-events: none;")
-        i.bi.bi-x-octagon.me-1
-        | Cancel
+      button.btn.btn-sm.btn-outline-danger(
+        type="submit"
+        aria-label=`Cancel ${what}`
+        data-bs-toggle="tooltip"
+        data-bs-title=`Cancel ${what}`
+      )
+        i.bi.bi-x-octagon
 
 mixin resourceCard(r, groupsById, returnTo)
   - const isLoopback = (a) => /^127\./.test(a) || a === '::1' || a === 'localhost'
@@ -233,9 +240,15 @@ mixin resourceCard(r, groupsById, returnTo)
             dd.col-sm-8.text-uppercase= r.type
             dt.col-sm-4 Status
             dd.col-sm-8: +resourceBadge(r.status, r.remove_requested_at)
-            +uptimeRows(r)
+            +uptimeRows(r, returnTo)
             dt.col-sm-4 Client version
-            dd.col-sm-8: +versionCell(r.client_version, updates.latest.client, r.update_to)
+            dd.col-sm-8
+              +versionCell(r.client_version, updates.latest.client, r.update_to)
+              //- Admins: Update only when a newer Client is published (or
+              //- Cancel, while one is requested). An unknown version (too
+              //- old to report it) can't be compared, so no button.
+              if user.role === 'admin' && (r.update_to || (r.client_version && isNewer(updates.latest.client, r.client_version)))
+                +updateButton(`/resources/${r.id}/update`, r.update_to, returnTo)
             dt.col-sm-4 IP addresses
             dd.col-sm-8
               if !local.length && !external
@@ -276,5 +289,4 @@ mixin resourceCard(r, groupsById, returnTo)
             if user.role === 'admin'
               dt.col-sm-4 Actions
               dd.col-sm-8.mb-0
-                +resourceActions(r, returnTo)
-                +cancelActivityButton(r, returnTo)
+                +resourceActions(r, returnTo, { update: false })