@andrian.yablonskyy/thub-coordinator 1.0.39 → 1.0.40

package/src/services/settings.js

@@ -0,0 +1,345 @@
+/**
+ * @file        packages/coordinator/src/services/settings.js
+ * @description Coordinator settings editable from the dashboard (/admin/settings): stored in the database, layered over the config file
+ *
+ * @author      Andrian Yablonskyy
+ * @copyright   Copyright (c) 2026 Andrian Yablonskyy. All rights reserved.
+ *
+ * This file is part of TestHub and is proprietary and confidential.
+ * Unauthorized copying, modification, distribution, or use of this file,
+ * via any medium, is strictly prohibited without prior written permission
+ * from AdSystem.PRO.
+ */
+
+'use strict';
+
+const { JOB_RETENTION } = require('./cleanup');
+
+// README §13.2. Precedence, lowest first: built-in default < config file <
+// dashboard setting (the `settings` table) < environment variable. The
+// dashboard can't write the config file — the systemd unit only lets the
+// service write its data directory — so its changes live in the database.
+//
+// `applies`: "now" — the code reads it on every use, so saving takes effect
+// at once; "restart" — read once at startup (timers, middleware), so it
+// takes effect when the Coordinator restarts. Settings that could cut the
+// dashboard off from the inside (listen, dataDir, sessionSecret) stay in
+// the config file.
+const int = (min, max) => (raw) => {
+    const n = Number(raw);
+    if (!Number.isInteger(n) || n < min || n > max){
+      throw new Error(`must be a whole number from ${min} to ${max}`);
+    }
+    return n;
+  },
+  bool = (raw) => raw === true || raw === 'true' || raw === 'on' || raw === '1',
+  oneOf = (values) => (raw) => {
+    const v = values.find((x) => String(x) === String(raw));
+    if (v === undefined){
+      throw new Error(`must be one of: ${values.join(', ')}`);
+    }
+    return v;
+  },
+
+  SETTINGS = [
+    {
+      key: 'publicUrl', group: 'General', label: 'Public URL', applies: 'now', env: 'THUB_PUBLIC_URL', type: 'text',
+      help: 'The https:// address Agents, Clients and browsers use. Job links and the dashboard\'s same-origin check use it.',
+      parse: (raw) => {
+        const s = String(raw).trim().replace(/\/+$/, '');
+        let url;
+        try {
+          url = new URL(s);
+        }
+        catch {
+          throw new Error('must be a URL like https://thub.example.com');
+        }
+        if (!/^https?:$/.test(url.protocol) || url.pathname !== '/' || url.search || url.hash){
+          throw new Error('must be just http(s)://host[:port], without a path');
+        }
+        return s;
+      }
+    },
+    {
+      key: 'clientJoinKey', group: 'General', label: 'Client join key', applies: 'now', env: 'THUB_CLIENT_JOIN_KEY', type: 'secret',
+      help: 'The shared secret Clients self-register with (joinKey in their config). Empty turns self-registration off. ' +
+        'Running Clients keep working; one with an old key can\'t register again after its next restart.',
+      parse: (raw) => {
+        const s = String(raw ?? '').trim();
+        if (s && s.length < 16){
+          throw new Error('must be at least 16 characters (or empty to turn registration off)');
+        }
+        return s || null;
+      }
+    },
+    {
+      key: 'trustProxy', group: 'General', label: 'Trusted proxies', applies: 'restart', type: 'text',
+      help: 'Whose X-Forwarded-For/-Proto to believe: loopback (a proxy on this host), a CIDR such as 10.0.0.0/8, ' +
+        'a number of hops, or false. Wrong values break HTTPS detection and sign-in behind a proxy.',
+      parse: (raw) => {
+        const s = String(raw).trim();
+        if (s === 'false' || s === 'true'){
+          return s === 'true';
+        }
+        if (/^\d+$/.test(s)){
+          return Number(s);
+        }
+        if (!/^[\w.:/,\s-]+$/.test(s) || !s){
+          throw new Error('must be loopback, uniquelocal, linklocal, IP addresses/CIDRs (comma-separated), a hop count, true or false');
+        }
+        return s;
+      }
+    },
+    {
+      key: 'session.secureCookie', group: 'General', label: 'Secure session cookie', applies: 'restart', type: 'select',
+      options: ['auto', 'true', 'false'],
+      help: 'auto: HTTPS-only whenever the public URL is https://. true/false force it.',
+      parse: (raw) => ({ auto: 'auto', true: true, false: false })[String(raw)] ?? oneOf(['auto', 'true', 'false'])(raw)
+    },
+    {
+      key: 'heartbeat.intervalSec', group: 'Heartbeat', label: 'Heartbeat interval (s)', applies: 'now', type: 'number',
+      help: 'How often Clients report. They switch to a new value with their next heartbeat (lowering it, the old one is ' +
+        'still allowed for a while). Clients older than this setting keep their own heartbeatIntervalSec (10 by default): ' +
+        'don\'t go below that while any are connected.',
+      parse: int(2, 300)
+    },
+    {
+      key: 'heartbeat.missedLimit', group: 'Heartbeat', label: 'Missed heartbeats allowed', applies: 'now', type: 'number',
+      help: 'A Client that misses this many in a row goes OUT_OF_SERVICE, and its running job becomes LOST.',
+      parse: int(2, 20)
+    },
+    {
+      key: 'heartbeat.sweepIntervalSec', group: 'Heartbeat', label: 'Check every (s)', applies: 'restart', type: 'number',
+      help: 'How often missed heartbeats, unaccepted assignments and job timeouts are checked.',
+      parse: int(1, 60)
+    },
+    {
+      key: 'rateLimit.requestsPerMinute', group: 'Rate limits', label: 'Requests per minute', applies: 'now', type: 'number',
+      help: 'Per caller: an agent or resource token, a dashboard user, else the IP address. Short bursts are fine; ' +
+        'past that, requests get 429 with Retry-After. A Client streaming logs makes up to ~240 a minute. 0: no limit.',
+      parse: int(0, 1_000_000)
+    },
+    {
+      key: 'rateLimit.loginPerMinute', group: 'Rate limits', label: 'Sign-in attempts per minute', applies: 'now', type: 'number',
+      help: 'Per IP address, against password guessing. 0: no limit.',
+      parse: int(0, 10_000)
+    },
+    {
+      key: 'jobs.defaultTimeoutSec', group: 'Jobs', label: 'Default job timeout (s)', applies: 'now', type: 'number',
+      help: 'For a job submitted without --timeout.', parse: int(60, 7 * 86400)
+    },
+    {
+      key: 'jobs.maxTimeoutSec', group: 'Jobs', label: 'Maximum job timeout (s)', applies: 'now', type: 'number',
+      help: 'A longer --timeout is cut to this.', parse: int(60, 7 * 86400)
+    },
+    {
+      key: 'scheduler.assignAckTimeoutSec', group: 'Scheduler', label: 'Accept timeout (s)', applies: 'now', type: 'number',
+      help: 'How long an assigned job waits for its Client to accept it before going back to the queue.', parse: int(5, 600)
+    },
+    {
+      key: 'scheduler.requeueOnLost', group: 'Scheduler', label: 'Retry lost jobs', applies: 'now', type: 'checkbox',
+      help: 'Requeue a job once when its Client goes offline mid-job.', parse: bool
+    },
+    {
+      key: 'scheduler.maxQueuedPerAgent', group: 'Scheduler', label: 'Max queued jobs per agent', applies: 'now', type: 'number',
+      help: 'Further submissions from that agent are refused until some run.', parse: int(1, 10000)
+    },
+    {
+      key: 'scheduler.tickIntervalSec', group: 'Scheduler', label: 'Safety-net pass (s)', applies: 'restart', type: 'number',
+      help: 'The scheduler also runs on every queued job and free Client; this periodic pass only catches anything missed.',
+      parse: int(1, 3600)
+    },
+    {
+      key: 'retention.jobRetention', group: 'Retention', label: 'Keep finished jobs', applies: 'now', type: 'select',
+      options: Object.keys(JOB_RETENTION), labels: Object.fromEntries(Object.entries(JOB_RETENTION).map(([k, v]) => [k, v.label])),
+      help: 'Older finished jobs, with their logs, are deleted hourly. forever: only by Clean up database.',
+      parse: oneOf(Object.keys(JOB_RETENTION))
+    },
+    {
+      key: 'updates.checkIntervalMin', group: 'Updates', label: 'Version check every (min)', applies: 'restart', type: 'number',
+      help: 'How often to look for new Coordinator/Agent/Client releases. 0 turns the periodic check off.', parse: int(0, 1440)
+    },
+    {
+      key: 'updates.registry', group: 'Updates', label: 'npm registry', applies: 'restart', type: 'text',
+      help: 'Empty: the public npm registry (or npm_config_registry). Set it for a private mirror.',
+      parse: (raw) => {
+        const s = String(raw ?? '').trim();
+        if (s && !/^https?:\/\/\S+$/.test(s)){
+          throw new Error('must be an http(s):// URL, or empty');
+        }
+        return s || null;
+      }
+    }
+  ],
+  BY_KEY = new Map(SETTINGS.map((s) => [s.key, s])),
+
+  getPath = (obj, key) => key.split('.').reduce((o, k) => o?.[k], obj),
+  setPath = (obj, key, value) => {
+    const parts = key.split('.'),
+      last = parts.pop(),
+      parent = parts.reduce((o, k) => (o[k] ??= {}), obj);
+    parent[last] = value;
+  },
+  same = (a, b) => JSON.stringify(a) === JSON.stringify(b);
+
+// `config` is the live object every service holds: "now" settings are
+// written into it, so they take effect on the next use.
+function createSettingsService(db, { config, env = process.env }){
+  const stmts = {
+      all: db.prepare('SELECT key, value, updated_at, updated_by FROM settings'),
+      upsert: db.prepare(
+        `INSERT INTO settings (key, value, updated_at, updated_by) VALUES (?, ?, ?, ?)
+         ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at, updated_by = excluded.updated_by`
+      ),
+      remove: db.prepare('DELETE FROM settings WHERE key = ?')
+    },
+    // What the config file (plus defaults) said, before any dashboard
+    // setting: what "Reset" goes back to.
+    fileValues = Object.fromEntries(SETTINGS.map((s) => [s.key, structuredClone(getPath(config, s.key))]));
+  let running = null; // the values this process started with (for "after restart")
+
+  function stored(){
+    const rows = {};
+    for (const r of stmts.all.all()){
+      if (BY_KEY.has(r.key)){
+        rows[r.key] = { value: JSON.parse(r.value), updatedAt: r.updated_at, updatedBy: r.updated_by };
+      }
+    }
+    return rows;
+  }
+
+  const envLocked = (s) => Boolean(s.env && env[s.env]);
+
+  // What each setting resolves to right now: env > dashboard > file.
+  function desired(rows = stored()){
+    return Object.fromEntries(SETTINGS.map((s) => [
+      s.key,
+      envLocked(s) ? getPath(config, s.key) : s.key in rows ? rows[s.key].value : fileValues[s.key]
+    ]));
+  }
+
+  // At startup: put the dashboard's settings into the config every service
+  // reads. Then remember what this process runs with.
+  function applyAtStartup(){
+    const rows = stored();
+    // A value saved by an older version that this one no longer accepts is
+    // skipped (with a warning), not allowed to stop the Coordinator.
+    for (const [key, { value }]of Object.entries(rows)){
+      try {
+        BY_KEY.get(key).parse(value === null ? '' : value);
+      }
+      catch (err){
+        console.warn(`Ignoring dashboard setting ${key}=${JSON.stringify(value)}: ${err.message}`);
+        delete rows[key];
+      }
+    }
+    const want = desired(rows);
+    for (const s of SETTINGS){
+      setPath(config, s.key, structuredClone(want[s.key]));
+    }
+    running = structuredClone(want);
+    return service;
+  }
+
+  // For the settings page.
+  function list(){
+    const rows = stored(),
+      want = desired(rows);
+    return SETTINGS.map((s) => ({
+      ...s,
+      value: want[s.key],
+      fileValue: fileValues[s.key],
+      source: envLocked(s) ? 'env' : s.key in rows ? 'dashboard' : 'file',
+      updatedAt: rows[s.key]?.updatedAt,
+      updatedBy: rows[s.key]?.updatedBy,
+      pendingRestart: s.applies === 'restart' && !same(want[s.key], running?.[s.key])
+    }));
+  }
+
+  // `changes`: { key: rawValue } for the settings to set, `reset`: keys to
+  // put back to the config file's value. Validates everything first, so a
+  // bad value saves nothing. Returns what changed.
+  function save({ changes = {}, reset = [] }, { by, guard } = {}){
+    const rows = stored(),
+      before = desired(rows),
+      parsed = {},
+      errors = [];
+    for (const [key, raw]of Object.entries(changes)){
+      const s = BY_KEY.get(key);
+      if (!s || envLocked(s) || reset.includes(key)){
+        continue;
+      }
+      try {
+        parsed[key] = s.parse(raw);
+      }
+      catch (err){
+        errors.push(`${s.label}: ${err.message}`);
+      }
+    }
+    const after = { ...before, ...parsed };
+    for (const key of reset){
+      if (BY_KEY.has(key) && !envLocked(BY_KEY.get(key))){
+        after[key] = fileValues[key];
+      }
+    }
+    if (after['jobs.defaultTimeoutSec'] > after['jobs.maxTimeoutSec']){
+      errors.push('Default job timeout: can\'t be longer than the maximum job timeout');
+    }
+    if (!errors.length && guard){
+      const problem = guard(after);
+      if (problem){
+        errors.push(problem);
+      }
+    }
+    if (errors.length){
+      throw Object.assign(new Error(errors.join('\n')), { status: 400 });
+    }
+
+    const now = new Date().toISOString(),
+      changed = [];
+    db.transaction(() => {
+      for (const s of SETTINGS){
+        if (envLocked(s)){
+          continue;
+        }
+        if (reset.includes(s.key) && s.key in rows){
+          stmts.remove.run(s.key);
+        }
+        else if (s.key in parsed && !same(parsed[s.key], before[s.key])){
+          stmts.upsert.run(s.key, JSON.stringify(parsed[s.key]), now, by || null);
+        }
+        else {
+          continue;
+        }
+        if (!same(after[s.key], before[s.key])){
+          changed.push(s);
+        }
+      }
+    })();
+    for (const s of changed){
+      if (s.applies === 'now'){
+        setPath(config, s.key, structuredClone(after[s.key]));
+      }
+    }
+    return {
+      changed: changed.map((s) => s.key),
+      now: changed.filter((s) => s.applies === 'now').map((s) => s.key),
+      restart: changed.filter((s) => s.applies === 'restart').map((s) => s.key)
+    };
+  }
+
+  // thub-admin settings reset — the way back if a setting locked you out:
+  // the named settings (all when none) go back to the config file's value
+  // at the next start.
+  function reset(keys = []){
+    if (!keys.length){
+      return db.prepare('DELETE FROM settings').run().changes;
+    }
+    return keys.reduce((n, k) => n + stmts.remove.run(k).changes, 0);
+  }
+
+  const service = { list, save, reset, applyAtStartup, SETTINGS };
+  return service;
+}
+
+module.exports = { createSettingsService, SETTINGS };

package/src/web/routes.js

@@ -27,10 +27,14 @@ const fs = require('node:fs'),
   } = require('@andrian.yablonskyy/thub-common'),
   { retentionCutoff, JOB_RETENTION } = require('../services/cleanup'),
   listPrefs = require('../services/list-prefs'),
-  search = require('../services/search');
-
-// Job log viewer: lines per page it loads (and the most one request may ask for).
-const LOG_PAGE = 1000,
+  search = require('../services/search'),
+  { createLoginGuard } = require('../login-guard'),
+  { formatBytes } = require('../services/job-artifacts');
+
+// When this process started: the settings page tells a restart happened by it changing.
+const STARTED_AT = new Date().toISOString(),
+  // Job log viewer: lines per page it loads (and the most one request may ask for).
+  LOG_PAGE = 1000,
   LOG_PAGE_MAX = 5000,
 
   // Resources page sort keys (list-prefs.js) -> comparators. Empty values
@@ -70,7 +74,8 @@ const LOG_PAGE = 1000,
 // views hitting the same kind of SSE stream the Agent uses (§6.4), just
 // authenticated by session cookie instead of a bearer token.
 function createWebRouter({ services, config }){
-  const router = express.Router();
+  const router = express.Router(),
+    loginGuard = createLoginGuard();
 
   // Shown as a toast (layout.pug): `danger` stays until closed, `warning`
   // closes after 30 s, `info`/`success` after 10 s — unless `sticky`.
@@ -170,6 +175,8 @@ function createWebRouter({ services, config }){
     res.locals.isNewer = isNewer;
     res.locals.nextCronRun = nextCronRun;
     res.locals.formatDateTime = formatDateTime;
+    // 1233 -> "1.2 KB" (job artifacts).
+    res.locals.fmtBytes = formatBytes;
     res.locals.currentPath = req.originalUrl;
     // A live-update re-fetch (server.js, req.thubPassive) renders the page
     // only for its data: flash messages stay for the next real page view.
@@ -232,16 +239,35 @@ function createWebRouter({ services, config }){
     res.render('login', { title: 'Sign in', ...(problem ? { messages: [problem] } : {}) });
   });
 
-  router.post('/login', (req, res) => {
+  // Backoff after failed attempts, per IP and per username, checked before
+  // any hashing; a cap on password checks running at once (login-guard.js).
+  // The answer to a wrong password and to an unknown user is the same, and
+  // takes as long (admin-users.js).
+  function throttled(res, { reason, retryAfterSec }, what){
+    res.set('Retry-After', String(retryAfterSec));
+    const text = reason === 'busy'
+      ? 'The Coordinator is busy checking other sign-ins — try again in a moment.'
+      : `Too many failed ${what} — try again in ${retryAfterSec} s.`;
+    return { status: reason === 'busy' ? 503 : 429, text };
+  }
+
+  router.post('/login', async (req, res) => {
     const problem = httpsProblem(req);
     if (problem){
       return res.status(400).render('login', { title: 'Sign in', messages: [problem] });
     }
     const { username, password } = req.body,
-      user = services.adminUsers.verify(username, password);
+      allowed = loginGuard.check(req.ip, username);
+    if (!allowed.ok){
+      const { status, text } = throttled(res, allowed, 'sign-ins');
+      return res.status(status).render('login', { title: 'Sign in', messages: [{ type: 'danger', text }] });
+    }
+    const user = await loginGuard.run(() => services.adminUsers.verify(username, password));
     if (!user){
+      loginGuard.failure(req.ip, username);
       return res.status(401).render('login', { title: 'Sign in', messages: [{ type: 'danger', text: 'Invalid credentials' }] });
     }
+    loginGuard.success(req.ip, username);
     req.session.user = user;
     req.session.cookie.maxAge = user.sessionTimeoutMin * 60 * 1000;
     res.redirect('/');
@@ -333,17 +359,29 @@ function createWebRouter({ services, config }){
   // profile form since it needs the *current* password re-entered, and a
   // mismatch between newPassword/confirmPassword is a form-level check
   // that has nothing to do with the other fields.
-  router.post('/profile/password', (req, res) => {
-    const { currentPassword, newPassword, confirmPassword } = req.body;
+  // The current password is checked like a sign-in: the same backoff, so a
+  // stolen session can't be used to guess it.
+  router.post('/profile/password', async (req, res) => {
+    const { currentPassword, newPassword, confirmPassword } = req.body,
+      { username } = req.session.user;
     if (newPassword !== confirmPassword){
       flash(req, 'danger', 'New password and confirmation do not match.');
       return res.redirect('/profile');
     }
+    const allowed = loginGuard.check(req.ip, username);
+    if (!allowed.ok){
+      flash(req, 'danger', throttled(res, allowed, 'password checks').text);
+      return res.redirect('/profile');
+    }
     try {
-      services.adminUsers.changePassword(req.session.user.id, currentPassword, newPassword);
+      await loginGuard.run(() => services.adminUsers.changePassword(req.session.user.id, currentPassword, newPassword));
+      loginGuard.success(req.ip, username);
       flash(req, 'success', 'Password changed.');
     }
     catch (err){
+      if (err.status === 401){
+        loginGuard.failure(req.ip, username);
+      }
       flash(req, 'danger', err.message);
     }
     res.redirect('/profile');
@@ -589,6 +627,24 @@ function createWebRouter({ services, config }){
     res.redirect(returnTo(req, '/resources'));
   });
 
+  // Groups tab (resource card): the Client's group membership, as a config
+  // revision the Client writes to its file (registry.setClientGroups).
+  router.post('/resources/:id/groups', requireAdminRole, (req, res) => {
+    try {
+      const ids = [].concat(req.body.groups || []).filter((g) => typeof g === 'string' && g),
+        r = services.registry.setClientGroups(req.params.id, ids, {
+          by: req.session.user.username,
+          knownGroupIds: services.groups.list().map((g) => g.id)
+        });
+      flash(req, 'success', `Groups of ${r.name} saved: ${r.group_ids.length ? `${r.group_ids.length} group(s)` : 'none'} — in effect now.\n` +
+        'The Client writes them to its config file on its next heartbeat, then restarts once it has no job running.');
+    }
+    catch (err){
+      flash(req, 'danger', err.message);
+    }
+    res.redirect(returnTo(req, '/resources'));
+  });
+
   // Export (resource card, admins — the file holds the join key): the
   // Client's config file as JSON, secrets excluded.
   router.get('/resources/:id/config/export', requireAdminRole, (req, res) => {
@@ -989,6 +1045,97 @@ function createWebRouter({ services, config }){
     });
   }
 
+  // Coordinator settings (services/settings.js, README §13.2): stored in
+  // the database over the config file. "Restart now" only under systemd,
+  // whose Restart=always brings the process back (INVOCATION_ID is set for
+  // every unit it starts); run by hand, exiting would just stop it.
+  const canRestart = Boolean(process.env.INVOCATION_ID);
+
+  router.get('/admin/settings', requireAdminRole, (req, res) => {
+    const items = services.settings.list(),
+      groups = [...new Set(items.map((s) => s.group))].map((name) => ({ name, items: items.filter((s) => s.group === name) }));
+    res.render('admin/settings', {
+      title: 'Settings',
+      active: 'settings',
+      groups,
+      pendingRestart: items.filter((s) => s.pendingRestart),
+      canRestart,
+      startedAt: STARTED_AT,
+      restarting: req.query.restarting || null,
+      readOnly: {
+        listen: config.listen,
+        dataDir: config.dataDir,
+        configPath: config.configPath || '(built-in defaults)'
+      }
+    });
+  });
+
+  router.post('/admin/settings', requireAdminRole, (req, res) => {
+    const posted = req.body.s || {},
+      changes = {},
+      reset = [].concat(req.body.reset || []);
+    for (const s of services.settings.SETTINGS){
+      if (s.type === 'secret'){
+        // Empty means "unchanged": the current secret is never put in the page.
+        if (req.body.clear?.[s.key] === '1'){
+          changes[s.key] = '';
+        }
+        else if (posted[s.key]){
+          changes[s.key] = posted[s.key];
+        }
+      }
+      else if (s.key in posted){
+        // A checkbox posts a hidden "false" and, when checked, "true" after it.
+        changes[s.key] = [].concat(posted[s.key]).at(-1);
+      }
+    }
+    try {
+      const result = services.settings.save({ changes, reset }, {
+        by: req.session.user.username,
+        // Don't let an admin lock everyone out: an HTTPS-only cookie
+        // can't come back over the plain-HTTP connection they're using.
+        guard: (after) => {
+          const secureCookie = after['session.secureCookie'],
+            httpsOnly = secureCookie === true || (secureCookie === 'auto' && /^https:\/\//i.test(after.publicUrl || ''));
+          return httpsOnly && !req.secure
+            ? 'Secure session cookie: you\'re connected over plain HTTP, so with this setting nobody could sign in this way ' +
+              'after the restart. Open the dashboard over https:// first (behind a proxy: it must send X-Forwarded-Proto: https), ' +
+              'or set it to false.'
+            : null;
+        }
+      });
+      if (!result.changed.length){
+        flash(req, 'info', 'Nothing changed.');
+      }
+      else {
+        const label = (k) => services.settings.SETTINGS.find((s) => s.key === k).label;
+        flash(req, result.restart.length ? 'warning' : 'success', [
+          result.now.length ? `Saved and applied: ${result.now.map(label).join(', ')}.` : '',
+          result.restart.length ? `Saved, applies after a restart: ${result.restart.map(label).join(', ')}.` : ''
+        ].filter(Boolean).join('\n'));
+      }
+    }
+    catch (err){
+      flash(req, 'danger', err.message);
+    }
+    res.redirect('/admin/settings');
+  });
+
+  router.post('/admin/settings/restart', requireAdminRole, (req, res) => {
+    if (!canRestart){
+      flash(req, 'danger', 'This Coordinator isn\'t running under systemd, so it can\'t restart itself — restart it by hand.');
+      return res.redirect('/admin/settings');
+    }
+    res.redirect(`/admin/settings?restarting=${encodeURIComponent(STARTED_AT)}`);
+    // After the redirect is out; systemd (Restart=always) starts it again.
+    res.on('finish', () => setTimeout(() => process.kill(process.pid, 'SIGTERM'), 500));
+  });
+
+  // Polled by the settings page while restarting (public/js/settings.js).
+  router.get('/admin/settings/status', requireAdminRole, (req, res) => {
+    res.set('Cache-Control', 'no-store').json({ startedAt: STARTED_AT });
+  });
+
   router.get('/admin/agents', requireAdminRole, (req, res) => renderAgents(req, res));
 
   router.post('/admin/agents', requireAdminRole, (req, res) => {

package/test/admin-users.test.js

@@ -23,11 +23,11 @@ function buildService(){
   return createAdminUsersService(db);
 }
 
-test('a newly created user gets profile defaults (UTC, auto theme, 60min timeout)', () => {
+test('a newly created user gets profile defaults (UTC, auto theme, 60min timeout)', async () => {
   const adminUsers = buildService();
   adminUsers.create({ username: 'alice', password: 'hunter2', role: 'admin' });
 
-  const user = adminUsers.verify('alice', 'hunter2');
+  const user = await adminUsers.verify('alice', 'hunter2');
   assert.equal(user.timezone, 'UTC');
   assert.equal(user.theme, 'auto');
   assert.equal(user.sessionTimeoutMin, 60);
@@ -80,48 +80,48 @@ test('a session timeout outside the offered presets is rejected (400)', () => {
   assert.throws(() => adminUsers.updateProfile(id, { sessionTimeoutMin: 7 }), (err) => err.status === 400);
 });
 
-test('resetPassword creates the user as admin when the username does not exist yet', () => {
+test('resetPassword creates the user as admin when the username does not exist yet', async () => {
   const adminUsers = buildService();
 
   adminUsers.resetPassword({ username: 'admin', password: 'newpass' });
-  const user = adminUsers.verify('admin', 'newpass');
+  const user = await adminUsers.verify('admin', 'newpass');
   assert.ok(user);
   assert.equal(user.role, 'admin');
 });
 
-test('resetPassword on an existing user changes only the password, not the role', () => {
+test('resetPassword on an existing user changes only the password, not the role', async () => {
   const adminUsers = buildService();
   adminUsers.create({ username: 'carol', password: 'oldpass', role: 'viewer' });
 
   adminUsers.resetPassword({ username: 'carol', password: 'newpass' });
 
-  assert.equal(adminUsers.verify('carol', 'oldpass'), null);
-  const user = adminUsers.verify('carol', 'newpass');
+  assert.equal(await adminUsers.verify('carol', 'oldpass'), null);
+  const user = await adminUsers.verify('carol', 'newpass');
   assert.ok(user);
   assert.equal(user.role, 'viewer'); // not silently promoted to admin
 });
 
-test('changePassword succeeds with the correct current password', () => {
+test('changePassword succeeds with the correct current password', async () => {
   const adminUsers = buildService(),
     { id } = adminUsers.create({ username: 'dave', password: 'oldpass', role: 'admin' });
 
-  adminUsers.changePassword(id, 'oldpass', 'newpass');
+  await adminUsers.changePassword(id, 'oldpass', 'newpass');
 
-  assert.equal(adminUsers.verify('dave', 'oldpass'), null);
-  assert.ok(adminUsers.verify('dave', 'newpass'));
+  assert.equal(await adminUsers.verify('dave', 'oldpass'), null);
+  assert.ok(await adminUsers.verify('dave', 'newpass'));
 });
 
-test('changePassword is rejected (401) with the wrong current password, leaving it unchanged', () => {
+test('changePassword is rejected (401) with the wrong current password, leaving it unchanged', async () => {
   const adminUsers = buildService(),
     { id } = adminUsers.create({ username: 'dave', password: 'oldpass', role: 'admin' });
 
-  assert.throws(() => adminUsers.changePassword(id, 'wrongpass', 'newpass'), (err) => err.status === 401);
-  assert.ok(adminUsers.verify('dave', 'oldpass'));
+  await assert.rejects(adminUsers.changePassword(id, 'wrongpass', 'newpass'), (err) => err.status === 401);
+  assert.ok(await adminUsers.verify('dave', 'oldpass'));
 });
 
-test('changePassword rejects an empty new password (400)', () => {
+test('changePassword rejects an empty new password (400)', async () => {
   const adminUsers = buildService(),
     { id } = adminUsers.create({ username: 'dave', password: 'oldpass', role: 'admin' });
 
-  assert.throws(() => adminUsers.changePassword(id, 'oldpass', ''), (err) => err.status === 400);
+  await assert.rejects(adminUsers.changePassword(id, 'oldpass', ''), (err) => err.status === 400);
 });