fhirsmith 0.18.1 → 1.0.0

package/server.js

@@ -6,34 +6,83 @@
 // Licensed under BSD-3: https://opensource.org/license/bsd-3-clause
 //
 
+// Production unless told otherwise: Express and other libraries check NODE_ENV (in development
+// mode Express's error pages include stack traces), and a server where nobody set it should get
+// the safe behaviour. `npm run dev` (node --watch) runs in development mode
+process.env.NODE_ENV ??= process.execArgv.includes('--watch') ? 'development' : 'production';
+
+// Logging starts before anything else, so that a server that fails to start (bad config.json, a
+// missing dependency, an unwritable folder) says why in the log as well as on stderr. It starts with
+// the logger defaults; config.logging is applied once config.json has been read
+const folders = require('./library/folder-setup');
+const Logger = require('./library/logger');
+let bootLog;
+try {
+  bootLog = Logger.getInstance().child({ module: 'server' });
+} catch (error) {
+  console.error('Failed to start logging:', error.message);
+  process.exit(1);
+}
+bootLog.info(`FHIRsmith starting (PID ${process.pid}, Node.js ${process.version}), data directory ${folders.dataDir()}`);
+
+// Until startup is complete, an uncaught exception is fatal: log it and exit (the logger flushes on exit).
+// Replaced by the long-running handlers once the server is set up
+function startupFailure(error) {
+  console.error('FATAL - startup failed:', error);
+  bootLog.error('FATAL - startup failed: ' + (error?.message || error), error);
+  process.exit(1);
+}
+process.on('uncaughtException', startupFailure);
+
 const express = require('express');
 const cors = require('cors');
 const path = require('path');
 const fs = require('fs');
 const os = require('os');
-const v8 = require('v8');
-const folders = require('./library/folder-setup');  // <-- ADD: load early
 const { statSync, readdirSync } = require('fs');
 const escape = require('escape-html');
 const { resolveWithin } = require('./library/path-safety');
-const { readCgroupMemoryLimit } = require('./library/cgroup-memory');
+const { readProcessMemory } = require('./library/cgroup-memory');
+const memoryGuard = require('./library/memory-guard');
+const { applyKeepAliveTimeouts } = require('./library/keep-alive');
+const { countStaticResponses } = require('./library/static-response-stats');
+const { trimNativeMemory } = require('./library/native-memory');
 const { NpmAudit } = require('./library/npm-audit');
 const Mailer = require('./library/mailer');
 const DailyReport = require('./library/daily-report');
+const { maxBodyBytes } = require('./library/body-limit');
 
-// Load configuration BEFORE logger
+// Load configuration (from the data directory)
 let config;
+const configPath = folders.filePath('config.json');
+bootLog.info(`Reading configuration from ${configPath}`);
 try {
-  const configPath = folders.filePath('config.json');  // <-- CHANGE: config now in data dir
-  const configData = fs.readFileSync(configPath, 'utf8');
-  config = JSON.parse(configData);
+  // JSON, with // and /* */ comments allowed (library/config-reader.js)
+  config = require('./library/config-reader').readConfig(configPath);
+  // the server block is optional (every setting in it has a default)
+  config.server = config.server || {};
 } catch (error) {
-  console.error('Failed to load configuration:', error.message);
+  console.error(`Failed to load configuration from ${configPath}:`, error.message);
+  bootLog.error(`Failed to load configuration from ${configPath}: ${error.message}`);
   process.exit(1);
 }
 
-const Logger = require('./library/logger');
-const serverLog = Logger.getInstance(config.logging || {}).child({ module: 'server' });
+// --role=master|tx-worker (cluster.md). With no role this is null and the configuration is
+// untouched: every `if (role)` below is skipped
+const { parseRole, applyRole, describeRole } = require('./library/role');
+let role;
+try {
+  role = parseRole(process.argv.slice(2), process.env);
+  if (role) {
+    applyRole(config, role);
+  }
+} catch (error) {
+  console.error(error.message);
+  process.exit(1);
+}
+
+Logger.getInstance().configure(config.logging || {});
+const serverLog = Logger.getInstance().child({ module: 'server' });
 const packageJson = require('./package.json');
 
 // Startup banner
@@ -43,11 +92,10 @@ serverLog.info(`========================================`);
 serverLog.info(`FHIRsmith v${packageJson.version} starting (PID ${process.pid})`);
 serverLog.info(`Node.js ${process.version} on ${os.type()} ${os.release()} (${os.arch()})`);
 serverLog.info(`Memory: ${freeMemGB} GB free / ${totalMemGB} GB total`);
-{
-  const cg = readCgroupMemoryLimit();
-  serverLog.info(cg.limit > 0
-    ? `Memory limit: ${(cg.limit / 1024 / 1024 / 1024).toFixed(1)} GB (from ${cg.source})`
-    : 'Memory limit: none found in cgroup - RSS guard disabled');
+memoryGuard.configure(config.server.memory || {});
+serverLog.info(memoryGuard.describe());
+if (role) {
+  serverLog.info(describeRole(role, config));
 }
 serverLog.info(`Data directory: ${folders.dataDir()}`);
 serverLog.info(`========================================`);
@@ -72,12 +120,17 @@ const IpsModule = require('./ips/ips.js');
 
 const htmlServer = require('./library/html-server');
 const ServerStats = require("./stats");
+const { connectMetrics, graphData, externalSummary } = require('./library/cluster-metrics');
+const { connectCounts } = require('./library/cluster-counts');
+const { connectClusterEvents } = require('./library/cluster-events');
+const { connectStatus, ownFigures, clusterView, processView, storageView, clusterSummary, processSummary, STATUS_INTERVAL_MS } = require('./library/cluster-status');
 const {Liquid} = require("liquidjs");
 const FolderModule = require("./folder/folder");
 const ExtensionTrackerModule = require("./extension-tracker/extension-tracker");
 
 htmlServer.useLog(serverLog);
 htmlServer.setSponsorMessage(config.sponsorMessage ? config.sponsorMessage : '');
+htmlServer.setAccessibilityPanel(config.accessibility?.checkPanel === true);
 
 const app = express();
 
@@ -90,18 +143,42 @@ const app = express();
 const DEFAULT_TRUST_PROXY = 'loopback, linklocal, uniquelocal';
 app.set('trust proxy', config.server.trustProxy ?? DEFAULT_TRUST_PROXY);
 
-const PORT = process.env.PORT || config.server.port || 3000;
+const PORT = (role && role.port) || process.env.PORT || config.server.port || 3000;
+// with a role, loopback unless server.cluster.bindAddress says otherwise (cluster.md); with
+// none, every interface, as before
+const HOST = role ? require('./library/role').bindAddress(config) : null;
 
 // Middleware
+// The requests this process has in flight, for the status panel (library/cluster-status.js)
+let inFlight = 0;
+app.use((req, res, next) => {
+  inFlight++;
+  let done = false;
+  const finished = () => {
+    if (!done) {
+      done = true;
+      inFlight--;
+    }
+  };
+  res.once('finish', finished);
+  res.once('close', finished);
+  next();
+});
+// Refuse new requests (503) while memory is near the limit - first, so a refused request
+// doesn't even have its body parsed. /dashboard stays reachable so the state can be seen.
+app.use(memoryGuard.middleware({ log: serverLog, exempt: ['/dashboard'],
+  onReject: role ? () => stats?.countRefusedRequest() : undefined }));
 // The body parsers, for every module but those that read their own bodies with their own
 // limits and checks (the ips module: a document is read only once its submitter's token has
 // been checked, and no bigger than the module's maxSize)
 const OWN_BODIES = ['/ips'];
 const ownsBody = (req) => OWN_BODIES.some(p => req.path === p || req.path.startsWith(p + '/'));
 const unlessOwnBody = (parser) => (req, res, next) => (ownsBody(req) ? next() : parser(req, res, next));
-app.use(unlessOwnBody(express.raw({ type: 'application/fhir+json', limit: '50mb' })));
-app.use(unlessOwnBody(express.raw({ type: 'application/fhir+xml', limit: '50mb' })));
-app.use(unlessOwnBody(express.json({ limit: '50mb' })));
+// Bodies up to server.maxBodyMB (default 50); a bigger one gets a 413
+const maxBody = maxBodyBytes(config.server);
+app.use(unlessOwnBody(express.raw({ type: 'application/fhir+json', limit: maxBody })));
+app.use(unlessOwnBody(express.raw({ type: 'application/fhir+xml', limit: maxBody })));
+app.use(unlessOwnBody(express.json({ limit: maxBody })));
 app.use((req, res, next) => {
   const requestId = req.headers['x-request-id'];
   if (requestId) {
@@ -120,6 +197,19 @@ app.use(cors(config.server.cors));
 const modules = {};
 
 let stats = null;
+let disconnectMetrics = () => {};
+let disconnectCounts = null;
+let disconnectClusterEvents = () => {};
+let disconnectStatus = () => {};
+// The graphs on / and /dashboard, one component for every server (library/cluster-metrics.js):
+// on the master under the launcher, every process it reports; otherwise this process's own
+// samples as one series, with its memory guard's refusal threshold. `level`: the graphs' titles'
+function metricsTemplate(page, level = 4) {
+  const label = !role ? 'This process' : `${role.role === 'master' ? 'Master' : 'Worker'} ${config.modules.tx.instanceCode}`;
+  const guard = memoryGuard._state;
+  // (legend: false - statusContent puts the graphs' key under "More")
+  return [`cluster-${page}`, { metricsJson: graphData(stats, label, Date.now(), { rejectAt: guard.rejectAt, group: Boolean(guard.group) }), level, legend: false }];
+}
 // periodic npm advisory check of the installed packages - see library/npm-audit.js
 let npmAudit = null;
 // the daily report, emailed at midnight when "email" and "dailyReport.to" are configured - see library/daily-report.js
@@ -128,6 +218,18 @@ let dailyReport = null;
 // Initialize modules based on configuration
 async function initializeModules() {
   stats = new ServerStats(config.stats, serverLog);
+  if (role) {
+    disconnectMetrics = connectMetrics(stats);
+    // a worker's request counts go to the master's statistics (library/cluster-counts.js)
+    disconnectCounts = connectCounts(stats, process, { role: role.role });
+    // and on the master, what the launcher saw, for the daily report (library/cluster-events.js)
+    disconnectClusterEvents = connectClusterEvents(stats, process, { role: role.role });
+    // this process's figures for the status panel, and on the master the launcher's snapshot
+    // of them all (library/cluster-status.js)
+    disconnectStatus = connectStatus(stats, process, { role: role.role,
+      intervalMs: Number(config.server.cluster?.statusSeconds ?? STATUS_INTERVAL_MS / 1000) * 1000,
+      figures: () => ownFigures(stats, { inFlight: () => inFlight, refusing: () => memoryGuard.shouldReject() }) });
+  }
   npmAudit = new NpmAudit(config.npmAudit || {}, Logger.getInstance().child({ module: 'npm-audit' }), __dirname, stats);
 
   // Initialize SHL module
@@ -334,6 +436,154 @@ async function loadTemplates() {
   }
 }
 
+// The request count in /dashboard/status.json: on the master under the launcher, what its
+// statistics hold, its own and the workers' it has merged (library/cluster-counts.js), which
+// arrive up to a minute late; otherwise this process's
+function requestCounts() {
+  if (role && role.role === 'master' && process.send) {
+    return {
+      count: stats.requestCount + stats.mergedRequestCount,
+      detail: `master: ${stats.requestCount}, workers: ${stats.mergedRequestCount}, static: ${stats.staticRequestCount}`
+    };
+  }
+  return { count: stats.requestCount, detail: `static: ${stats.staticRequestCount}` };
+}
+
+// The status panel's view (library/cluster-status.js, tx/html/status-panel.liquid): the
+// cluster's on the master under the launcher, once its first snapshot has come; else this
+// process's own, labelled as what it is
+function statusView() {
+  const [uptime, uptimeSub] = uptimeParts(Date.now() - stats.startTime);
+  const host = { used: os.totalmem() - os.freemem(), total: os.totalmem() };
+  const cluster = clusterStatus();
+  if (cluster) {
+    return clusterView(cluster, { uptime, uptimeSub, host });
+  }
+  // a fresh enough measure, as the guard takes it
+  memoryGuard.shouldReject();
+  const g = memoryGuard._state;
+  const own = readProcessMemory('self');
+  const minutes = (Date.now() - stats.startTime) / 60000;
+  const code = role ? config.modules.tx.instanceCode : null;
+  return processView({
+    figures: ownFigures(stats, { inFlight: () => inFlight, refusing: () => memoryGuard.shouldReject() }),
+    uptime, uptimeSub, host, static: stats.staticRequestCount,
+    avgPerMin: minutes >= 1 ? stats.requestCount / minutes : null,
+    anon: own ? own.anon : null,
+    guard: { measure: g.sampleRss, limit: g.limit, rejectAt: g.rejectAt, abortAt: g.abortAt, rejectFraction: g.rejectFraction,
+      abortFraction: g.abortFraction, group: Boolean(g.group), fileBacked: g.excludeFileBacked }
+  }, !role ? { heading: 'Server', meta: ['one process'] } : {
+    heading: `${role.role === 'master' ? 'Master' : 'Worker'} ${code}`,
+    meta: role.role === 'master' && process.send ? ['this process', 'waiting for the launcher\'s first snapshot'] : ['this process only']
+  });
+}
+
+// Every background task, in the status panel's table style (tx/html/status-panel.liquid), in a
+// box of its own that scrolls rather than the page; under "More"
+function backgroundTasks(level) {
+  const table = stats.taskDetails();
+  return table ? `<section class="fs-status fs-status-tasks"><h${level} class="fs-status-head">All background tasks</h${level}>` +
+    `<div class="fs-status-scroll">${table}</div></section>` : '';
+}
+
+// The status line at the top of / and /dashboard (tx/html/status-line.liquid): requests, uptime
+// and heap, the cluster's on the master under the launcher, else this process's; and the problems
+function statusLine() {
+  const [uptime, uptimeSub] = uptimeParts(Date.now() - stats.startTime);
+  // low disk space is a problem, and also on its own line above "More", in red (see statusContent)
+  const logs = logFigures();
+  const disk = logs ? logs.disk : null;
+  const failedTasks = stats.failedTasks();
+  const other = npmAudit && npmAudit.enabled && npmAudit.checkedAt && npmAudit.isSerious() ? [`npm audit: ${npmAudit.summary()}`] : [];
+  const cluster = clusterStatus();
+  if (cluster) {
+    return clusterSummary(cluster, { uptime, uptimeSub, disk, failedTasks, other });
+  }
+  // a fresh enough measure, as the guard takes it
+  memoryGuard.shouldReject();
+  const g = memoryGuard._state;
+  const minutes = (Date.now() - stats.startTime) / 60000;
+  return processSummary({
+    figures: ownFigures(stats, { inFlight: () => inFlight, refusing: () => memoryGuard.shouldReject() }),
+    uptime, uptimeSub, avgPerMin: minutes >= 1 ? stats.requestCount / minutes : null,
+    guard: { measure: g.sampleRss, rejectAt: g.rejectAt }, disk, failedTasks, other
+  });
+}
+
+// under this share of the disk free, a warning in red above "More"
+const DISK_LOW = 0.1;
+
+// Low disk space (the disk the logs are on), as a warning in red; '' when there is enough
+function diskWarning() {
+  const logs = logFigures();
+  const disk = logs ? logs.disk : null;
+  if (!disk || !(disk.total > 0) || disk.free >= DISK_LOW * disk.total) {
+    return '';
+  }
+  const gb = (bytes) => (bytes / 2 ** 30).toFixed(1);
+  return `<span style="color:#b00"><strong><span aria-hidden="true">⚠ </span>Disk space low:</strong> ` +
+    `${gb(disk.free)} GB free of ${gb(disk.total)} GB (${Math.round(100 * disk.free / disk.total)}%)</span>`;
+}
+
+// The top of / and /dashboard: the status line, the graphs, the background tasks at work, one
+// line of what the server needs looking at (`notes`: html, with low disk space added), then under
+// "More" the graphs' key, the status panel, storage, every background task and `more` (html).
+// Headings start at `level`, the page's sections' own.
+async function statusContent(liquid, page, level, { notes = [], details = '', more = '' } = {}) {
+  let content = await liquid.renderFile('status-line', { line: statusLine(), level });
+  content += await liquid.renderFile(...metricsTemplate(page, level + 1));
+  content += await liquid.renderFile('status-tasks', { tasks: stats.taskSummary(), level });
+  const line = [...notes, diskWarning()].filter(Boolean);
+  if (line.length) {
+    content += `<div id="npm-audit" style="font-size:13px;margin:0 0 8px"><p style="margin:0 0 4px">${line.join(' · ')}</p>${details}</div>`;
+  }
+  content += `<details class="fs-status-more"><summary><h${level}>More</h${level}>: the graphs' key, every process, memory, storage, every background task</summary>`;
+  // the graphs' key (tx/html/cluster-metrics-script.liquid puts it here)
+  content += '<div id="serverMetricsLegend" class="server-metrics-legend"></div>';
+  content += await liquid.renderFile('status-panel', { view: statusView(), level: level + 1 });
+  const external = externalStatus();
+  if (external) {
+    content += await liquid.renderFile('external-metrics', { external, level: level + 1 });
+  }
+  content += await liquid.renderFile('status-panel', { view: storageStatusView(), level: level + 1 });
+  content += backgroundTasks(level + 1);
+  content += more;
+  content += '</details>';
+  return content;
+}
+
+// The external requests - those that had to ask another server (tx ecosystem resolution,
+// ServerStats.countExternalRequest) - in the graphs' samples (every process's, on the master
+// under the launcher): {count, average, span} for tx/html/external-metrics.liquid, or null if
+// there have been none in them
+function externalStatus(now = Date.now()) {
+  const summary = externalSummary(stats, now);
+  if (!summary) {
+    return null;
+  }
+  const { count, average, hours } = summary;
+  return {
+    count: count.toLocaleString(),
+    average: average >= 1000 ? `${(average / 1000).toFixed(1)} s` : `${Math.round(average)} ms`,
+    span: hours >= 23.5 ? '24 hours' : hours >= 1.5 ? `${Math.round(hours)} hours` : hours >= 1 || Math.round(hours * 60) <= 1 ? (hours >= 1 ? 'hour' : 'minute') : `${Math.round(hours * 60)} minutes`
+  };
+}
+
+// The storage tiles below the graphs: closure tables, logs, disk
+function storageStatusView() {
+  const logs = logFigures();
+  const closure = stats.closureStats();
+  return storageView({
+    heading: 'Storage', closure, logs: logs && logs.files !== null ? logs : null, disk: logs && logs.disk
+  });
+}
+
+// The launcher's snapshot of every process (library/cluster-status.js), on the master under
+// the launcher once one has come; else null, and the table is this process's
+function clusterStatus() {
+  return role && role.role === 'master' && stats.clusterStatus ? stats.clusterStatus : null;
+}
+
 async function buildRootPageContent() {
   stats.requestCount++;
   let content = npmAudit ? npmAudit.renderBanner() : '';
@@ -431,6 +681,7 @@ async function buildRootPageContent() {
       content += '</li>';
     }
     content += '</ul>';
+    content += '</li>';
   }
 
 
@@ -454,72 +705,12 @@ async function buildRootPageContent() {
 
   content += '<hr/>';
 
-  // Calculate uptime
-  const uptimeMs = Date.now() - stats.startTime;
-  const uptimeStr = uptimeText(uptimeMs);
-
-  // Memory usage
-  const memUsage = process.memoryUsage();
-  const heapStats = v8.getHeapStatistics();
-
-  // V8 heap: used vs limit
-  const v8UsedMB = (memUsage.heapUsed / 1024 / 1024).toFixed(0);
-  const v8LimitMB = (heapStats.heap_size_limit / 1024 / 1024).toFixed(0);
-  const v8PCT = (memUsage.heapUsed * 100) / heapStats.heap_size_limit;
-
-  // Process RSS vs cgroup limit (or system total)
-  const rssMB = (memUsage.rss / 1024 / 1024).toFixed(0);
-  const memLimit = readCgroupMemoryLimit().limit || os.totalmem();
-  const memLimitMB = (memLimit / 1024 / 1024).toFixed(0);
-  const processPCT = (memUsage.rss * 100) / memLimit;
-
-  // System memory
-  const totalMemMB = (os.totalmem() / 1024 / 1024).toFixed(0);
-  const usedMemMB = ((os.totalmem() - os.freemem()) / 1024 / 1024).toFixed(0);
-  const sysMemPCT = ((os.totalmem() - os.freemem()) * 100) / os.totalmem();
-
-  // Average requests per minute
-  const uptimeMinutesTotal = uptimeMs / 60000;
-  const avgReqPerMin = uptimeMinutesTotal > 0 ? (stats.requestCount / uptimeMinutesTotal).toFixed(1) : '0.0';
-
-  content += '<table class="grid">';
-  content += '<tr>';
-  content += `<td><strong>Uptime:</strong> ${escape(uptimeStr)}</td>`;
-  content += `<td><strong>Request Count:</strong> ${stats.requestCount} (static: ${stats.staticRequestCount})</td>`;
-  content += `<td><strong>Avg Requests/min:</strong> ${avgReqPerMin}</td>`;
-  content += '</tr>';
-  content += '<tr>';
-  content += `<td style="background-color:${pctColor(v8PCT)}"><strong>V8 Memory:</strong> ${v8UsedMB} MB of ${v8LimitMB} MB (${v8PCT.toFixed(0)}%)</td>`;
-  content += `<td style="background-color:${pctColor(processPCT)}"><strong>Process Memory:</strong> ${rssMB} MB of ${memLimitMB} MB (${processPCT.toFixed(0)}%)</td>`;
-  content += `<td style="background-color:${pctColor(sysMemPCT)}"><strong>System Memory:</strong> ${usedMemMB} MB of ${totalMemMB} MB (${sysMemPCT.toFixed(0)}%)</td>`;
-  content += '</tr>';
-  content += '<tr>';
-  content += `<td><strong>Expansion Cache:</strong> ${stats.expansionItems()} items</td>`;
-  content += `<td><strong>Client Cache:</strong> ${stats.clientCaches()} caches / ${stats.clientConcepts()} concepts</td>`;
-  content += `<td><strong>Max Client Cache:</strong> ${stats.maxClientCaches()} caches / ${stats.maxClientConcepts()} concepts</td>`;
-  content += '</tr>';
-  const closure = stats.closureStats();
-  if (closure) {
-    content += '<tr>';
-    content += `<td><strong>Closure Tables:</strong> ${closure.tables} tables` +
-      (closure.retentionDays ? ` (unused for ${closure.retentionDays} days = deleted)` : '') + `</td>`;
-    content += `<td><strong>Closure Entries:</strong> ${closure.entries} entries / ${closure.concepts} concepts</td>`;
-    content += `<td><strong>Closure Database:</strong> ${closure.bytes === null ? 'in memory' : formatBytes(closure.bytes)}</td>`;
-    content += '</tr>';
-  }
-  content += getLogStats();
-  content += '</table>';
-
-  // ===== Metrics Graphs =====
+  // the status line, the graphs and the tasks at work; the rest under "More" (library/cluster-status.js)
   const liquid = new Liquid({
     root: path.join(__dirname, 'tx', 'html'),
     extname: '.liquid'
   });
-  content += await liquid.renderFile('home-metrics', {
-    historyJson: JSON.stringify(stats.history),
-    startTime: stats.startTime
-  });
-  content += stats.taskDetails();
+  content += await statusContent(liquid, 'home-metrics', 3);
 
   content += '</div>';
   return content;
@@ -531,11 +722,16 @@ process.on('unhandledRejection', (reason, promise) => {
   serverLog.error('Unhandled Rejection:', reason);
 });
 
-process.on('uncaughtException', (error) => {
-  console.error('FATAL - Uncaught Exception:', error);
-  serverLog.error('FATAL - Uncaught Exception:', error);
-  process.exitCode = 1;
-});
+// Once the server is listening, startup is over: an uncaught exception is logged, and the
+// server carries on (startupFailure, which exits, covers everything until then - see startServer)
+function startupComplete() {
+  process.off('uncaughtException', startupFailure);
+  process.on('uncaughtException', (error) => {
+    console.error('FATAL - Uncaught Exception:', error);
+    serverLog.error('FATAL - Uncaught Exception:', error);
+    process.exitCode = 1;
+  });
+}
 
 app.get('/', async (req, res) => {
   // If an override index.html exists, serve it instead of the FHIRsmith home page
@@ -597,20 +793,14 @@ app.get('/', async (req, res) => {
 app.get('/fhirsmith', (req, res) => serveFhirsmithHome(req, res));
 
 // Serve static files
-// Count static file hits separately from API/page requests
-app.use((req, res, next) => {
-  res.on('finish', () => {
-    if (res.statusCode >= 200 && res.statusCode < 300) {
-      stats.staticRequestCount++;
-    }
-  });
-  next();
-});
+// Count static file hits separately from API/page requests: only responses the static
+// handlers below send themselves, not a later route's response to a request that fell through
+const staticFiles = express.Router();
 if (config.server?.webBase) {
   const overrideDir = path.resolve(config.server.webBase);
   // Anything under overrideDir may be served; nothing outside it may - see
   // library/path-safety.js for why path.join is not enough on its own.
-  app.use((req, res, next) => {
+  staticFiles.use((req, res, next) => {
     const filePath = resolveWithin(overrideDir, req.path);
     if (filePath === null) {
       return next();
@@ -627,7 +817,36 @@ if (config.server?.webBase) {
     });
   });
 }
-app.use(express.static(path.join(__dirname, 'static')));
+staticFiles.use(express.static(path.join(__dirname, 'static')));
+app.use(countStaticResponses(staticFiles, () => { stats.staticRequestCount++; }));
+
+// The status panel's figures, machine-readable (cluster.md, "Status"). Under /dashboard, so the
+// memory guard never refuses it. On the master under the launcher: the launcher's snapshot of
+// every process (library/cluster-status.js); otherwise this process's own figures
+app.get('/dashboard/status.json', (req, res) => {
+  stats.requestCount++;
+  const requests = requestCounts();
+  const counts = { count: requests.count, detail: requests.detail };
+  const cluster = clusterStatus();
+  let body;
+  if (cluster) {
+    body = { scope: 'cluster', generatedAt: Date.now(), requests: counts, ...cluster };
+  } else {
+    const guard = memoryGuard._state;
+    const own = readProcessMemory('self');
+    body = {
+      scope: 'process', generatedAt: Date.now(), requests: counts,
+      role: role ? role.role : null, code: role ? config.modules.tx.instanceCode : null,
+      process: { ...ownFigures(stats, { inFlight: () => inFlight, refusing: () => memoryGuard.shouldReject() }),
+        uptimeMs: Date.now() - stats.startTime, rss: process.memoryUsage().rss,
+        anon: own ? own.anon : null, file: own ? own.file : null },
+      memory: { limit: guard.limit, limitSource: guard.source, rejectAt: guard.rejectAt, abortAt: guard.abortAt,
+        rejectFraction: guard.rejectFraction, abortFraction: guard.abortFraction, group: Boolean(guard.group) }
+    };
+  }
+  res.setHeader('Cache-Control', 'no-store');
+  res.json(body);
+});
 
 // Dashboard endpoint - server name, stats, graphs, and background tasks (no modules list)
 app.get('/dashboard', async (req, res) => {
@@ -639,52 +858,21 @@ app.get('/dashboard', async (req, res) => {
     }
 
     const startTime = Date.now();
-    // Calculate uptime
-    const uptimeMs = Date.now() - stats.startTime;
-    const uptimeStr = uptimeText(uptimeMs);
-
-    // Memory usage
-    const memUsage = process.memoryUsage();
-    const heapStats = v8.getHeapStatistics();
-    const v8PCT = (memUsage.heapUsed * 100) / heapStats.heap_size_limit;  // % of V8 heap limit used
-
-    // Process RSS as % of cgroup memory limit (or system total as fallback)
-    const memLimit = readCgroupMemoryLimit().limit || os.totalmem();
-    const processPCT = (memUsage.rss * 100) / memLimit;
-
-    const totalMemBytes = os.totalmem();
-    const freeMemBytes = os.freemem();
-    const sysMemPCT = ((totalMemBytes - freeMemBytes) * 100) / totalMemBytes;  // % of system memory used
-
-    const memMaxPCT = Math.max(v8PCT, processPCT, sysMemPCT);
-
-    const fstats = fs.statfsSync(folders.logsDir());
-    const diskPCT = 100 - ((fstats.bavail * 100) / fstats.blocks);  // % of disk used
-
-    let content = '<style>table.grid{margin-bottom:10px;border:1px solid black;margin-right:auto}table.grid th,table.grid td{border:1px solid silver;padding:3px 7px 2px;font-size:12px;line-height:1.4em;font-family:verdana;vertical-align:top}table.grid th{font-weight:bold}table.grid td{font-weight:normal}</style>';
-    content += '<table class="grid">';
-    content += '<tr>';
-    content += `<td><strong>Uptime:</strong> ${escape(uptimeStr)}</td>`;
-    content += `<td><strong>Request Count:</strong> ${stats.requestCount} (static: ${stats.staticRequestCount})</td>`;
-    content += `<td style="background-color:${pctColor(memMaxPCT)}"><strong>Memory:</strong> ${v8PCT.toFixed(0)}% V8, ${processPCT.toFixed(0)}% Process, ${sysMemPCT.toFixed(0)}% System</td>`;
-    content += `<td style="background-color:${pctColor(diskPCT)}"><strong>Disk:</strong> ${diskPCT.toFixed(0)}%</td>`;
-    content += '</tr>';
-    content += '</table>';
-
-    // ===== Metrics Graphs =====
+    // the status line, the graphs and the tasks at work; the rest under "More" (library/cluster-status.js)
     const liquid = new Liquid({
       root: path.join(__dirname, 'tx', 'html'),
       extname: '.liquid'
     });
-    content += await liquid.renderFile('dash-metrics', {
-      historyJson: JSON.stringify(stats.history),
-      startTime: stats.startTime
+    // the background tasks' table (the bare template has no fhir.css)
+    let content = '<style>table.grid{margin-bottom:10px;border:1px solid black;margin-right:auto}table.grid th,table.grid td{border:1px solid silver;padding:3px 7px 2px;font-size:12px;line-height:1.4em;font-family:verdana;vertical-align:top}table.grid th{font-weight:bold}table.grid td{font-weight:normal}</style>';
+    // the page's title is its h1 (root-bare-template.html), so its sections are h2
+    // the release and npm audit on one line above "More" (the advice and the findings, when there
+    // are any, after it); the data folder under "More"
+    content += await statusContent(liquid, 'dash-metrics', 2, {
+      notes: npmAudit ? [npmAudit.renderLine()] : [],
+      details: npmAudit ? npmAudit.renderFindings() : '',
+      more: `<p>Data: ${escape(folders.dataDir())}</p>`
     });
-    content += stats.taskDetails();
-    if (npmAudit) {
-      content += npmAudit.renderDashboard();
-    }
-    content += `<p>Data: ${folders.dataDir()}</p>`;
 
     content = '<div class="row mb-4"><div class="col-12">' + content + '</div></div>';
 
@@ -704,23 +892,17 @@ app.get('/dashboard', async (req, res) => {
   }
 });
 
-function formatBytes(bytes) {
-  if (bytes < 1024) {
-    return `${bytes} bytes`;
-  }
-  if (bytes < 1024 * 1024) {
-    return `${(bytes / 1024).toFixed(1)} KB`;
-  }
-  if (bytes < 1024 * 1024 * 1024) {
-    return `${(bytes / 1024 / 1024).toFixed(1)} MB`;
-  }
-  return `${(bytes / 1024 / 1024 / 1024).toFixed(2)} GB`;
-}
-
 /**
  * "3d 4h 5m 6s" for this run, followed by the proportion of time the server has
  * been up over all its recorded runs, when the statistics database is keeping that.
  */
+// [the uptime, "97.3% overall" or null], for the Uptime tile
+function uptimeParts(uptimeMs) {
+  const text = uptimeText(uptimeMs);
+  const m = / \((.*)\)$/.exec(text);
+  return m ? [text.substring(0, m.index), m[1]] : [text, null];
+}
+
 function uptimeText(uptimeMs) {
   const uptimeSeconds = Math.floor(uptimeMs / 1000);
   const uptimeDays = Math.floor(uptimeSeconds / 86400);
@@ -741,15 +923,6 @@ function uptimeText(uptimeMs) {
   return uptimeStr;
 }
 
-function pctColor(pct) {
-  // Gradient from green (#deffe0) at 0% to red (#ffd3d1) at 100%
-  const t = Math.max(0, Math.min(100, pct)) / 100;
-  const r = Math.round(222 + 33 * t);   // 222 -> 255
-  const g = Math.round(255 - 44 * t);   // 255 -> 211
-  const b = Math.round(224 - 15 * t);   // 224 -> 209
-  return `rgb(${r}, ${g}, ${b})`;
-}
-
 // Health check endpoint
 app.get('/health', async (req, res) => {
   const healthStatus = {
@@ -781,45 +954,29 @@ app.get('/health', async (req, res) => {
  * Get log directory statistics: file count, total size, and disk space info
  * @returns {string} HTML table row(s) with log stats
  */
-function getLogStats() {
+// The logs and the disk they are on, for the storage tiles: files and bytes null if the logs
+// can't be read, disk null if the disk can't
+function logFigures() {
   const logDir = folders.logsDir();
-
+  let disk = null;
+  try {
+    // statfs available in Node 18.15+
+    const fstats = fs.statfsSync(logDir);
+    disk = { free: fstats.bavail * fstats.bsize, total: fstats.blocks * fstats.bsize };
+  } catch {
+    // unavailable
+  }
   try {
     const files = readdirSync(logDir).filter(f => f.endsWith('.log'));
-    let totalSize = 0;
+    let bytes = 0;
     for (const file of files) {
-      totalSize += statSync(path.join(logDir, file)).size;
-    }
-
-    const sizeMB = (totalSize / 1024 / 1024).toFixed(2);
-
-    let diskInfo = '';
-    try {
-      // statfs available in Node 18.15+
-      const fstats = fs.statfsSync(logDir);
-      const blockSize = fstats.bsize;
-      const freeSpace = fstats.bavail * blockSize;
-      const totalSpace = fstats.blocks * blockSize;
-      const freeGB = (freeSpace / 1024 / 1024 / 1024).toFixed(2);
-      const totalGB = (totalSpace / 1024 / 1024 / 1024).toFixed(2);
-      diskInfo = `<td><strong>Disk Space:</strong> ${freeGB} GB of ${totalGB} GB</td>`;
-    } catch {
-      diskInfo = '<td><strong>Disk Space:</strong> unavailable</td>';
+      bytes += statSync(path.join(logDir, file)).size;
     }
-
-    // Log rotation limit from logger config
+    // how many files the logger keeps (its rotation)
     const loggerOpts = Logger.getInstance().options;
-    const maxFiles = loggerOpts.file.maxFiles;
-    const maxSize = loggerOpts.file.maxSize;
-    const limitInfo = `${maxFiles} files × ${maxSize} each`;
-
-    return '<tr>'
-        + `<td><strong>Existing Logs:</strong> ${files.length} (${sizeMB} MB)</td>`
-        + `<td><strong>Retention Policy:</strong> ${limitInfo}</td>`
-        + diskInfo
-        + '</tr>';
-  } catch (e) {
-    return `<tr><td colspan="3"><strong>Logs:</strong> unable to read (${e.message})</td></tr>`;
+    return { files: files.length, bytes, kept: loggerOpts.file.maxFiles, disk };
+  } catch {
+    return { files: null, bytes: null, kept: null, disk };
   }
 }
 
@@ -835,11 +992,33 @@ async function startServer() {
       throw error;
     });
 
+    // With a role: loading the library leaves 0.6-0.7 GB free but resident in the C allocator;
+    // give it back before taking requests, since N+1 processes share one limit (cluster.md)
+    if (role && memoryGuard._state.trimAllocator) {
+      const before = process.memoryUsage.rss();
+      if (trimNativeMemory() !== null) {
+        serverLog.info(`Memory: malloc_trim after loading - RSS ${memoryGuard.gb(before)} GB -> ${memoryGuard.gb(process.memoryUsage.rss())} GB`);
+      }
+    }
+
     // Start server
-    app.listen(PORT, () => {
+    const listening = () => {
       stats.markStarted(packageJson.version);
-      serverLog.info(`=== Server running on http://localhost:${PORT} ===`);
-    });
+      serverLog.info(HOST ? `=== Server running on http://${HOST.includes(':') ? `[${HOST}]` : HOST}:${PORT} (server.cluster.bindAddress) ===`
+        : `=== Server running on http://localhost:${PORT} ===`);
+      startupComplete();
+      if (role && process.send) {
+        // to the launcher (cluster.js): ready, from this process itself
+        try {
+          process.send({ type: require('./library/role').READY, pid: process.pid });
+        } catch {
+          // the launcher has gone; the disconnect handler below shuts this process down
+        }
+      }
+    };
+    const server = HOST ? app.listen(PORT, HOST, listening) : app.listen(PORT, listening);
+    // idle connections stay open longer than nginx keeps them (see library/keep-alive.js)
+    applyKeepAliveTimeouts(server);
     npmAudit.start();
     dailyReport = new DailyReport(config.dailyReport, {
       log: Logger.getInstance().child({ module: 'daily-report' }),
@@ -848,7 +1027,9 @@ async function startServer() {
       mailer: new Mailer(config.email, Logger.getInstance().child({ module: 'email' })),
       npmAudit: npmAudit,
       version: packageJson.version,
-      serverId: config.hostName
+      serverId: config.hostName,
+      // under the launcher: the report has a paragraph on the cluster
+      cluster: Boolean(role && role.role === 'master' && process.send)
     });
     dailyReport.start();
     if (modules.packages && config.modules?.packages?.enabled) {
@@ -884,6 +1065,10 @@ async function shutdown() {
   }
   shuttingDown = true;
   serverLog.info('\nShutting down server...');
+  if (disconnectCounts) {
+    // a worker's last request counts, while the launcher still listens
+    await disconnectCounts.flush();
+  }
   if (npmAudit) {
     npmAudit.stop();
   }
@@ -904,11 +1089,26 @@ async function shutdown() {
     }
   }
   stats.finishStats();
+  disconnectMetrics();
+  disconnectStatus();
+  if (disconnectCounts) {
+    disconnectCounts();
+  }
+  disconnectClusterEvents();
   serverLog.info('Server shutdown complete');
   process.exit(0);
 }
 process.on('SIGINT', () => void shutdown());
 process.on('SIGTERM', () => void shutdown());
+// Started by the launcher (cluster.js, an IPC channel): when the launcher goes, even
+// SIGKILLed, so does this process, rather than hold its port against the next launcher's
+// workers. With no one left to SIGKILL a shutdown that hangs, exit after 30 s regardless
+if (role && process.connected) {
+  process.once('disconnect', () => {
+    setTimeout(() => process.exit(1), 30000).unref();
+    void shutdown();
+  });
+}
 
 async function serveFhirsmithHome(req, res) {
   // Check if client wants HTML response
@@ -993,4 +1193,4 @@ async function serveFhirsmithHome(req, res) {
 startServer().catch((err) => {
   serverLog.error('Fatal error during startup:', err);
   process.exit(1);
-});
+});