fhirsmith 0.17.0 → 0.18.1

package/registry/registry-template.html

@@ -62,7 +62,7 @@
   <a href="/tx-reg/resolve" style="color: gold">Resolve</a>  &nbsp;|&nbsp;
   <a href="/tx-reg/software" style="color: gold">Software</a>  &nbsp;|&nbsp;
   <a href="/tx-reg/log" style="color: gold">Crawler Log</a>  &nbsp;|&nbsp;
-  <a href="/tx-reg/openapi" style="color: gold">API</a>
+  <a href="/tx-reg/openapi" style="color: gold">API</a>[%account%]
 		</div>  <!-- /container -->
 		</div>  <!-- /container -->
 </div>

package/registry/registry.js

@@ -11,6 +11,8 @@ const folders = require('../library/folder-setup');
 const escape = require('escape-html');
 const registryOpenApi = require('./openapi');
 const { FhirsmithReleases, isFhirsmith, describeAge } = require('./fhirsmith-releases');
+const { formatUptime } = require('./uptime');
+const { RegistryAuth } = require('./auth');
 
 // The query parameters of the public API. These are the contract published in openapi.yaml,
 // and tests/registry/openapi.test.js checks that the two agree - so change both together.
@@ -25,6 +27,35 @@ function safeHref(url) {
   return typeof url === 'string' && /^https?:\/\//i.test(url.trim()) ? escape(url) : '#';
 }
 
+/**
+ * The base URL that a server's endpoints share: http://tx.fhir.org for http://tx.fhir.org/r4
+ * and http://tx.fhir.org/r5. One endpoint is its own base
+ */
+function commonBaseUrl(urls) {
+  const list = urls.filter(u => u);
+  if (list.length === 0) {
+    return '';
+  }
+  if (list.length === 1) {
+    return list[0];
+  }
+  let prefix = list[0];
+  for (const url of list.slice(1)) {
+    let i = 0;
+    while (i < prefix.length && i < url.length && prefix[i] === url[i]) {
+      i++;
+    }
+    prefix = prefix.substring(0, i);
+  }
+  // back up to a path boundary, unless the prefix already is one
+  if (!list.every(u => u === prefix || u.startsWith(prefix + '/'))) {
+    prefix = prefix.substring(0, prefix.lastIndexOf('/'));
+  }
+  prefix = prefix.replace(/\/+$/, '');
+  // less than scheme + host is not a base URL at all
+  return /^https?:\/\/[^/]+/.test(prefix) ? prefix : list[0];
+}
+
 class RegistryModule {
   constructor(stats) {
     this.router = express.Router();
@@ -41,6 +72,8 @@ class RegistryModule {
     this.currentData = null;
     this.dataLock = false;
     this.stats = stats;
+    // open until initialize() reads modules.registry.security
+    this.auth = new RegistryAuth({}, { logger: this.logger, renderPage: () => {} });
   }
 
   /**
@@ -57,7 +90,9 @@ class RegistryModule {
         timeout: config.timeout || 30000,
         userAgent: config.userAgent || 'FHIRRegistryServer/1.0',
         apiKeys: config.apiKeys || {},
-        maxResponseMB: config.maxResponseMB
+        maxResponseMB: config.maxResponseMB,
+        uptimeWindowDays: config.uptimeWindowDays,
+        uptimeRetentionDays: config.uptimeRetentionDays
       };
 
       this.crawler = new RegistryCrawler(crawlerConfig, this.stats);
@@ -76,6 +111,17 @@ class RegistryModule {
       // Load saved data if available
       await this.loadSavedData();
 
+      // Securing the API (see auth.js). With no modules.registry.security, the API is open
+      this.auth = new RegistryAuth(config.security, {
+        logger: this.logger,
+        renderPage: (req, res, title, content, status) => {
+          res.status(status || 200);
+          res.setHeader('Content-Type', 'text/html');
+          res.send(this.renderPage('registry', title, content, this.api.getStatistics(), req));
+        }
+      });
+      await this.auth.initialize();
+
       // Set up routes
       this.setupRoutes();
 
@@ -113,6 +159,13 @@ class RegistryModule {
     } catch (error) {
       this.logger.info('No saved registry data found, will fetch fresh data');
     }
+    try {
+      const fs = require('fs').promises;
+      const uptimePath = folders.ensureFilePath('registry', 'registry-uptime.json');
+      this.crawler.loadUptime(JSON.parse(await fs.readFile(uptimePath, 'utf8')));
+    } catch (error) {
+      this.logger.info('No saved uptime history found');
+    }
   }
 
   /**
@@ -125,6 +178,8 @@ class RegistryModule {
 
       const data = this.crawler.saveData();
       await fs.writeFile(dataPath, JSON.stringify(data, null, 2));
+      const uptimePath = folders.ensureFilePath('registry', 'registry-uptime.json');
+      await fs.writeFile(uptimePath, JSON.stringify(this.crawler.saveUptime()));
       this.logger.debug('Saved registry data to disk');
     } catch (error) {
       this.logger.error('Failed to save registry data:', error);
@@ -255,11 +310,23 @@ class RegistryModule {
       next();
     });
 
+    // The JSON API is what security protects; the HTML pages stay open
+    const guard = this.auth.apiGuard();
+    const apiGuard = (req, res, next) => {
+      const acceptsHtml = req.headers.accept && req.headers.accept.includes('text/html');
+      return acceptsHtml ? next() : guard(req, res, next);
+    };
+    // failed authentications are limited (successful ones aren't counted)
+    const apiLimiter = this.auth.apiLimiter();
+    // the session, where there are users, so the pages know who is logged in (it makes none)
+    const session = this.auth.optionalSession();
+    this.auth.setupRoutes(this.router);
+
     // Main registry page
-    this.router.get('/', this.handleMainPage.bind(this));
-    this.router.get('/resolve', this.handleResolveEndpoint.bind(this));
-    this.router.get('/log', this.handleLogEndpoint.bind(this));
-    this.router.get('/software', this.handleSoftwarePage.bind(this));
+    this.router.get('/', apiLimiter, apiGuard, session, this.handleMainPage.bind(this));
+    this.router.get('/resolve', apiLimiter, apiGuard, session, this.handleResolveEndpoint.bind(this));
+    this.router.get('/log', session, this.handleLogEndpoint.bind(this));
+    this.router.get('/software', session, this.handleSoftwarePage.bind(this));
 
     // OpenAPI description of this API: /openapi.json, /openapi.yaml, and /openapi (an HTML
     // reference for browsers, the JSON otherwise)
@@ -280,8 +347,8 @@ class RegistryModule {
         if (!htmlServer.hasTemplate('registry')) {
           htmlServer.loadTemplate('registry', path.join(__dirname, 'registry-template.html'));
         }
-        const html = htmlServer.renderPage('registry', 'Terminology Server Registry API',
-          registryOpenApi.renderHtml(), this.api ? this.api.getStatistics() : {});
+        const html = this.renderPage('registry', 'Terminology Server Registry API',
+          registryOpenApi.renderHtml(), this.api ? this.api.getStatistics() : {}, req);
         res.setHeader('Content-Type', 'text/html');
         res.send(html);
       } catch (error) {
@@ -291,6 +358,18 @@ class RegistryModule {
     });
   }
 
+  /**
+   * The registry's pages: html-server's template, plus the login/account link in the
+   * navbar (pages rendered without a request show the login link)
+   */
+  renderPage(template, title, content, stats, req) {
+    if (template === 'registry' && !htmlServer.hasTemplate('registry')) {
+      htmlServer.loadTemplate('registry', path.join(__dirname, 'registry-template.html'));
+    }
+    const html = htmlServer.renderPage(template, title, content, stats);
+    return html.replace('[%account%]', () => this.auth ? this.auth.navLinks(req || {}) : '');
+  }
+
   /**
    * Render HTML page for code system or value set query
    * Combines functionality from sendHtmlCS and sendHtmlVS
@@ -341,14 +420,15 @@ class RegistryModule {
         htmlServer.loadTemplate('registry', templatePath);
       }
 
-      return htmlServer.renderPage(
+      return this.renderPage(
         'registry',
         'FHIR Terminology Server Registry',
         this.buildHtmlContent(),
         {
           ...this.api.getStatistics(),
           templateVars: templateVars
-        }
+        },
+        req
       );
     } catch (error) {
       this.logger.error('Error rendering page:', error);
@@ -458,11 +538,11 @@ class RegistryModule {
         stats.crawlInProgress = this.crawlInProgress;
         stats.lastCrawl = this.lastCrawlTime;
 
-        const html = htmlServer.renderPage(
+        const html = this.renderPage(
           'registry',
           'FHIR Terminology Server Registry',
           content,
-          stats
+          stats, req
         );
 
         res.setHeader('Content-Type', 'text/html');
@@ -491,7 +571,7 @@ class RegistryModule {
       const content = this.buildSoftwareContent();
       const stats = this.api.getStatistics();
       stats.processingTime = Date.now() - startTime;
-      const html = htmlServer.renderPage('registry', 'Terminology Server Software', content, stats);
+      const html = this.renderPage('registry', 'Terminology Server Software', content, stats, req);
       res.setHeader('Content-Type', 'text/html');
       res.send(html);
     } catch (error) {
@@ -509,24 +589,8 @@ class RegistryModule {
         '<p>The initial crawl is in progress. Please refresh in a moment.</p></div>';
     }
 
-    const rows = [];
-    for (const registry of data.registries) {
-      for (const server of registry.servers || []) {
-        for (const version of server.versions || []) {
-          rows.push({
-            name: server.name || server.code || '',
-            url: version.address || '',
-            fhirVersion: version.version || '',
-            software: version.software && version.software !== 'unknown' ? version.software : '',
-            softwareVersion: version.softwareVersion || '',
-            softwareReleaseDate: version.softwareReleaseDate || '',
-            error: version.error,
-            lastSuccess: version.lastSuccess
-          });
-        }
-      }
-    }
-    rows.sort((a, b) => a.name.localeCompare(b.name) || a.url.localeCompare(b.url));
+    const rows = this.softwareRows(data);
+    const uptime = this.crawler ? this.crawler.uptime : null;
 
     let html = '<h3>Terminology Server Software</h3>';
     const latest = this.releases ? this.releases.latest() : null;
@@ -539,29 +603,102 @@ class RegistryModule {
 
     html += '<table class="grid">';
     html += '<thead><tr><th>Server</th><th>URL</th><th>FHIR</th><th>Software</th><th>Version</th>' +
-      '<th>Released</th><th>Age</th></tr></thead><tbody>';
+      '<th>Released</th><th>Age</th><th>Uptime</th></tr></thead><tbody>';
     for (const row of rows) {
       html += '<tr>';
       html += `<td>${escape(row.name)}</td>`;
       html += `<td><a href="${safeHref(row.url)}" target="_blank" rel="noopener">${escape(row.url)}</a>`;
-      if (row.error) {
-        html += ` <span class="text-danger" title="${escape(row.error)}">(unreachable)</span>`;
-      }
+      html += this._renderUnreachable(row);
       html += '</td>';
-      html += `<td>${escape(row.fhirVersion)}</td>`;
+      html += `<td>${escape(row.fhirVersions.join(', '))}</td>`;
       html += `<td>${row.software ? escape(row.software.replace('Reference Server', 'HealthIntersections')) : '<i>unknown</i>'}</td>`;
       html += `<td>${row.softwareVersion ? escape(row.softwareVersion) : '<i>unknown</i>'}</td>`;
       html += this._renderReleaseCells(row, now);
+      html += this._renderUptimeCell(uptime ? uptime.uptime(row.endpoints.map(e => e.url), now) : null);
       html += '</tr>';
     }
     html += '</tbody></table>';
 
+    html += '<p class="text-muted"><small>';
+    if (uptime) {
+      const days = Math.round(uptime.windowMs / (24 * 60 * 60 * 1000));
+      html += `Uptime is the share of this registry's scans over the last ${days} days that the server answered ` +
+        '(every scan probes each of its endpoints; a server with one of two endpoints down is half down for that scan). ' +
+        'Scans the registry itself missed are not counted. ';
+    }
+    html += 'Endpoints of one server that run the same software are shown as one row. ';
     if (this.releases && this.releases.source !== 'none') {
-      html += `<p class="text-muted"><small>FHIRsmith release dates from ${escape(this.releases.source)}.</small></p>`;
+      html += `FHIRsmith release dates from ${escape(this.releases.source)}.`;
     }
+    html += '</small></p>';
     return html;
   }
 
+  /**
+   * The rows of the software page: one per server, or per software a server runs, since
+   * a server's endpoints (its FHIR versions) are nearly always the same software
+   * @returns {Array<{name, url, fhirVersions, software, softwareVersion, softwareReleaseDate, endpoints}>}
+   */
+  softwareRows(data) {
+    const rows = [];
+    for (const registry of data.registries) {
+      for (const server of registry.servers || []) {
+        const groups = new Map(); // software|version -> row
+        for (const version of server.versions || []) {
+          const software = version.software && version.software !== 'unknown' ? version.software : '';
+          const softwareVersion = version.softwareVersion || '';
+          const key = `${software}|${softwareVersion}`;
+          let row = groups.get(key);
+          if (!row) {
+            row = {
+              name: server.name || server.code || '',
+              url: '',
+              fhirVersions: [],
+              software,
+              softwareVersion,
+              softwareReleaseDate: version.softwareReleaseDate || '',
+              endpoints: []
+            };
+            groups.set(key, row);
+            rows.push(row);
+          }
+          row.fhirVersions.push(version.version || '');
+          row.endpoints.push({ url: version.address || '', fhirVersion: version.version || '', error: version.error || '' });
+          if (!row.softwareReleaseDate && version.softwareReleaseDate) {
+            row.softwareReleaseDate = version.softwareReleaseDate;
+          }
+        }
+        for (const row of groups.values()) {
+          row.url = commonBaseUrl(row.endpoints.map(e => e.url));
+          row.fhirVersions.sort();
+        }
+      }
+    }
+    rows.sort((a, b) => a.name.localeCompare(b.name) || a.url.localeCompare(b.url));
+    return rows;
+  }
+
+  _renderUnreachable(row) {
+    const down = row.endpoints.filter(e => e.error);
+    if (down.length === 0) {
+      return '';
+    }
+    const title = down.map(e => `${e.fhirVersion}: ${e.error}`).join('\n');
+    const what = down.length === row.endpoints.length ? 'unreachable' :
+      `${down.map(e => e.fhirVersion).join(', ')} unreachable`;
+    return ` <span class="text-danger" title="${escape(title)}">(${escape(what)})</span>`;
+  }
+
+  _renderUptimeCell(u) {
+    if (!u) {
+      return '<td></td>';
+    }
+    const cls = u.failures === 0 ? 'text-success' : u.percent >= 99 ? '' : u.percent >= 95 ? 'text-warning' : 'text-danger';
+    const since = new Date(u.since).toISOString().substring(0, 10);
+    const title = `${u.probes - u.failures} of ${u.probes} probes answered since ${since}`;
+    return `<td><span class="${cls}" title="${escape(title)}">${formatUptime(u)}</span></td>`;
+  }
+
   _renderReleaseCells(row, now) {
     if (!isFhirsmith(row.software) || !this.releases) {
       return this._renderReportedReleaseCells(row, now);
@@ -1140,6 +1277,49 @@ class RegistryModule {
     };
   }
 
+  /**
+   * The paragraph for the server's daily report: how many client requests the
+   * API answered, how many servers are registered, and which of them (if any)
+   * answered none of the registry's scans in the period.
+   */
+  dailyReport({since, now}) {
+    const {plural} = require('../library/daily-report');
+    const counts = this.stats.countsSince(since, now);
+    const api = counts.operations.resolve || 0;
+    const sentences = [`The API answered ${plural(api, 'client request')} (${plural(counts.total, 'request')} in all, including the web pages).`];
+    const data = this.api ? this.api.getData() : null;
+    if (!data || !data.registries) {
+      sentences.push('The registry data is not available (no crawl has completed).');
+      return sentences.join(' ');
+    }
+    let servers = 0;
+    let endpoints = 0;
+    const down = [];
+    const uptime = this.crawler ? this.crawler.uptime : null;
+    for (const registry of data.registries) {
+      for (const server of registry.servers || []) {
+        servers++;
+        const addresses = (server.versions || []).map(v => v.address || v.url).filter(Boolean);
+        endpoints += addresses.length;
+        if (uptime && addresses.length > 0) {
+          const u = uptime.uptime(addresses, now, since);
+          if (u && u.failures === u.probes) {
+            down.push(server.name || server.code || addresses[0]);
+          }
+        }
+      }
+    }
+    sentences.push(`${plural(servers, 'server')} ${servers === 1 ? 'is' : 'are'} registered, with ${plural(endpoints, 'endpoint')}.`);
+    if (!uptime) {
+      sentences.push('No scan history is kept, so availability is unknown.');
+    } else if (down.length === 0) {
+      sentences.push('Every server answered at least one scan during the period.');
+    } else {
+      sentences.push(`${down.length === 1 ? 'This server' : 'These servers'} did not answer any scan during the period: ${down.join(', ')}.`);
+    }
+    return sentences.join(' ');
+  }
+
   /**
    * Shutdown the module
    */
@@ -1155,6 +1335,9 @@ class RegistryModule {
     if (this.crawler) {
       this.crawler.shutdown();
     }
+    if (this.auth) {
+      this.auth.shutdown();
+    }
     // Save current data
     if (this.crawler && this.currentData) {
       await this.saveData();
@@ -1239,11 +1422,11 @@ class RegistryModule {
             const stats = this.api.getStatistics();
             stats.processingTime = Date.now() - startTime;
 
-            const html = htmlServer.renderPage(
+            const html = this.renderPage(
               'registry',
               'FHIR Terminology Server Resolver',
               content,
-              stats
+              stats, req
             );
 
             res.setHeader('Content-Type', 'text/html');
@@ -1294,18 +1477,20 @@ class RegistryModule {
             const stats = this.api.getStatistics();
             stats.processingTime = Date.now() - startTime;
 
-            const html = htmlServer.renderPage(
+            const html = this.renderPage(
               'registry',
               'FHIR Terminology Server Resolution Results',
               content,
-              stats
+              stats, req
             );
 
             res.setHeader('Content-Type', 'text/html');
             return res.send(html);
           } catch (error) {
             this.logger.error('Error rendering resolve result page:', error);
-            // Fall back to JSON if template rendering fails
+            // not the JSON instead: this request was let through as a page, without a token
+            res.status(500).setHeader('Content-Type', 'text/html');
+            return res.send('<!DOCTYPE html><html><body><p>The resolution page could not be shown (see the server log).</p></body></html>');
           }
         }
         res.json(result);
@@ -1566,11 +1751,11 @@ class RegistryModule {
             const stats = this.api.getStatistics();
             stats.processingTime = Date.now() - startTime;
 
-            const html = htmlServer.renderPage(
+            const html = this.renderPage(
               'registry',
               'FHIR Terminology Server Registry - Logs',
               content,
-              stats
+              stats, req
             );
 
             res.setHeader('Content-Type', 'text/html');
@@ -1649,4 +1834,5 @@ class RegistryModule {
 RegistryModule.DISCOVERY_PARAMS = DISCOVERY_PARAMS;
 RegistryModule.RESOLVE_PARAMS = RESOLVE_PARAMS;
 
-module.exports = RegistryModule;
+module.exports = RegistryModule;
+module.exports.commonBaseUrl = commonBaseUrl;

package/registry/uptime.js

@@ -0,0 +1,154 @@
+// registry/uptime.js
+// Uptime history for the registered servers: one record per endpoint per scan, saying
+// whether the endpoint answered. The registry's own scan is the probe - nothing else is
+// pinged - so a scan that didn't happen (the registry was down, or the master list could
+// not be read) is not held against any server.
+
+const DAY = 24 * 60 * 60 * 1000;
+
+class UptimeHistory {
+  /**
+   * @param {object} [options]
+   * @param {number} [options.retentionDays] how long records are kept (default 90)
+   * @param {number} [options.windowDays] the period uptime is reported over (default 30)
+   */
+  constructor(options = {}) {
+    this.retentionMs = (options.retentionDays || 90) * DAY;
+    this.windowMs = (options.windowDays || 30) * DAY;
+    this.records = new Map(); // address -> [[timeMs, ok(1|0)], ...] in time order
+  }
+
+  /**
+   * Record the outcome of one probe of an endpoint
+   * @param {string} address the endpoint URL
+   * @param {boolean} ok whether the scan of it succeeded
+   * @param {number} [time] when the scan ran (ms)
+   */
+  record(address, ok, time = Date.now()) {
+    if (!address) {
+      return;
+    }
+    let list = this.records.get(address);
+    if (!list) {
+      list = [];
+      this.records.set(address, list);
+    }
+    list.push([time, ok ? 1 : 0]);
+    this.prune(address, time);
+  }
+
+  prune(address, now = Date.now()) {
+    const list = this.records.get(address);
+    if (!list) {
+      return;
+    }
+    const cutoff = now - this.retentionMs;
+    let drop = 0;
+    while (drop < list.length && list[drop][0] < cutoff) {
+      drop++;
+    }
+    if (drop > 0) {
+      list.splice(0, drop);
+    }
+  }
+
+  /**
+   * Uptime over the reporting window for a set of endpoints, as the fraction of their
+   * probes that succeeded: a server with two endpoints of which one is down for a scan
+   * is half down for that scan.
+   * @param {string|string[]} addresses one endpoint, or the endpoints of one server
+   * @param {number} [now]
+   * @param {number} [from] the start of the period; the reporting window by default
+   * @returns {{percent: number, probes: number, failures: number, since: number}|null}
+   *   null when there have been no probes in the window
+   */
+  uptime(addresses, now = Date.now(), from = now - this.windowMs) {
+    const list = Array.isArray(addresses) ? addresses : [addresses];
+    let probes = 0;
+    let failures = 0;
+    let since = Infinity;
+    for (const address of list) {
+      for (const [time, ok] of this.records.get(address) || []) {
+        if (time < from || time > now) {
+          continue;
+        }
+        probes++;
+        if (!ok) {
+          failures++;
+        }
+        if (time < since) {
+          since = time;
+        }
+      }
+    }
+    if (probes === 0) {
+      return null;
+    }
+    return { percent: 100 * (probes - failures) / probes, probes, failures, since };
+  }
+
+  /**
+   * Forget what is past retention. Only age decides: an endpoint missing from one crawl (a
+   * sub-registry that failed to fetch, a crawl cut short at shutdown) keeps its history,
+   * and one that has really gone ages out with its last probes
+   * @param {Iterable<string>} [addresses] the endpoints of the last crawl (no longer used:
+   *   a crawl can miss endpoints that still exist)
+   */
+  retain(addresses, now = Date.now()) {
+    for (const address of [...this.records.keys()]) {
+      this.prune(address, now);
+      if (this.records.get(address).length === 0) {
+        this.records.delete(address);
+      }
+    }
+  }
+
+  toJSON() {
+    const endpoints = {};
+    for (const [address, list] of this.records) {
+      if (list.length > 0) {
+        endpoints[address] = list.map(([t, ok]) => [t, ok]);
+      }
+    }
+    return { version: '1', endpoints };
+  }
+
+  static fromJSON(json, options) {
+    const history = new UptimeHistory(options);
+    if (!json || json.version !== '1' || typeof json.endpoints !== 'object') {
+      return history;
+    }
+    for (const [address, list] of Object.entries(json.endpoints || {})) {
+      if (!Array.isArray(list)) {
+        continue;
+      }
+      const clean = list
+        .filter(r => Array.isArray(r) && Number.isFinite(r[0]))
+        .map(r => [r[0], r[1] ? 1 : 0])
+        .sort((a, b) => a[0] - b[0]);
+      if (clean.length > 0) {
+        history.records.set(address, clean);
+      }
+    }
+    return history;
+  }
+}
+
+/**
+ * Format an uptime for display: 100%, 99.9%, 97%, ... - the precision that is honest
+ * for the number of probes behind it
+ */
+function formatUptime(u) {
+  if (!u) {
+    return '';
+  }
+  if (u.failures === 0) {
+    return '100%';
+  }
+  if (u.percent >= 99) {
+    return `${(Math.floor(u.percent * 10) / 10).toFixed(1)}%`;
+  }
+  return `${Math.floor(u.percent)}%`;
+}
+
+module.exports = { UptimeHistory, formatUptime };