fhirsmith 0.13.4 → 0.14.1

package/server.js

@@ -16,6 +16,7 @@ const folders = require('./library/folder-setup');  // <-- ADD: load early
 const { statSync, readdirSync } = require('fs');
 const escape = require('escape-html');
 const { resolveWithin } = require('./library/path-safety');
+const { readCgroupMemoryLimit } = require('./library/cgroup-memory');
 
 // Load configuration BEFORE logger
 let config;
@@ -39,6 +40,12 @@ serverLog.info(`========================================`);
 serverLog.info(`FHIRsmith v${packageJson.version} starting (PID ${process.pid})`);
 serverLog.info(`Node.js ${process.version} on ${os.type()} ${os.release()} (${os.arch()})`);
 serverLog.info(`Memory: ${freeMemGB} GB free / ${totalMemGB} GB total`);
+{
+  const cg = readCgroupMemoryLimit();
+  serverLog.info(cg.limit > 0
+    ? `Memory limit: ${(cg.limit / 1024 / 1024 / 1024).toFixed(1)} GB (from ${cg.source})`
+    : 'Memory limit: none found in cgroup - RSS guard disabled');
+}
 serverLog.info(`Data directory: ${folders.dataDir()}`);
 serverLog.info(`========================================`);
 
@@ -57,6 +64,7 @@ const PublisherModule = require('./publisher/publisher.js');
 const TokenModule = require('./token/token.js');
 const NpmProjectorModule = require('./npmprojector/npmprojector.js');
 const TXModule = require('./tx/tx.js');
+const TestingModule = require('./testing/testing.js');
 
 const htmlServer = require('./library/html-server');
 const ServerStats = require("./stats");
@@ -71,7 +79,12 @@ const app = express();
 
 // Behind nginx (or any reverse proxy): honor X-Forwarded-* so req.protocol and
 // req.hostname reflect what the client actually requested, not the loopback hop.
-app.set('trust proxy', config.server.trustProxy ?? true);
+// Only proxies on the same host or a private network are trusted by default. `true` would
+// trust every X-Forwarded-For hop, so anyone who could reach the port directly could name
+// their own IP address and walk around every rate limit (express-rate-limit refuses to be
+// quiet about it: ERR_ERL_PERMISSIVE_TRUST_PROXY). See nginx.md.
+const DEFAULT_TRUST_PROXY = 'loopback, linklocal, uniquelocal';
+app.set('trust proxy', config.server.trustProxy ?? DEFAULT_TRUST_PROXY);
 
 const PORT = process.env.PORT || config.server.port || 3000;
 
@@ -219,6 +232,18 @@ async function initializeModules() {
       throw error;
     }
   }
+  if (config.modules?.testing?.enabled) {
+    try {
+      serverLog.info('Initializing module: testing...');
+      modules.testing = new TestingModule(stats.forModule('testing'));
+      await modules.testing.initialize(config.modules.testing);
+      app.use('/testing', modules.testing.router);
+    } catch (error) {
+      serverLog.error('Failed to initialize testing module:', error);
+      throw error;
+    }
+  }
+
   // Initialize TX module
   // Note: TX module registers its own endpoints directly on the app
   // because it supports multiple endpoints at different paths
@@ -346,6 +371,13 @@ async function buildRootPageContent() {
     content += '</li>';
   }
 
+  if (config.modules?.testing?.enabled) {
+    content += '<li class="list-group-item">';
+    content += '<a href="/testing" class="text-decoration-none">Test Reports</a>: ';
+    content += 'TestReports submitted by TxTester and other test tools';
+    content += '</li>';
+  }
+
   if (config.modules.folder && config.modules.folder.enabled) {
     content += '<li class="list-group-item">';
     content += '<strong>Cache Folder</strong>: ';
@@ -405,13 +437,7 @@ async function buildRootPageContent() {
 
   // Process RSS vs cgroup limit (or system total)
   const rssMB = (memUsage.rss / 1024 / 1024).toFixed(0);
-  let memLimit;
-  try {
-    const raw = fs.readFileSync('/sys/fs/cgroup/memory.max', 'utf8').trim();
-    memLimit = raw === 'max' ? os.totalmem() : parseInt(raw);
-  } catch {
-    memLimit = os.totalmem();
-  }
+  const memLimit = readCgroupMemoryLimit().limit || os.totalmem();
   const memLimitMB = (memLimit / 1024 / 1024).toFixed(0);
   const processPCT = (memUsage.rss * 100) / memLimit;
 
@@ -440,6 +466,15 @@ async function buildRootPageContent() {
   content += `<td><strong>Client Cache:</strong> ${stats.clientCaches()} caches / ${stats.clientConcepts()} concepts</td>`;
   content += `<td><strong>Max Client Cache:</strong> ${stats.maxClientCaches()} caches / ${stats.maxClientConcepts()} concepts</td>`;
   content += '</tr>';
+  const closure = stats.closureStats();
+  if (closure) {
+    content += '<tr>';
+    content += `<td><strong>Closure Tables:</strong> ${closure.tables} tables` +
+      (closure.retentionDays ? ` (unused for ${closure.retentionDays} days = deleted)` : '') + `</td>`;
+    content += `<td><strong>Closure Entries:</strong> ${closure.entries} entries / ${closure.concepts} concepts</td>`;
+    content += `<td><strong>Closure Database:</strong> ${closure.bytes === null ? 'in memory' : formatBytes(closure.bytes)}</td>`;
+    content += '</tr>';
+  }
   content += getLogStats();
   content += '</table>';
 
@@ -591,13 +626,7 @@ app.get('/dashboard', async (req, res) => {
     const v8PCT = (memUsage.heapUsed * 100) / heapStats.heap_size_limit;  // % of V8 heap limit used
 
     // Process RSS as % of cgroup memory limit (or system total as fallback)
-    let memLimit;
-    try {
-      const raw = fs.readFileSync('/sys/fs/cgroup/memory.max', 'utf8').trim();
-      memLimit = raw === 'max' ? os.totalmem() : parseInt(raw);
-    } catch {
-      memLimit = os.totalmem();
-    }
+    const memLimit = readCgroupMemoryLimit().limit || os.totalmem();
     const processPCT = (memUsage.rss * 100) / memLimit;
 
     const totalMemBytes = os.totalmem();
@@ -649,6 +678,19 @@ app.get('/dashboard', async (req, res) => {
   }
 });
 
+function formatBytes(bytes) {
+  if (bytes < 1024) {
+    return `${bytes} bytes`;
+  }
+  if (bytes < 1024 * 1024) {
+    return `${(bytes / 1024).toFixed(1)} KB`;
+  }
+  if (bytes < 1024 * 1024 * 1024) {
+    return `${(bytes / 1024 / 1024).toFixed(1)} MB`;
+  }
+  return `${(bytes / 1024 / 1024 / 1024).toFixed(2)} GB`;
+}
+
 function pctColor(pct) {
   // Gradient from green (#deffe0) at 0% to red (#ffd3d1) at 100%
   const t = Math.max(0, Math.min(100, pct)) / 100;

package/stats.js

@@ -124,20 +124,13 @@ class ServerStats {
       // event-loop block into noise; the max is what shows request-freezing
       // stalls (see the 2026-07-30 tx.fhir.org incident).
       const loopMax = this.eventLoopMonitor.max / 1e6;
-      // Two distinct caches: the expansion cache (count entries) and the client
-      // (resource) cache (count concepts held - a sense of how much it's carrying).
-      let expansionItems = 0;
-      let clientConcepts = 0;
-      for (let m of this.cachingModules) {
-        if (typeof m.expansionItemCount === 'function') {
-          expansionItems = expansionItems + m.expansionItemCount();
-        }
-        if (typeof m.clientConceptCount === 'function') {
-          clientConcepts = clientConcepts + m.clientConceptCount();
-        }
-      }
+      // The client (resource) cache, as concepts held - a sense of how much it's carrying.
+      // (The expansion cache isn't tracked over time: it fills to its limit over a few
+      // days and then sits there, so a history of it says nothing. The home page still
+      // shows its current size.)
+      const clientConcepts = this._sumCachingModules('clientConceptCount');
 
-      this.history.push({time: now, mem: currentMem - this.startMem, rpm: requestsPerMin, tat: requestsTat, block: loopDelay, blockMax: loopMax, expansion: expansionItems, clientConcepts: clientConcepts});
+      this.history.push({time: now, mem: currentMem - this.startMem, rpm: requestsPerMin, tat: requestsTat, block: loopDelay, blockMax: loopMax, clientConcepts: clientConcepts});
 
       this.eventLoopMonitor.reset();
       this.requestCountSnapshot = combinedCount;
@@ -328,6 +321,19 @@ class ServerStats {
   maxClientCaches() { return this._sumCachingModules('maxClientCacheCount'); }
   maxClientConcepts() { return this._sumCachingModules('maxClientConceptCount'); }
 
+  // $closure usage, from whichever module keeps closure tables; null if none does
+  closureStats() {
+    for (let m of this.cachingModules) {
+      if (typeof m.closureStats === 'function') {
+        const s = m.closureStats();
+        if (s) {
+          return s;
+        }
+      }
+    }
+    return null;
+  }
+
   getTaskColor(status) {
     switch (status) {
       case "started": return "LightGrey";

package/testing/readme.md

@@ -0,0 +1,144 @@
+# Testing module
+
+Receives FHIR TestReports - from TxTester, and from any other tool that produces them - and
+makes them available through a FHIR API and a set of web pages. It runs at `/testing` when
+`modules.testing.enabled` is true.
+
+## Submitting reports
+
+    POST /testing/TestReport
+    Content-Type: application/fhir+json
+    Authorization: Bearer <token>        (only if a token is configured)
+
+R4 and R5 TestReports are treated the same. A report is accepted if it is JSON, has
+`resourceType: TestReport`, and has `name`, `status`, `result`, `tester`, `issued` (a valid
+dateTime) and at least one `participant` with a `uri`. `score`, if present, must be a number.
+Nothing else is checked.
+
+The server gives the report a new id (any id sent is ignored), sets `meta.lastUpdated` to the
+time it was received, and drops `meta.versionId`: reports can't be changed once received, so
+there are no versions. Sending the same report twice stores it twice.
+
+Responses follow the spec: `201 Created` with a `Location` header, and the stored report,
+or nothing (`Prefer: return=minimal`), or an OperationOutcome (`Prefer: return=OperationOutcome`).
+Errors are OperationOutcomes: 400 (not JSON, or not an acceptable TestReport), 401 (token),
+413 (too big), 415 (not JSON), 429 (rate limit).
+
+The submitter's IP address is recorded, for tracing abuse. It is never returned by the API or
+shown on any page.
+
+FHIRsmith's own tx test run writes its results as `test-cases-report.json` in the
+FHIRsmith root; `node utilities/submit-test-report.js [-server url] [-token token]` sends it
+to a /testing server (by default https://testing.fhir.org/testing).
+
+## Reading and searching
+
+    GET /testing/TestReport/{id}
+    GET /testing/TestReport?{params}
+    GET /testing/metadata
+
+Both return HTML to a browser (an `Accept` that includes `text/html`); `_format=json` or
+`_format=html` overrides that.
+
+None of the search parameters are formally defined yet, and the CapabilityStatement lists them
+without definitions. What they do:
+
+| Parameter | Type | Searches | Notes |
+|---|---|---|---|
+| `_id` | token | id | |
+| `name` | string | name | starts with, case-insensitive; `:exact`, `:contains`, `:missing` |
+| `tester` | string | tester | as for name |
+| `status` | token | status | `:not`, `:missing` |
+| `result` | token | result | `:not`, `:missing` |
+| `testscript` | uri | testScript (R5 canonical, or R4 `reference`) | `:below` (starts with), `:missing` |
+| `participant` | uri | any participant.uri | `:below` |
+| `score` | number | score | prefixes; implicit precision for eq (`90` means 89.5 to 90.5); `:missing` |
+| `issued` | date | issued | prefixes; dates are ranges, so `issued=2026-09` is all of September |
+| `_lastUpdated` | date | when the report was received | prefixes |
+
+Commas OR values, repeated parameters AND. `_sort` takes any of these except `_id`, comma
+separated, `-` for descending; the default is newest received first. `_count` defaults to 50
+and is capped at 500; paging is by `_offset`, with first/previous/next/last links. `total` is
+always returned, and `_summary=count` returns only that.
+
+Unknown parameters are ignored, with a warning OperationOutcome in the Bundle, unless the
+request has `Prefer: handling=strict`, in which case they're a 400. Values that can't be
+understood (a bad date or number) are always a 400.
+
+## Web pages
+
+* `/testing` - the list of reports, with filters (these map onto the search parameters above),
+  sortable columns and paging
+* `/testing/summary` - a grid of test script x participant, with the latest report (by issued
+  date) in each cell and a link to all the runs. `?by=tester` makes the columns testers.
+* `/testing/TestReport/{id}` - the report: header, participants, setup / tests / teardown with
+  each action's result and message, anything else in the report as JSON, and the raw JSON.
+
+Everything in a report comes from whoever sent it, so all of it is escaped, and only http(s)
+URLs become links. The narrative (`text.div`) is never rendered as HTML - it appears only as
+text in the raw JSON.
+
+## What the pages show
+
+The list has a column each for the test engine, client and server participants (the client
+column only appears once some report has a client; a participant of any other type is shown
+as a server). **Tests** is the number of entries in `TestReport.test`. **Run Length** is how long the run took, from the earliest `test.period.start`
+to the latest `test.period.end`; it is blank when the report's tests have no periods. Both are
+worked out when the report is received, and for reports received before they existed, the
+first time the server starts with this version.
+
+Where a test script or participant URL, ignoring any `|version`, is one of the **named
+links**, its name is shown instead of the URL - linked to the named link's link, with the
+full URL in the tooltip and the version after the name.
+
+## Administration
+
+`/testing/login`. The administrator logs in as `admin` with the `adminPassword` from the
+configuration, and manages the other users at `/testing/admin/users`. A user can have either
+or both of two rights:
+
+* **edit named links** - `/testing/admin/links`: the canonical URLs that are shown by name
+* **delete reports** - a Delete button on each report's page
+
+Logins work as the publisher's do: a session cookie (set `sessionSecret`, and `cookieSecure:
+false` if the server is reached over plain http), bcrypt password hashes, and a rate limited
+login (`loginAttemptsPerWindow`, default 20 per 15 minutes). Rights are checked on every
+request, so changing or deleting a user takes effect at once. Every form post must come from
+the server's own pages: each form carries a per-session CSRF token (lusca) that the post must
+return, the cookie is SameSite=Lax, and a post whose `Origin` is another host is refused
+(behind a proxy, that needs the proxy to pass on `Host` or `X-Forwarded-Host`). The session
+and CSRF checks are only on the web pages: the FHIR API is authenticated by the header
+token, and sets no cookie.
+
+## Deleting
+
+    DELETE /testing/TestReport/{id}
+    Authorization: Bearer <adminToken>
+
+Only when `adminToken` is configured; without it, nothing can be deleted through the API.
+
+## Configuration
+
+```json
+"testing": {
+  "enabled": true,
+  "database": "testing.db",
+  "token": "secret",
+  "tokenHeader": "Authorization",
+  "adminToken": "another-secret",
+  "maxSize": 512000,
+  "rateLimit": { "windowMinutes": 1, "max": 60 },
+  "retentionDays": 365
+}
+```
+
+* `database` - relative paths are under the data folder's `databases` directory
+* `token` - if set, every POST must carry it in `tokenHeader` (a `Bearer ` prefix is optional).
+  If not set, anyone can submit.
+* `tokenHeader` - defaults to `Authorization`. `adminToken` uses the same header.
+* `maxSize` - bytes; default 500 KB. Note that the server's body parsers accept up to 50 MB
+  before this module sees the request; a `Content-Length` over the limit is refused before
+  the body is looked at.
+* `rateLimit` - POSTs per IP address per window; `max: 0` turns it off. Default 60 a minute.
+* `retentionDays` - if set, reports received longer ago than this are deleted, at startup and
+  daily. Default: keep everything.