fhirsmith 0.16.1 → 0.17.0

package/tx/problems.js

@@ -36,7 +36,7 @@ class ProblemFinder {
         }
         let list = await this.unknownVersionsHtml(sourceUnknownVersions, provider, sp);
         if (list) {
-          result = result + `<h4>${sp}</h4>` + list;
+          result = result + `<h4>${escape(sp)}</h4>` + list;
         }
       }
     }
@@ -51,14 +51,16 @@ class ProblemFinder {
     entries.sort((a, b) => a[0].localeCompare(b[0]));
     let html = '<table class="grid"><thead><tr><th>System</th><th>Unknown Versions</th><th>Known Versions</th></tr></thead><tbody>';
     for (const [system, vset] of entries) {
-      const systemEsc = escape(system);
+      // values from the loaded packages: URL-encoded in links, HTML-escaped everywhere
+      const elements = '&_elements=url%2Cversion%2Cname%2Ctitle%2Cstatus%2Ccontent%2Cdate';
       const versions = [...vset].sort((a, b) => a.localeCompare(b));
       const versionRefs = [];
       for (const v of versions) {
-        versionRefs.push(`<a href="ValueSet?system=${systemEsc}|${escape(v)}&source=${source}&_elements=url%2Cversion%2Cname%2Ctitle%2Cstatus%2Ccontent%2Cdate">${v}</a>`);
+        const href = `ValueSet?system=${encodeURIComponent(system + '|' + v)}&source=${encodeURIComponent(source)}${elements}`;
+        versionRefs.push(`<a href="${escape(href)}">${escape(v)}</a>`);
       }
-      const knownVersions = [...await provider.listCodeSystemVersions(system)].join('<br/>');
-      html += `<tr><td><a href="CodeSystem?url=${systemEsc}&_elements=url%2Cversion%2Cname%2Ctitle%2Cstatus%2Ccontent%2Cdate">${system}</a></td>`+
+      const knownVersions = [...await provider.listCodeSystemVersions(system)].map(v => escape(v)).join('<br/>');
+      html += `<tr><td><a href="${escape(`CodeSystem?url=${encodeURIComponent(system)}${elements}`)}">${escape(system)}</a></td>`+
         `<td>${versionRefs.join('<br/>')}</td><td>${knownVersions}</td></tr>`;
     }
     html += '</tbody></table>';

package/tx/provider.js

@@ -1,4 +1,5 @@
 const { CodeSystem } = require("./library/codesystem");
+const { assertNoActiveContent } = require('../library/narrative-check');
 const {VersionUtilities} = require("../library/version-utilities");
 const { FhirCodeSystemProvider} = require("./cs/cs-cs");
 const {OperationContext} = require("./operation-context");
@@ -184,7 +185,10 @@ class Provider {
     for (const resource of resources) {
       let cs;
       try {
-        cs = new CodeSystem(await contentLoader.loadFile(resource, contentLoader.fhirVersion()));
+        const json = await contentLoader.loadFile(resource, contentLoader.fhirVersion());
+        // narrative is shown as-is in a page, so it must not carry script (see narrative-check.js)
+        assertNoActiveContent(json);
+        cs = new CodeSystem(json);
       } catch (e) {
         // one unacceptable resource (e.g. contained resources the server doesn't support)
         // must not stop the rest of the package loading

package/tx/tx-html.js

@@ -9,6 +9,7 @@ const htmlServer = require('../library/html-server');
 const Logger = require('../library/logger');
 const packageJson = require("../package.json");
 const escape = require('escape-html');
+const { checkResourceNarratives } = require('../library/narrative-check');
 const {ExpandWorker} = require("./workers/expand");
 const ValueSet = require("./library/valueset");
 const {CodeSystemXML} = require("./xml/codesystem-xml");
@@ -1017,7 +1018,7 @@ class TxHtmlRenderer {
       } else if (_fmt == "html/xml") {
         html += await this.renderResourceXml(json);
       } else if (_fmt == "html/narrative") {
-        html += await this.renderResourceWithNarrative(json, json.text?.div);
+        html += await this.renderOriginalNarrative(json);
       } else if (_fmt == "html/ops") {
         html += await this.liquid.renderFile('codesystem-operations', {
           opsId: this.generateResourceId(),
@@ -1040,9 +1041,10 @@ class TxHtmlRenderer {
 
   tab(b, name, rtype, type, id) {
     if (b) {
-      return `<li class="active"><a href="#">${name}</a></li>`;
+      return `<li class="active"><a href="#">${escape(name)}</a></li>`;
     } else {
-      return `<li><a href="${this.path}/${rtype}/${id}?_format=${type}">${name}</a></li>`;
+      const href = `${this.path}/${encodeURIComponent(rtype)}/${encodeURIComponent(id)}?_format=${encodeURIComponent(type)}`;
+      return `<li><a href="${escape(href)}">${escape(name)}</a></li>`;
     }
   }
   /**
@@ -1078,7 +1080,7 @@ class TxHtmlRenderer {
       } else if (_fmt == "html/xml") {
         html += await this.renderResourceXml(json);
       } else if (_fmt == "html/narrative") {
-        html += await this.renderResourceWithNarrative(json, json.text?.div);
+        html += await this.renderOriginalNarrative(json);
       } else if (_fmt == "html/ops") {
         html += await this.liquid.renderFile('valueset-operations', {
           opsId: this.generateResourceId(),
@@ -1218,7 +1220,7 @@ class TxHtmlRenderer {
       } else if (_fmt == "html/json") {
         html += await this.renderResourceJson(json);
       } else if (_fmt == "html/narrative") {
-        html += await this.renderResourceWithNarrative(json, json.text?.div);
+        html += await this.renderOriginalNarrative(json);
       } else if (_fmt == "html/ops") {
         const sourceSet = new Set();
         const codeSet = new Set();
@@ -1725,7 +1727,7 @@ class TxHtmlRenderer {
     html += '<div class="card-header">Bundle</div>';
     html += '<div class="card-body">';
     html += `<p><strong>Type:</strong> ${escape(json.type)}</p>`;
-    html += `<p><strong>Total:</strong> ${json.total || 'N/A'}</p>`;
+    html += `<p><strong>Total:</strong> ${escape(String(json.total ?? 'N/A'))}</p>`;
     html += '</div>';
     html += '</div>';
 
@@ -1826,6 +1828,18 @@ class TxHtmlRenderer {
     return html;
   }
 
+  /**
+   * The resource's own narrative, as it is - unless it carries active content, which is
+   * refused at load time but may reach here from a source that isn't loaded that way
+   */
+  async renderOriginalNarrative(json) {
+    const found = checkResourceNarratives({ resourceType: json.resourceType, id: json.id, text: json.text });
+    if (found) {
+      return `<div class="alert alert-danger">This narrative is not shown: ${escape(found)}</div>`;
+    }
+    return this.renderResourceWithNarrative(json, json.text?.div);
+  }
+
   async renderResourceJson(json) {
     let html = "";
     html += `<div class="json-content" style="margin-top: 10px;">`;
@@ -1864,7 +1878,8 @@ class TxHtmlRenderer {
     return await this.liquid.renderFile('operations-form', {
       vcSystemId,
       inferSystemId,
-      valueSetsJson: JSON.stringify(json.valueSets || [])
+      // inside a <script>: no '<', so no value can close it
+      valueSetsJson: JSON.stringify(json.valueSets || []).replace(/</g, '\\u003c')
     });
   }
 

package/tx/tx.js

@@ -211,7 +211,7 @@ class TXModule {
           }
         };
         void prune(); // errors are logged inside
-        this.timers.push(setInterval(prune, 24 * 60 * 60 * 1000));
+        this.timers.push(setInterval(() => void prune(), 24 * 60 * 60 * 1000));
       }
     }
 
@@ -305,13 +305,14 @@ class TXModule {
     if (this.stats) {
       this.stats.addTask("Client Cache", "5 min");
     }
-    this.timers.push(setInterval(async () => {
+    const updateCodeSystemList = async () => {
       try {
         await endpointInfo.provider.updateCodeSystemList();
       } catch (error) {
         this.log.error(`Error updating CodeSystem list for ${endpointPath}: ${error.message}`);
       }
-    }, 60 * 1000));
+    };
+    this.timers.push(setInterval(() => void updateCodeSystemList(), 60 * 1000)); // errors are logged inside
     this.log.info(`CodeSystem list update scheduled for ${endpointPath}`);
     this.timers.push(setInterval(() => {
       endpointInfo.resourceCache.prune(cacheTimeoutMs);

package/tx/vs/vs-database.js

@@ -4,6 +4,9 @@ const sqlite3 = require('sqlite3').verbose();
 const { VersionUtilities } = require('../../library/version-utilities');
 const ValueSet = require("../library/valueset");
 
+// Settings key written once a package cache has been fully populated (see isComplete)
+const CACHE_COMPLETE_KEY = 'cache-complete';
+
 // Columns that can be returned directly without parsing JSON
 const INDEXED_COLUMNS = ['id', 'url', 'version', 'date', 'description', 'name', 'publisher', 'status', 'title'];
 
@@ -485,6 +488,29 @@ class ValueSetDatabase {
     });
   }
 
+  /**
+   * True if this cache was fully populated: markComplete() was reached. A cache
+   * whose population was cut short (the process killed part way through) has no
+   * marker - and so does a file too damaged to open or read - so it reports false.
+   * @returns {Promise<boolean>}
+   */
+  async isComplete() {
+    try {
+      return (await this.getSetting(CACHE_COMPLETE_KEY)) != null;
+    } catch (e) {
+      return false;
+    }
+  }
+
+  /**
+   * Record that this cache has been fully populated. Called once, after the last
+   * value set has been written
+   * @returns {Promise<void>}
+   */
+  async markComplete() {
+    await this.setSetting(CACHE_COMPLETE_KEY, new Date().toISOString());
+  }
+
   /**
    * Insert or update a single ValueSet in the database
    * @param {Object} valueSet - The ValueSet resource

package/tx/vs/vs-package.js

@@ -1,4 +1,5 @@
 const path = require('path');
+const { assertNoActiveContent } = require('../../library/narrative-check');
 const fs = require('fs').promises;
 const { AbstractValueSetProvider } = require('./vs-api');
 const { PackageContentLoader } = require('../../library/package-manager');
@@ -21,7 +22,9 @@ class PackageValueSetProvider extends AbstractValueSetProvider {
     super();
     validateParameter(packageLoader, "packageLoader", PackageContentLoader);
     this.packageLoader = packageLoader;
-    this.dbPath = path.join(packageLoader.packageFolder, '.valuesets.db');
+    // -2: caches from before value sets were checked for active content in their narrative
+    // are rebuilt (see initialize)
+    this.dbPath = path.join(packageLoader.packageFolder, '.valuesets-2.db');
     this.database = new ValueSetDatabase(this.dbPath);
     this.valueSetMap = new Map();
     this.initialized = false;
@@ -45,12 +48,21 @@ class PackageValueSetProvider extends AbstractValueSetProvider {
       return;
     }
     await this.packageLoader.initialize();
-
-    const dbExists = await this.database.exists();
+    await fs.rm(path.join(this.packageLoader.packageFolder, '.valuesets.db'), { force: true }); // the old cache
+
+    // A cache is only used once its population finished. One that was cut short
+    // (the process killed part way through) would otherwise be trusted forever,
+    // silently missing whatever hadn't been written yet
+    let dbExists = await this.database.exists();
+    if (dbExists && !(await this.database.isComplete())) {
+      require('../../library/logger').getInstance().child({ module: 'tx' })
+        .warn(`Package ${this.packageLoader.pid ? this.packageLoader.pid() : ''}: value set cache ${this.dbPath} is incomplete - rebuilding it`);
+      await this.#discardDatabase();
+      dbExists = false;
+    }
 
     if (!dbExists) {
-      await this.database.create();
-      await this._populateDatabase();
+      await this.#buildDatabase();
     }
 
     this.valueSetMap = await this.database.loadAllValueSets(this.sourcePackage());
@@ -60,11 +72,8 @@ class PackageValueSetProvider extends AbstractValueSetProvider {
     // package-manager.js): if any cached value set still carries a "system|version"
     // include or a pipe in its url, rebuild this cache from the package.
     if (dbExists && this.#hasPipeVersionedCanonicals()) {
-      await this.database.close();
-      await fs.rm(this.dbPath, { force: true });
-      this.database = new ValueSetDatabase(this.dbPath);
-      await this.database.create();
-      await this._populateDatabase();
+      await this.#discardDatabase();
+      await this.#buildDatabase();
       this.valueSetMap = await this.database.loadAllValueSets(this.sourcePackage());
     }
 
@@ -84,6 +93,27 @@ class PackageValueSetProvider extends AbstractValueSetProvider {
     this.initialized = true;
   }
 
+  /**
+   * Create the cache and fill it from the package. The complete marker is written
+   * last, so a build that doesn't finish is detected (and redone) next time
+   * @private
+   */
+  async #buildDatabase() {
+    await this.database.create();
+    await this._populateDatabase();
+    await this.database.markComplete();
+  }
+
+  /**
+   * Close and delete the cache file, leaving a fresh (not yet created) database
+   * @private
+   */
+  async #discardDatabase() {
+    await this.database.close();
+    await fs.rm(this.dbPath, { force: true });
+    this.database = new ValueSetDatabase(this.dbPath);
+  }
+
   /**
    * True if any loaded value set still has a pipe-versioned canonical (url or
    * compose include/exclude system) - the signature of a cache written before
@@ -134,6 +164,8 @@ class PackageValueSetProvider extends AbstractValueSetProvider {
           // don't store a value set the server can't use (e.g. one that contains a
           // CodeSystem): it would be listed by search, and fail when read
           checkContained(valueSet);
+          // its narrative is shown as-is in a page, so it must not carry script
+          assertNoActiveContent(valueSet);
         } catch (e) {
           require('../../library/logger').getInstance().child({ module: 'tx' })
             .warn(`Package ${this.packageLoader.pid ? this.packageLoader.pid() : ''}: skipping ValueSet ${valueSet.url}: ${e.message}`);

package/tx/workers/expand.js

@@ -836,7 +836,7 @@ class ValueSetExpander {
               notClosed.value = true;
             }
             const prep = await cs.getPrepContext(true);
-            const ctxt = await cs.searchFilter(prep, filter, false);
+            const ctxt = await cs.searchFilter(prep, filter, true);
             let set = await cs.executeFilters(prep);
             if (this.count === 0 && valueSets.length === 0) {
               // count=0 (count only): report the filtered size directly without

package/tx/workers/metadata.js

@@ -472,7 +472,12 @@ class MetadataHandler {
         { name: 'useSupplement' },
         { name: 'valueSet' },
         { name: 'valueSetVersion' }
-      ]
+      ],
+      textFilter: 'On code systems loaded from files and packages, a text filter ignores case, and accents on Latin, Greek and Cyrillic letters. ' +
+        'It matches concepts whose code, display, definition or a designation contains the text. ' +
+        'A filter of several words also matches a display or designation in which each word starts a word, ' +
+        'in any order; for this, punctuation and spaces only separate words. The closest matches come first. ' +
+        'Other code systems, such as SNOMED CT, LOINC and RxNorm, match text in their own way.'
     };
   }
 

package/utilities/submit-test-report.js

@@ -11,7 +11,7 @@
  *
  *   node utilities/submit-test-report.js [-server url] [-file path] [-token token] [-header name]
  *
- *   -server  the /testing base; default https://testing.fhir.org/testing
+ *   -server  the /testing base; default https://npm.fhir.org/testing
  *   -file    default test-cases-report.json in the FHIRsmith root
  *   -token   if the server requires one; or set FHIRSMITH_TESTING_TOKEN
  *   -header  the header the token goes in; default Authorization (sent as "Bearer <token>")

package/xig/xig.js

@@ -19,12 +19,14 @@ const escape = require('escape-html');
 
 const Logger = require('../library/logger');
 const {describeCron} = require("../library/cron-utilities");
+const {tokenMatches, tokenConfigured} = require('../library/request-token');
 const xigLog = Logger.getInstance().child({ module: 'xig' });
 
 const router = express.Router();
 
 // Configuration
-const XIG_DB_URL = 'http://fhir.org/guides/stats/xig.db';
+// https: the database is trusted content (its HTML is shown as is), so it must come from fhir.org
+const XIG_DB_URL = 'https://fhir.org/guides/stats/xig.db';
 const XIG_DB_PATH = folders.filePath('xig', 'xig.db');
 const TEMPLATE_PATH = path.join(__dirname, 'xig-template.html');
 
@@ -1718,6 +1720,11 @@ function downloadFile(url, destination, maxRedirects = 5) {
               }
             }
 
+            // never be redirected off https
+            if (new URL(redirectUrl).protocol !== 'https:') {
+              reject(Object.assign(new Error(`Refused a redirect to ${redirectUrl}: the database is only downloaded over https`), { downloadMeta }));
+              return;
+            }
             attemptDownload(redirectUrl, redirectCount + 1);
             return;
           }
@@ -2203,6 +2210,23 @@ function getDatabaseAgeInfo() {
   };
 }
 
+// The form on /xig/stats that starts an update (POST /xig/update), when an updateToken is configured
+function buildUpdateForm() {
+  if (!tokenConfigured(xigSettings.updateToken)) {
+    return '<p class="text-muted">An update can be started from here when an updateToken is set in the xig configuration.</p>';
+  }
+  let html = '<h4>Update the database now</h4>';
+  html += '<form method="POST" action="/xig/update" class="mb-4">';
+  html += '<p>Downloads the current database from fhir.org (about 3 GB) and switches to it. ';
+  html += updateInProgress ? '<strong>An update is running now.</strong>' : 'It takes several minutes.';
+  html += '</p>';
+  html += '<label for="xig-update-token">Update token</label> ';
+  html += '<input type="password" id="xig-update-token" name="token" required autocomplete="off" maxlength="200"> ';
+  html += '<button type="submit" class="btn btn-primary">Start the update</button>';
+  html += '</form>';
+  return html;
+}
+
 function buildStatsTable(statsData) {
   let html = '<table class="table table-striped table-bordered">';
   html += '<thead class="table-dark">';
@@ -2429,16 +2453,25 @@ router.get('/', async (req, res) => {
       const title = 'FHIR Resources';
 
       // Parse query parameters
+      // Each a string: a repeated parameter (?text=a&text=b) arrives as an array, which
+      // the filters and SQL builders would fail on. Capped in length
+      const qp = (name, dflt = '') => {
+        let v = req.query[name];
+        if (Array.isArray(v)) {
+          v = v[0];
+        }
+        return typeof v === 'string' && v !== '' ? v.substring(0, 200) : dflt;
+      };
       const queryParams = {
-        ver: req.query.ver || '',
-        auth: req.query.auth || '',
-        realm: req.query.realm || '',
-        type: req.query.type || '',
-        rt: req.query.rt || '',
-        text: req.query.text || '',
-        pkg: req.query.pkg || '',
-        onlyUsed: req.query.onlyUsed || '',
-        offset: req.query.offset || '0'
+        ver: qp('ver'),
+        auth: qp('auth'),
+        realm: qp('realm'),
+        type: qp('type'),
+        rt: qp('rt'),
+        text: qp('text'),
+        pkg: qp('pkg'),
+        onlyUsed: qp('onlyUsed'),
+        offset: qp('offset', '0')
       };
 
       // Parse offset for pagination
@@ -2600,7 +2633,7 @@ router.get('/stats', async (req, res) => {
         introContent += `</div>`;
       }
 
-      const fullContent = introContent + content;
+      const fullContent = introContent + content + buildUpdateForm();
 
       const stats = await gatherPageStatistics();
       stats.processingTime = Date.now() - startTime;
@@ -3125,8 +3158,44 @@ router.get('/cache', async (req, res) => {
   }
 });
 
-router.post('/update', async (req, res) => {
+// POST /xig/update re-downloads the database (about 3 GB), so it needs the updateToken from
+// the xig config: in the x-update-token header, or (from the form on /xig/stats) in a
+// "token" form field. With no updateToken it is disabled.
+//
+// The form gets a page back straight away, while the update runs on (it takes minutes);
+// /xig/stats shows how it went. Other callers wait for the update and get JSON.
+router.post('/update', express.urlencoded({ extended: false, limit: '2kb' }), async (req, res) => {
+  const fromForm = !!req.is('application/x-www-form-urlencoded');
+  const page = (status, alert, message) => {
+    const content = `<div class="alert alert-${alert}">${escape(message)}</div><p><a href="/xig/stats">Back to the status page</a></p>`;
+    res.status(status).setHeader('Content-Type', 'text/html');
+    res.send(renderPage('Update the XIG database', content));
+  };
+  const refuse = (message) => fromForm ? page(403, 'danger', message) : res.status(403).json({ status: 'ERROR', error: 'forbidden: ' + message });
+
   try {
+    if (!tokenConfigured(xigSettings.updateToken)) {
+      refuse('no updateToken is configured, so this endpoint is disabled');
+      return;
+    }
+    const supplied = req.headers['x-update-token'] ?? req.body?.token;
+    if (!tokenMatches(xigSettings.updateToken, supplied)) {
+      xigLog.warn('Rejected /xig/update: missing or invalid token (from ' + req.ip + ')');
+      refuse('missing or invalid x-update-token');
+      return;
+    }
+
+    if (fromForm) {
+      if (updateInProgress) {
+        page(200, 'info', 'An update is already running. The status page shows how it is going.');
+        return;
+      }
+      xigLog.info('Manual update triggered from the status page');
+      updateXigDatabase().catch((err) => xigLog.error('XIG database update failed: ' + err.message));
+      page(202, 'success', 'The update has started. It takes several minutes; the status page shows how it is going.');
+      return;
+    }
+
     xigLog.info('Manual update triggered via API');
     await updateXigDatabase();
     res.json({
@@ -3143,10 +3212,16 @@ router.post('/update', async (req, res) => {
 });
 
 let globalStats;
+let xigSettings = {};
+// The xig section of the config (initializeXigModule; tests set it directly)
+function setXigSettings(xigConfig) {
+  xigSettings = xigConfig || {};
+}
 // Initialize the XIG module
 async function initializeXigModule(stats, xigConfig) {
   try {
     globalStats = stats;
+    setXigSettings(xigConfig);
     loadTemplate();
 
     await initializeDatabase();
@@ -3196,6 +3271,7 @@ function shutdown() {
 // Export everything
 module.exports = {
   router,
+  setXigSettings,
   updateXigDatabase,
   getDatabaseInfo,
   shutdown,

package/gr.sh

@@ -1,3 +0,0 @@
-#!/usr/bin/env bash 
-git tag v$1
-git push origin v$1