fhirsmith 0.16.0 → 0.17.0

package/tx/problems.js

@@ -36,7 +36,7 @@ class ProblemFinder {
         }
         let list = await this.unknownVersionsHtml(sourceUnknownVersions, provider, sp);
         if (list) {
-          result = result + `<h4>${sp}</h4>` + list;
+          result = result + `<h4>${escape(sp)}</h4>` + list;
         }
       }
     }
@@ -51,14 +51,16 @@ class ProblemFinder {
     entries.sort((a, b) => a[0].localeCompare(b[0]));
     let html = '<table class="grid"><thead><tr><th>System</th><th>Unknown Versions</th><th>Known Versions</th></tr></thead><tbody>';
     for (const [system, vset] of entries) {
-      const systemEsc = escape(system);
+      // values from the loaded packages: URL-encoded in links, HTML-escaped everywhere
+      const elements = '&_elements=url%2Cversion%2Cname%2Ctitle%2Cstatus%2Ccontent%2Cdate';
       const versions = [...vset].sort((a, b) => a.localeCompare(b));
       const versionRefs = [];
       for (const v of versions) {
-        versionRefs.push(`<a href="ValueSet?system=${systemEsc}|${escape(v)}&source=${source}&_elements=url%2Cversion%2Cname%2Ctitle%2Cstatus%2Ccontent%2Cdate">${v}</a>`);
+        const href = `ValueSet?system=${encodeURIComponent(system + '|' + v)}&source=${encodeURIComponent(source)}${elements}`;
+        versionRefs.push(`<a href="${escape(href)}">${escape(v)}</a>`);
       }
-      const knownVersions = [...await provider.listCodeSystemVersions(system)].join('<br/>');
-      html += `<tr><td><a href="CodeSystem?url=${systemEsc}&_elements=url%2Cversion%2Cname%2Ctitle%2Cstatus%2Ccontent%2Cdate">${system}</a></td>`+
+      const knownVersions = [...await provider.listCodeSystemVersions(system)].map(v => escape(v)).join('<br/>');
+      html += `<tr><td><a href="${escape(`CodeSystem?url=${encodeURIComponent(system)}${elements}`)}">${escape(system)}</a></td>`+
         `<td>${versionRefs.join('<br/>')}</td><td>${knownVersions}</td></tr>`;
     }
     html += '</tbody></table>';

package/tx/provider.js

@@ -1,4 +1,5 @@
 const { CodeSystem } = require("./library/codesystem");
+const { assertNoActiveContent } = require('../library/narrative-check');
 const {VersionUtilities} = require("../library/version-utilities");
 const { FhirCodeSystemProvider} = require("./cs/cs-cs");
 const {OperationContext} = require("./operation-context");
@@ -184,7 +185,10 @@ class Provider {
     for (const resource of resources) {
       let cs;
       try {
-        cs = new CodeSystem(await contentLoader.loadFile(resource, contentLoader.fhirVersion()));
+        const json = await contentLoader.loadFile(resource, contentLoader.fhirVersion());
+        // narrative is shown as-is in a page, so it must not carry script (see narrative-check.js)
+        assertNoActiveContent(json);
+        cs = new CodeSystem(json);
       } catch (e) {
         // one unacceptable resource (e.g. contained resources the server doesn't support)
         // must not stop the rest of the package loading

package/tx/tx-html.js

@@ -9,6 +9,7 @@ const htmlServer = require('../library/html-server');
 const Logger = require('../library/logger');
 const packageJson = require("../package.json");
 const escape = require('escape-html');
+const { checkResourceNarratives } = require('../library/narrative-check');
 const {ExpandWorker} = require("./workers/expand");
 const ValueSet = require("./library/valueset");
 const {CodeSystemXML} = require("./xml/codesystem-xml");
@@ -1017,7 +1018,7 @@ class TxHtmlRenderer {
       } else if (_fmt == "html/xml") {
         html += await this.renderResourceXml(json);
       } else if (_fmt == "html/narrative") {
-        html += await this.renderResourceWithNarrative(json, json.text?.div);
+        html += await this.renderOriginalNarrative(json);
       } else if (_fmt == "html/ops") {
         html += await this.liquid.renderFile('codesystem-operations', {
           opsId: this.generateResourceId(),
@@ -1040,9 +1041,10 @@ class TxHtmlRenderer {
 
   tab(b, name, rtype, type, id) {
     if (b) {
-      return `<li class="active"><a href="#">${name}</a></li>`;
+      return `<li class="active"><a href="#">${escape(name)}</a></li>`;
     } else {
-      return `<li><a href="${this.path}/${rtype}/${id}?_format=${type}">${name}</a></li>`;
+      const href = `${this.path}/${encodeURIComponent(rtype)}/${encodeURIComponent(id)}?_format=${encodeURIComponent(type)}`;
+      return `<li><a href="${escape(href)}">${escape(name)}</a></li>`;
     }
   }
   /**
@@ -1078,7 +1080,7 @@ class TxHtmlRenderer {
       } else if (_fmt == "html/xml") {
         html += await this.renderResourceXml(json);
       } else if (_fmt == "html/narrative") {
-        html += await this.renderResourceWithNarrative(json, json.text?.div);
+        html += await this.renderOriginalNarrative(json);
       } else if (_fmt == "html/ops") {
         html += await this.liquid.renderFile('valueset-operations', {
           opsId: this.generateResourceId(),
@@ -1218,7 +1220,7 @@ class TxHtmlRenderer {
       } else if (_fmt == "html/json") {
         html += await this.renderResourceJson(json);
       } else if (_fmt == "html/narrative") {
-        html += await this.renderResourceWithNarrative(json, json.text?.div);
+        html += await this.renderOriginalNarrative(json);
       } else if (_fmt == "html/ops") {
         const sourceSet = new Set();
         const codeSet = new Set();
@@ -1329,6 +1331,30 @@ class TxHtmlRenderer {
     return html;
   }
 
+  /**
+   * The page for a read of an id that nothing has, but that several resources had in their
+   * source (see ReadWorker.notFound): a list of them, to pick from.
+   *
+   * @param {string} resourceType
+   * @param {string} id - the id that was asked for
+   * @param {Array<{id, url, version, title, source}>} matches
+   * @returns {string} html
+   */
+  renderIdChoices(resourceType, id, matches) {
+    let html = `<p>There is no ${escape(resourceType)} with the id <code>${escape(id)}</code> on this server. ` +
+      `Every resource here has an id that starts with a prefix for the source it came from, and ` +
+      `${matches.length} ${escape(resourceType)} resources had the id <code>${escape(id)}</code> in their source:</p>`;
+    html += '<table class="grid"><tr><th>Id</th><th>URL</th><th>Version</th><th>Title</th><th>Source</th></tr>';
+    for (const m of matches) {
+      // relative to /{type}/{id}, like the redirect when there is only one
+      html += `<tr><td><a href="${escape(encodeURIComponent(m.id))}">${escape(m.id)}</a></td>` +
+        `<td>${escape(m.url || '')}</td><td>${escape(m.version || '')}</td>` +
+        `<td>${escape(m.title || '')}</td><td>${escape(m.source || '')}</td></tr>`;
+    }
+    html += '</table>';
+    return html;
+  }
+
   /**
    * Render Bundle resource
    */
@@ -1701,7 +1727,7 @@ class TxHtmlRenderer {
     html += '<div class="card-header">Bundle</div>';
     html += '<div class="card-body">';
     html += `<p><strong>Type:</strong> ${escape(json.type)}</p>`;
-    html += `<p><strong>Total:</strong> ${json.total || 'N/A'}</p>`;
+    html += `<p><strong>Total:</strong> ${escape(String(json.total ?? 'N/A'))}</p>`;
     html += '</div>';
     html += '</div>';
 
@@ -1802,6 +1828,18 @@ class TxHtmlRenderer {
     return html;
   }
 
+  /**
+   * The resource's own narrative, as it is - unless it carries active content, which is
+   * refused at load time but may reach here from a source that isn't loaded that way
+   */
+  async renderOriginalNarrative(json) {
+    const found = checkResourceNarratives({ resourceType: json.resourceType, id: json.id, text: json.text });
+    if (found) {
+      return `<div class="alert alert-danger">This narrative is not shown: ${escape(found)}</div>`;
+    }
+    return this.renderResourceWithNarrative(json, json.text?.div);
+  }
+
   async renderResourceJson(json) {
     let html = "";
     html += `<div class="json-content" style="margin-top: 10px;">`;
@@ -1840,7 +1878,8 @@ class TxHtmlRenderer {
     return await this.liquid.renderFile('operations-form', {
       vcSystemId,
       inferSystemId,
-      valueSetsJson: JSON.stringify(json.valueSets || [])
+      // inside a <script>: no '<', so no value can close it
+      valueSetsJson: JSON.stringify(json.valueSets || []).replace(/</g, '\\u003c')
     });
   }
 

package/tx/tx.js

@@ -211,7 +211,7 @@ class TXModule {
           }
         };
         void prune(); // errors are logged inside
-        this.timers.push(setInterval(prune, 24 * 60 * 60 * 1000));
+        this.timers.push(setInterval(() => void prune(), 24 * 60 * 60 * 1000));
       }
     }
 
@@ -305,13 +305,14 @@ class TXModule {
     if (this.stats) {
       this.stats.addTask("Client Cache", "5 min");
     }
-    this.timers.push(setInterval(async () => {
+    const updateCodeSystemList = async () => {
       try {
         await endpointInfo.provider.updateCodeSystemList();
       } catch (error) {
         this.log.error(`Error updating CodeSystem list for ${endpointPath}: ${error.message}`);
       }
-    }, 60 * 1000));
+    };
+    this.timers.push(setInterval(() => void updateCodeSystemList(), 60 * 1000)); // errors are logged inside
     this.log.info(`CodeSystem list update scheduled for ${endpointPath}`);
     this.timers.push(setInterval(() => {
       endpointInfo.resourceCache.prune(cacheTimeoutMs);
@@ -516,8 +517,12 @@ class TXModule {
             return res.status(400).json(buildOperationOutcome('error', 'invalid', `Invalid XML: ${e.message}`));
           }
         }
-      } else if (contentType != 'application/x-www-form-urlencoded') {
-        return res.status(415).json(buildOperationOutcome('error', 'invalid', `Unsupported Media Type: ${contentType}`));
+      } else if (contentType.split(';')[0].trim().toLowerCase() != 'application/x-www-form-urlencoded') {
+        // compare the media type alone: a form post may carry a charset parameter
+        const message = contentType
+          ? `Unsupported Media Type: ${contentType}`
+          : `No Content-Type was given for this ${req.method}; use application/fhir+json or application/fhir+xml`;
+        return res.status(415).json(buildOperationOutcome('error', 'invalid', message));
       }
 
       if (req.body) {
@@ -994,7 +999,7 @@ class TXModule {
             ));
           }
           let worker = new ReadWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
-          await worker.handle(req, res, resourceType);
+          await worker.handle(req, res, resourceType, (req, res, resourceType, id, matches) => this.renderIdChoices(req, res, resourceType, id, matches));
         } finally {
           this.countRequest(endpointPath, 'read', Date.now() - start);
         }
@@ -1432,6 +1437,17 @@ class TXModule {
     return count;
   }
 
+  // A read of an unprefixed id that several resources had in their source: the page
+  // that lists them (status 300, Multiple Choices). See ReadWorker.notFound.
+  async renderIdChoices(req, res, resourceType, id, matches) {
+    const txhtml = new TxHtmlRenderer(new Renderer(req.txOpContext, req.txEndpoint.provider), this.liquid, this.languages, this.i18n, req.txEndpoint.path);
+    const content = txhtml.renderIdChoices(resourceType, id, matches);
+    const html = await txhtml.renderPage(`${resourceType}/${id}`, content, req.txEndpoint, req.txStartTime);
+    res.status(300);
+    res.setHeader('Content-Type', 'text/html');
+    return res.send(html);
+  }
+
   // $closure usage for the home page: null when closure isn't turned on
   closureStats() {
     if (!this.closureStore) {

package/tx/vs/vs-database.js

@@ -4,6 +4,9 @@ const sqlite3 = require('sqlite3').verbose();
 const { VersionUtilities } = require('../../library/version-utilities');
 const ValueSet = require("../library/valueset");
 
+// Settings key written once a package cache has been fully populated (see isComplete)
+const CACHE_COMPLETE_KEY = 'cache-complete';
+
 // Columns that can be returned directly without parsing JSON
 const INDEXED_COLUMNS = ['id', 'url', 'version', 'date', 'description', 'name', 'publisher', 'status', 'title'];
 
@@ -485,6 +488,29 @@ class ValueSetDatabase {
     });
   }
 
+  /**
+   * True if this cache was fully populated: markComplete() was reached. A cache
+   * whose population was cut short (the process killed part way through) has no
+   * marker - and so does a file too damaged to open or read - so it reports false.
+   * @returns {Promise<boolean>}
+   */
+  async isComplete() {
+    try {
+      return (await this.getSetting(CACHE_COMPLETE_KEY)) != null;
+    } catch (e) {
+      return false;
+    }
+  }
+
+  /**
+   * Record that this cache has been fully populated. Called once, after the last
+   * value set has been written
+   * @returns {Promise<void>}
+   */
+  async markComplete() {
+    await this.setSetting(CACHE_COMPLETE_KEY, new Date().toISOString());
+  }
+
   /**
    * Insert or update a single ValueSet in the database
    * @param {Object} valueSet - The ValueSet resource

package/tx/vs/vs-package.js

@@ -1,4 +1,5 @@
 const path = require('path');
+const { assertNoActiveContent } = require('../../library/narrative-check');
 const fs = require('fs').promises;
 const { AbstractValueSetProvider } = require('./vs-api');
 const { PackageContentLoader } = require('../../library/package-manager');
@@ -21,7 +22,9 @@ class PackageValueSetProvider extends AbstractValueSetProvider {
     super();
     validateParameter(packageLoader, "packageLoader", PackageContentLoader);
     this.packageLoader = packageLoader;
-    this.dbPath = path.join(packageLoader.packageFolder, '.valuesets.db');
+    // -2: caches from before value sets were checked for active content in their narrative
+    // are rebuilt (see initialize)
+    this.dbPath = path.join(packageLoader.packageFolder, '.valuesets-2.db');
     this.database = new ValueSetDatabase(this.dbPath);
     this.valueSetMap = new Map();
     this.initialized = false;
@@ -45,12 +48,21 @@ class PackageValueSetProvider extends AbstractValueSetProvider {
       return;
     }
     await this.packageLoader.initialize();
-
-    const dbExists = await this.database.exists();
+    await fs.rm(path.join(this.packageLoader.packageFolder, '.valuesets.db'), { force: true }); // the old cache
+
+    // A cache is only used once its population finished. One that was cut short
+    // (the process killed part way through) would otherwise be trusted forever,
+    // silently missing whatever hadn't been written yet
+    let dbExists = await this.database.exists();
+    if (dbExists && !(await this.database.isComplete())) {
+      require('../../library/logger').getInstance().child({ module: 'tx' })
+        .warn(`Package ${this.packageLoader.pid ? this.packageLoader.pid() : ''}: value set cache ${this.dbPath} is incomplete - rebuilding it`);
+      await this.#discardDatabase();
+      dbExists = false;
+    }
 
     if (!dbExists) {
-      await this.database.create();
-      await this._populateDatabase();
+      await this.#buildDatabase();
     }
 
     this.valueSetMap = await this.database.loadAllValueSets(this.sourcePackage());
@@ -60,11 +72,8 @@ class PackageValueSetProvider extends AbstractValueSetProvider {
     // package-manager.js): if any cached value set still carries a "system|version"
     // include or a pipe in its url, rebuild this cache from the package.
     if (dbExists && this.#hasPipeVersionedCanonicals()) {
-      await this.database.close();
-      await fs.rm(this.dbPath, { force: true });
-      this.database = new ValueSetDatabase(this.dbPath);
-      await this.database.create();
-      await this._populateDatabase();
+      await this.#discardDatabase();
+      await this.#buildDatabase();
       this.valueSetMap = await this.database.loadAllValueSets(this.sourcePackage());
     }
 
@@ -84,6 +93,27 @@ class PackageValueSetProvider extends AbstractValueSetProvider {
     this.initialized = true;
   }
 
+  /**
+   * Create the cache and fill it from the package. The complete marker is written
+   * last, so a build that doesn't finish is detected (and redone) next time
+   * @private
+   */
+  async #buildDatabase() {
+    await this.database.create();
+    await this._populateDatabase();
+    await this.database.markComplete();
+  }
+
+  /**
+   * Close and delete the cache file, leaving a fresh (not yet created) database
+   * @private
+   */
+  async #discardDatabase() {
+    await this.database.close();
+    await fs.rm(this.dbPath, { force: true });
+    this.database = new ValueSetDatabase(this.dbPath);
+  }
+
   /**
    * True if any loaded value set still has a pipe-versioned canonical (url or
    * compose include/exclude system) - the signature of a cache written before
@@ -134,6 +164,8 @@ class PackageValueSetProvider extends AbstractValueSetProvider {
           // don't store a value set the server can't use (e.g. one that contains a
           // CodeSystem): it would be listed by search, and fail when read
           checkContained(valueSet);
+          // its narrative is shown as-is in a page, so it must not carry script
+          assertNoActiveContent(valueSet);
         } catch (e) {
           require('../../library/logger').getInstance().child({ module: 'tx' })
             .warn(`Package ${this.packageLoader.pid ? this.packageLoader.pid() : ''}: skipping ValueSet ${valueSet.url}: ${e.message}`);

package/tx/workers/expand.js

@@ -836,7 +836,7 @@ class ValueSetExpander {
               notClosed.value = true;
             }
             const prep = await cs.getPrepContext(true);
-            const ctxt = await cs.searchFilter(prep, filter, false);
+            const ctxt = await cs.searchFilter(prep, filter, true);
             let set = await cs.executeFilters(prep);
             if (this.count === 0 && valueSets.length === 0) {
               // count=0 (count only): report the filtered size directly without

package/tx/workers/metadata.js

@@ -472,7 +472,12 @@ class MetadataHandler {
         { name: 'useSupplement' },
         { name: 'valueSet' },
         { name: 'valueSetVersion' }
-      ]
+      ],
+      textFilter: 'On code systems loaded from files and packages, a text filter ignores case, and accents on Latin, Greek and Cyrillic letters. ' +
+        'It matches concepts whose code, display, definition or a designation contains the text. ' +
+        'A filter of several words also matches a display or designation in which each word starts a word, ' +
+        'in any order; for this, punctuation and spaces only separate words. The closest matches come first. ' +
+        'Other code systems, such as SNOMED CT, LOINC and RxNorm, match text in their own way.'
     };
   }
 

package/tx/workers/read.js

@@ -8,6 +8,9 @@ const { TerminologyWorker } = require('./worker');
 const {debugLog} = require("../operation-context");
 
 const { buildOperationOutcome } = require('../library/operation-outcome');
+const { originalId } = require('../library/resource-ids');
+const { acceptsHtml } = require('../tx-html');
+
 class ReadWorker extends TerminologyWorker {
   /**
    * @param {OperationContext} opContext - Operation context
@@ -32,10 +35,12 @@ class ReadWorker extends TerminologyWorker {
    * @param {express.Request} req - Express request (with txProvider attached)
    * @param {express.Response} res - Express response
    * @param {string} resourceType - The resource type (CodeSystem, ValueSet, ConceptMap)
-   * @param {Object} log - Logger instance
+   * @param {Function} [renderChoices] - (req, res, resourceType, id, matches) renders the
+   *   page that lists the resources an unprefixed id could mean (see notFound)
    */
-  async handle(req, res, resourceType) {
+  async handle(req, res, resourceType, renderChoices = null) {
     const { id } = req.params;
+    this.renderChoices = renderChoices;
 
     this.log.debug(`Read ${resourceType}/${id}`);
 
@@ -114,7 +119,7 @@ class ReadWorker extends TerminologyWorker {
       }
     }
 
-    return res.status(404).json(buildOperationOutcome('error', 'not-found', `CodeSystem/${id} not found`));
+    return await this.notFound(req, res, 'CodeSystem', id);
   }
 
   /**
@@ -131,7 +136,7 @@ class ReadWorker extends TerminologyWorker {
       }
     }
 
-    return res.status(404).json(buildOperationOutcome('error', 'not-found', `ValueSet/${id} not found`));
+    return await this.notFound(req, res, 'ValueSet', id);
   }
   /**
    * Handle ConceptMap read
@@ -147,7 +152,84 @@ class ReadWorker extends TerminologyWorker {
       }
     }
 
-    return res.status(404).json(buildOperationOutcome('error', 'not-found', `ConceptMap/${id} not found`));
+    return await this.notFound(req, res, 'ConceptMap', id);
+  }
+
+  /**
+   * Nothing has the id asked for. Published IGs link to resources by the id they had in
+   * their source (ValueSet/2.16.840.1.113762.1.4.1), from before every id was put in its
+   * provider's id space (ValueSet/vsac-2.16.840.1.113762.1.4.1). So for a browser, look
+   * for resources that had that id in their source: if there's one, redirect to it, and
+   * if there are several, list them. Anything other than a browser still gets a 404 -
+   * an unprefixed id can't say which resource it means (see library/resource-ids.js).
+   */
+  async notFound(req, res, resourceType, id) {
+    if (this.renderChoices && acceptsHtml(req)) {
+      const matches = await this.findByOriginalId(resourceType, id);
+      if (matches.length === 1) {
+        req.logInfo = `redirected to ${matches[0].id}`;
+        // relative, so that it works whatever path the server is behind; keeps the query
+        const q = req.originalUrl.indexOf('?');
+        return res.redirect(302, encodeURIComponent(matches[0].id) + (q >= 0 ? req.originalUrl.substring(q) : ''));
+      }
+      if (matches.length > 1) {
+        req.logInfo = `${matches.length} matches`;
+        return await this.renderChoices(req, res, resourceType, id, matches);
+      }
+    }
+    return res.status(404).json(buildOperationOutcome('error', 'not-found', `${resourceType}/${id} not found`));
+  }
+
+  /**
+   * The resources of a type that had this id in their source, across all the providers.
+   *
+   * @returns {Promise<Array<{id, url, version, title, source}>>} in provider order
+   */
+  async findByOriginalId(resourceType, id) {
+    const found = [];
+    const add = (resource) => {
+      const json = resource.jsonObj || resource;
+      if (json.id && !found.some(m => m.id === json.id)) {
+        found.push({
+          id: json.id,
+          url: json.url || null,
+          version: json.version || null,
+          title: json.title || json.name || null,
+          source: typeof resource.sourcePackage === 'string' ? resource.sourcePackage : null
+        });
+      }
+    };
+
+    switch (resourceType) {
+      case 'CodeSystem':
+        // the map holds a code system under more than one key (url, url|version)
+        for (const cs of new Set(this.provider.codeSystems.values())) {
+          this.deadCheck('findByOriginalId-cs');
+          if (originalId(cs.jsonObj.id) === id) {
+            add(cs);
+          }
+        }
+        break;
+      case 'ValueSet':
+        for (const vsp of this.provider.valueSetProviders) {
+          this.deadCheck('findByOriginalId-vs');
+          const vs = vsp.spaceId ? await vsp.fetchValueSetById(`${vsp.spaceId}-${id}`) : null;
+          if (vs) {
+            add(vs);
+          }
+        }
+        break;
+      case 'ConceptMap':
+        for (const cmp of this.provider.conceptMapProviders) {
+          this.deadCheck('findByOriginalId-cm');
+          const cm = cmp.spaceId ? await cmp.fetchConceptMapById(`${cmp.spaceId}-${id}`) : null;
+          if (cm) {
+            add(cm);
+          }
+        }
+        break;
+    }
+    return found;
   }
 }
 

package/utilities/submit-test-report.js

@@ -11,7 +11,7 @@
  *
  *   node utilities/submit-test-report.js [-server url] [-file path] [-token token] [-header name]
  *
- *   -server  the /testing base; default https://testing.fhir.org/testing
+ *   -server  the /testing base; default https://npm.fhir.org/testing
  *   -file    default test-cases-report.json in the FHIRsmith root
  *   -token   if the server requires one; or set FHIRSMITH_TESTING_TOKEN
  *   -header  the header the token goes in; default Authorization (sent as "Bearer <token>")