fhirsmith 0.18.1 → 1.0.0

package/tx/params.js

@@ -502,6 +502,19 @@ class TxParameters {
     return '';
   }
 
+  // The code system version the request's version rules ask for - force-system-version,
+  // else system-version, else check-system-version - or null when they name none
+  requestedSystemVersion(systemURI) {
+    const rules = this.rulesForSystem(systemURI).filter(t => !t.vs);
+    for (const mode of ['override', 'default', 'check']) {
+      const rule = rules.find(t => t.mode === mode);
+      if (rule) {
+        return rule.version;
+      }
+    }
+    return null;
+  }
+
   rulesForSystem(systemURI) {
     let result = [];
     for (let t of this.FVersionRules) {
@@ -521,6 +534,18 @@ class TxParameters {
     }
   }
 
+  // The languages the request named - displayLanguage, else the HTTP Accept-Language - or
+  // null when it named none (workingLanguages() falls back to a default; this does not)
+  requestedLanguages() {
+    if (this.FHasDisplayLanguages && this.FDisplayLanguages) {
+      return this.FDisplayLanguages;
+    }
+    if (this.FHasHTTPLanguages && this.FHTTPLanguages) {
+      return this.FHTTPLanguages;
+    }
+    return null;
+  }
+
   workingLanguages() {
     if (this.FDisplayLanguages) {
       return this.FDisplayLanguages;

package/tx/problems.js

@@ -11,13 +11,11 @@ class ProblemFinder {
         unknownVersions[vsp.sourcePackage()] = sourceUnknownVersions;
       }
 
-      let list = await vsp.listAllValueSets();
-      for (let url of list) {
-        let vs = await vsp.fetchValueSet(url);
-        if (vs && vs.jsonObj.compose) {
+      await vsp.scanAllValueSets(async (vs) => {
+        if (vs.jsonObj.compose) {
           await this.scanValueSet(vs.jsonObj.compose, sourceUnknownVersions);
         }
-      }
+      });
     }
     let result = '';
     for (let sp of provider.listValueSetSourceCodes()) {

package/tx/provider.js

@@ -1,3 +1,4 @@
+const { chooseImplicitValueSetFactory } = require('./library/implicit-valuesets');
 const { CodeSystem } = require("./library/codesystem");
 const { assertNoActiveContent } = require('../library/narrative-check');
 const {VersionUtilities} = require("../library/version-utilities");
@@ -11,6 +12,31 @@ const {PackageValueSetProvider} = require("./vs/vs-package");
 const ValueSet = require("./library/valueset");
 const {PackageConceptMapProvider} = require("./cm/cm-package");
 const { applyIdPrefix, CORE_PREFIX } = require("./library/resource-ids");
+const { CodeSystemFactoryProvider } = require("./cs/cs-api");
+const { VirtualCodeSystemProvider } = require("./cs/cs-virtual");
+
+
+/**
+ * How many codes resolveCodes() locates at once. Enough to keep a database-backed code
+ * system busy without queueing a whole page's worth of queries at the same time.
+ */
+const RESOLVE_CODES_CONCURRENCY = 8;
+
+/**
+ * fn over items, at most `limit` at a time, with the results in the order of the items.
+ */
+async function mapConcurrently(items, limit, fn) {
+  const results = new Array(items.length);
+  let next = 0;
+  const worker = async () => {
+    while (next < items.length) {
+      const i = next++;
+      results[i] = await fn(items[i], i);
+    }
+  };
+  await Promise.all(Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, worker));
+  return results;
+}
 
 /**
  * This class holds what information is in context
@@ -23,6 +49,86 @@ const { applyIdPrefix, CORE_PREFIX } = require("./library/resource-ids");
  * - a list of concept map providers that can provide concept map instances
  *
  */
+/**
+ * The provider's CodeSystem map (keys url and url|version), with an index of the CodeSystems it
+ * holds by the url they supplement, so that loadSupplements() looks only at those instead of
+ * every loaded CodeSystem on every code-system lookup.
+ *
+ * The index is part of the map, not a cache: set, delete and clear are the only ways to change a
+ * Map, and each updates the index in the same call. Provider.addCodeSystem and deleteCodeSystem
+ * are the only callers that change this map.
+ *
+ * supplementsOf(url) gives every value whose CodeSystem.isSupplementFor(url, version) can be
+ * true, in the map's own order, and possibly some for which it is false; loadSupplements still
+ * asks isSupplementFor of each. isSupplementFor is true only when `supplements` equals url or
+ * starts with url + '|', and then the part of `supplements` before any '|' is the part of url
+ * before any '|': that is the index key.
+ */
+class CodeSystemMap extends Map {
+  constructor(entries) {
+    super();
+    this.supplementIndex = new Map(); // key used by supplementsOf -> Set of map keys
+    this.indexedUnder = new Map();    // map key -> its key in supplementIndex
+    this.position = new Map();        // map key -> insertion number, to keep the map's order
+    this.nextPosition = 0;
+    if (entries) {
+      for (const [k, v] of entries) this.set(k, v);
+    }
+  }
+
+  /** the index key for a url, or for a CodeSystem's supplements value: the part before any '|' */
+  static indexKey(url) {
+    return String(url).split('|')[0];
+  }
+
+  set(k, v) {
+    if (!super.has(k)) {
+      this.position.set(k, this.nextPosition++); // a Map keeps a replaced key where it was
+    }
+    this.#unindex(k);
+    super.set(k, v);
+    const supplements = v && v.jsonObj ? v.jsonObj.supplements : null;
+    if (supplements) {
+      const u = CodeSystemMap.indexKey(supplements);
+      let keys = this.supplementIndex.get(u);
+      if (!keys) this.supplementIndex.set(u, keys = new Set());
+      keys.add(k);
+      this.indexedUnder.set(k, u);
+    }
+    return this;
+  }
+
+  delete(k) {
+    this.#unindex(k);
+    this.position.delete(k);
+    return super.delete(k);
+  }
+
+  clear() {
+    this.supplementIndex.clear();
+    this.indexedUnder.clear();
+    this.position.clear();
+    super.clear();
+  }
+
+  // removes k from the index under the key it was indexed under when it was set
+  #unindex(k) {
+    const u = this.indexedUnder.get(k);
+    if (u === undefined) return;
+    this.indexedUnder.delete(k);
+    const keys = this.supplementIndex.get(u);
+    keys.delete(k);
+    if (keys.size === 0) this.supplementIndex.delete(u);
+  }
+
+  /** the values that may supplement url (any version), in the order values() gives them */
+  supplementsOf(url) {
+    const keys = this.supplementIndex.get(CodeSystemMap.indexKey(url));
+    if (!keys) return [];
+    return [...keys].sort((a, b) => this.position.get(a) - this.position.get(b)).map(k => super.get(k));
+  }
+}
+
 class Provider {
   i18n;
   fhirVersion;
@@ -57,6 +163,10 @@ class Provider {
 
   baseUrl = null;
   path;
+  /**
+   * {EcosystemResolver|null} finds other servers for code systems that aren't held here (modules.tx.ecosystem)
+   */
+  ecosystem = null;
   cacheFolder = null;
   startTime = Date.now();
   startMemory = process.memoryUsage();
@@ -98,7 +208,7 @@ class Provider {
       factory = this.codeSystemFactories.get(vurlMM);
     }
     if (factory != null) {
-      return factory.build(opContext, supplements);
+      return this.buildOnce(opContext, factory, supplements);
     }
     let cs = this.codeSystems.get(vurl);
     if (cs == null && vurlMM) {
@@ -107,7 +217,61 @@ class Provider {
     if (cs != null) {
       return this.createCodeSystemProvider(opContext, cs, supplements);
     }
-    return null;
+    return await this.findInEcosystem(opContext, system, version, supplements);
+  }
+
+  /**
+   * A code system (and version) that isn't held here, provided by another server in the
+   * ecosystem, if modules.tx.ecosystem.use is true, and the request didn't come from
+   * another server doing the same thing (see EcosystemResolver)
+   *
+   * @returns {Promise<VirtualCodeSystemProvider|null>}
+   */
+  async findInEcosystem(opContext, system, version, supplements) {
+    if (!this.ecosystem || opContext.ecosystemForwarded) {
+      return null;
+    }
+    const server = await this.ecosystem.findServer(system, version, this.fhirVersion, opContext);
+    if (!server) {
+      return null;
+    }
+    // the registry's claims are broad (tx.fhir.org is authoritative for all of http://hl7.org/*),
+    // so the server has to say that it actually has the code system - and the version: the one
+    // asked for, or, for a version that is a url, the latest release of it (a SNOMED CT edition
+    // asked for without a release date). A partial semver is not completed this way
+    const versions = await this.ecosystem.listVersions(server, system, opContext);
+    if (versions == null) {
+      return null;
+    }
+    const codes = versions.map(v => v.code);
+    let actual = version;
+    if (!version) {
+      // no version asked for: it's the server's default version - the only version it has, or
+      // the one it says is the default
+      const defaults = versions.length === 1 ? versions : versions.filter(v => v.isDefault);
+      if (defaults.length !== 1) {
+        throw new TerminologyError(`Unable to use ${server} for ${system}: its TerminologyCapabilities doesn't say which version of it is the default`);
+      }
+      actual = defaults[0].code;
+    } else if (!codes.includes(version)) {
+      const matches = codes.filter(v => v.startsWith(version + '/')).sort();
+      if (matches.length === 0) {
+        return null;
+      }
+      actual = matches[matches.length - 1];
+    }
+    opContext.log('Using ' + server + ' for ' + system + (actual ? '|' + actual : ''));
+    return new VirtualCodeSystemProvider(opContext, supplements, server, system, actual, this.ecosystem);
+  }
+
+  /**
+   * factory.build(), once per operation, factory and set of supplements (see
+   * OperationContext.buildProviderOnce). The caller registers the provider for closing.
+   */
+  buildOnce(opContext, factory, supplements) {
+    // the factory is the code system and version; the supplements make a different provider
+    const variant = (supplements || []).map(cs => cs.vurl || cs.url).sort().join(' ');
+    return opContext.buildProviderOnce(factory, variant, () => factory.build(opContext, supplements));
   }
 
   loadSupplements(url, version, statedSupplements) {
@@ -115,9 +279,12 @@ class Provider {
     for (let csp of this.codeSystemFactories.values()) {
       csp.listSupplements(supplements, url, version, statedSupplements);
     }
-    for (let cs of this.codeSystems.values()) {
+    // the index gives the same candidates in the same order as values(), less the ones that
+    // can't be supplements of url (see CodeSystemMap)
+    const candidates = this.codeSystems instanceof CodeSystemMap ? this.codeSystems.supplementsOf(url) : this.codeSystems.values();
+    for (let cs of candidates) {
       if (cs.isSupplementFor(url, version)) {
-        if (cs.isLangPack() || this.isStatedSupplement(cs, statedSupplements)) {
+        if (cs.isGlobalSupplement() || this.isStatedSupplement(cs, statedSupplements)) {
           supplements.set(cs.vurl, cs);
         }
       }
@@ -161,9 +328,12 @@ class Provider {
    * share ids, and so does other content (hl7.terminology), so no endpoint can serve them on
    * their own ids (see tx/library/resource-ids.js)
    *
+   * @param {Object} [options]
+   * @param {boolean} [options.readOnly] - a tx-worker (cluster.md): the package's value set and
+   *   concept map caches are the master's, opened read-only (see Library.packageCachesReadOnly)
    * @returns {Promise<PackageContentLoader|undefined>} the loaded package (mode "npm")
    */
-  async loadNpm(packageManager, cacheFolder,  details, isDefault, mode) {
+  async loadNpm(packageManager, cacheFolder,  details, isDefault, mode, options = {}) {
     // Parse packageId and version from details (e.g., "hl7.terminology.r4#6.0.2")
     let packageId = details;
     let version = null;
@@ -210,11 +380,11 @@ class Provider {
       applyIdPrefix(cs, CORE_PREFIX);
       this.addCodeSystem(cs);
     }
-    const vs = new PackageValueSetProvider(contentLoader);
+    const vs = new PackageValueSetProvider(contentLoader, { readOnly: !!options.readOnly });
     await vs.initialize();
     vs.assignIds(CORE_PREFIX);
     this.valueSetProviders.push(vs);
-    const cm = new PackageConceptMapProvider(contentLoader);
+    const cm = new PackageConceptMapProvider(contentLoader, { readOnly: !!options.readOnly });
     await cm.initialize();
     cm.assignIds(CORE_PREFIX);
     this.conceptMapProviders.push(cm);
@@ -254,7 +424,7 @@ class Provider {
     return null;
   }
 
-  async findValueSet(opContext, url, version) {
+  async findValueSet(opContext, url, version, systemVersionFor = null) {
     for (const vp of this.valueSetProviders) {
       if (opContext) opContext.deadCheck('findValueSet');
       let vs = await vp.fetchValueSet(url, version);
@@ -262,7 +432,7 @@ class Provider {
         return vs;
       }
     }
-    let vs1 = await this.findKnownValueSet(url, version);
+    let vs1 = await this.findKnownValueSet(url, version, systemVersionFor);
     return vs1;
   }
 
@@ -325,7 +495,18 @@ class Provider {
     return result;
   }
 
-  async findKnownValueSet(url, version) {
+  async findKnownValueSet(url, version, systemVersionFor = null) {
+    // A code system with several versions loaded (SNOMED CT editions) says which form of
+    // implicit value set url is: the generic form comes from the version the request asks
+    // for (systemVersionFor, from system-version and the like) or else the default version
+    // (the one registered under the bare system), the edition form from the latest version
+    // of that edition, and the version form from that version alone - not from whichever
+    // version happens to be asked first.
+    const chosen = chooseImplicitValueSetFactory(this.codeSystemFactories.values(), system => this.codeSystemFactories.get(system), url, systemVersionFor);
+    if (chosen) {
+      const vs = await chosen.buildKnownValueSet(url, version);
+      return vs != null ? new ValueSet(vs) : null;
+    }
     for (let csp of this.codeSystemFactories.values()) {
       let vs = await csp.buildKnownValueSet(url, version);
       if (vs != null) {
@@ -351,10 +532,17 @@ class Provider {
       const targetBase = targetSystem.split('|')[0];
       let uris = new Set();
       let asked = new Set(); // a factory is registered under several keys (system, system|version...)
+      // A source version that names a version without a date (a SNOMED CT edition URI, say)
+      // matches every version of it the server holds; it means the one it resolves to, as it
+      // does everywhere else - asking each would give the same translation once per release
+      const chosen = sourceVersion ? this.codeSystemFactories.get(sourceSystem + '|' + sourceVersion) : null;
       for (let csp of this.codeSystemFactories.values()) {
         if (asked.has(csp)) {
           continue;
         }
+        if (chosen && csp !== chosen && csp.system() === sourceSystem) {
+          continue;
+        }
         if (csp.system() === sourceSystem || csp.system() === targetBase || !uris.has(csp.system())) {
           uris.add(csp.system());
           asked.add(csp);
@@ -441,10 +629,31 @@ class Provider {
   }
 
   async resolveCode(opContext, system, version, code) {
+    validateParameter(code, "code", String);
+    return (await this.resolveCodes(opContext, system, version, [code]))[0];
+  }
+
+  /**
+   * resolveCode() for many codes of one code system: where each code lives (a link) and
+   * its display, or null, in the order of the codes.
+   *
+   * The code system is found and its provider built once for all of them, and the codes
+   * are located RESOLVE_CODES_CONCURRENCY at a time rather than one after another.
+   *
+   * @param {OperationContext} opContext
+   * @param {string} system
+   * @param {string|null} version
+   * @param {string[]} codes
+   * @returns {Promise<Array<{link: string, description: string}|null>>}
+   */
+  async resolveCodes(opContext, system, version, codes) {
     validateParameter(opContext, "opContext", OperationContext);
     validateParameter(system, "system", String);
     validateOptionalParameter(version, "version", String);
-    validateParameter(code, "code", String);
+    validateArrayParameter(codes, "codes", String);
+    if (codes.length === 0) {
+      return [];
+    }
 
     if (system.includes("|")) {
       const url = system.substring(0, system.indexOf("|"));
@@ -464,41 +673,68 @@ class Provider {
     if (factory == null && vurlMM) {
       factory = this.codeSystemFactories.get(vurlMM);
     }
-    if (factory != null) {
-      const csp = await factory.build(opContext, []);
-      opContext.registerProvider(csp);
-      const c = csp ? csp.locate(code) : null;
-      if (c) {
-        if (factory.iteratable()) {
-          return {
-            link: this.path + "/CodeSystem/x-" + factory.id(),
-            description: csp.display(c)
-          }
-        } else {
-           const link = factory.codeLink(c);
-           if (link) {
-             return {
-               link: link,
-               description: csp.display(c)
-             }
-           }
-        }
-      }
-    }
+    // A code system from a factory gives a code a link only if the factory is iteratable
+    // (US states) or has a codeLink() of its own. For the rest (RxNorm, LOINC, SNOMED CT,
+    // ...) what locate() finds would be thrown away, so they are not built or asked.
+    // That matters: RxNorm looks for a code it does not have in its archive table, which
+    // takes 36 ms a code, and 4,529 of Medication Brand Name's codes are not current.
+    const linksCodes = factory != null
+      && (factory.iteratable() || factory.codeLink !== CodeSystemFactoryProvider.prototype.codeLink);
+    const csp = linksCodes ? await this.resolveCodeProvider(opContext, factory) : null;
     let cs = this.codeSystems.get(vurl);
     if (cs == null && vurlMM) {
       cs = this.codeSystems.get(vurlMM);
     }
-    if (cs != null) {
-      const c = cs.codeMap.get(code);
-      if (c) {
-        return {
-          link: this.path + "/CodeSystem/" + cs.id + '#' + code,
-          description: c.display
-        };
+
+    return mapConcurrently(codes, RESOLVE_CODES_CONCURRENCY, async (code) => {
+      if (csp) {
+        const located = await csp.locate(code);
+        const c = located ? located.context : null;
+        if (c) {
+          if (factory.iteratable()) {
+            return {
+              link: this.path + "/CodeSystem/x-" + factory.id(),
+              description: await csp.display(c)
+            }
+          } else {
+             const link = factory.codeLink(c);
+             if (link) {
+               return {
+                 link: link,
+                 description: await csp.display(c)
+               }
+             }
+          }
+        }
       }
-    }
-    return null;
+      if (cs != null) {
+        const c = cs.codeMap.get(code);
+        if (c) {
+          return {
+            link: this.path + "/CodeSystem/" + cs.id + '#' + code,
+            description: c.display
+          };
+        }
+      }
+      return null;
+    });
+  }
+
+  /**
+   * The provider resolveCode() uses for a code system factory in this operation: built
+   * the first time it is asked for (see buildOnce), registered for closing when the
+   * operation ends, and reused after that. A page asks resolveCode about every code it shows.
+   *
+   * @param {OperationContext} opContext
+   * @param {CodeSystemFactoryProvider} factory
+   * @returns {Promise<CodeSystemProvider|null>}
+   */
+  resolveCodeProvider(opContext, factory) {
+    // the same provider getCodeSystemProvider() builds for this code system with no supplements
+    return this.buildOnce(opContext, factory, []).then(p => {
+      opContext.registerProvider(p);
+      return p;
+    });
   }
 
   /**
@@ -632,27 +868,20 @@ class Provider {
    * moved from the FHIR core packages to terminology.hl7.org has been QA'd there since, and
    * its versions went backwards (coverage-selfpay is 4.0.1 in hl7.fhir.r4.core but 1.0.1 in
    * hl7.terminology), so version precedence can't decide between them. Instead, a code system
-   * from an hl7.terminology package replaces every code system with the same url from a FHIR
-   * core package - none of the core versions stay available.
+   * from an hl7.terminology package is always the default for its url - what a reference with
+   * no version gets - whatever versions the FHIR core packages have.
    *
-   * Except: hl7.fhir.r4.core has the v2 tables 0006, 0360 and 0391 in two versions each,
-   * with the version in the url (http://terminology.hl7.org/CodeSystem/v2-0006|2.1 - see
-   * urlHadVersion in loadNpm). hl7.terminology doesn't publish those versions, so they stay
-   * available by url|version; THO's code system is the default for the url.
+   * The core versions stay available by url|version, where THO doesn't provide that version
+   * itself: a reference that names its version (url|5.0.0) means the core one, and there's
+   * nothing else it can mean. They are never the default while THO provides the url (see also
+   * deleteCodeSystem).
    */
   addCodeSystem(cs) {
     if (Provider.#isTHOPackage(cs.sourcePackage)) {
-      for (const [key, t] of [...this.codeSystems]) {
-        if (t.url === cs.url && VersionUtilities.isCorePackage(t.sourcePackage)
-          && (key === t.url || !t.urlHadVersion)) {
-          this.codeSystems.delete(key);
-        }
-      }
-    } else if (VersionUtilities.isCorePackage(cs.sourcePackage) && !cs.urlHadVersion) {
-      // THO's code system is always the default for its url
+      // THO's code system takes over the default from a core one; the core versions stay
       const t = this.codeSystems.get(cs.url);
-      if (t && Provider.#isTHOPackage(t.sourcePackage)) {
-        return;
+      if (t && t.url === cs.url && VersionUtilities.isCorePackage(t.sourcePackage)) {
+        this.codeSystems.delete(cs.url);
       }
     }
     const existing = this.codeSystems.get(cs.url);
@@ -677,45 +906,63 @@ class Provider {
    * THO precedence for value sets and concept maps, as for code systems (see addCodeSystem):
    * once the library's providers have been added after this endpoint's core package, the core
    * package's value sets and concept maps whose urls an hl7.terminology package also provides
-   * are left out. (No core value set or concept map has the version in its url.)
+   * are only available to a reference that names their version (and THO doesn't provide that
+   * version itself). They are never what a reference with no version gets, and they aren't
+   * read by id, found by search, or used for translation unless asked for.
    */
   applyTHOPrecedence() {
-    const thoValueSets = new Set();
+    const thoValueSets = new Map();
     for (const vp of this.valueSetProviders) {
       if (vp instanceof PackageValueSetProvider && Provider.#isTHOPackage(vp.sourcePackage())) {
         for (const vs of vp.valueSetMap.values()) {
-          thoValueSets.add(vs.url);
+          Provider.#addVersion(thoValueSets, vs);
         }
       }
     }
-    const thoConceptMaps = new Set();
+    const thoConceptMaps = new Map();
     for (const cmp of this.conceptMapProviders) {
       if (cmp instanceof PackageConceptMapProvider && Provider.#isTHOPackage(cmp.sourcePackage())) {
         for (const cm of cmp.conceptMapMap.values()) {
-          thoConceptMaps.add(cm.url);
+          Provider.#addVersion(thoConceptMaps, cm);
         }
       }
     }
     for (const vp of this.valueSetProviders) {
       if (vp instanceof PackageValueSetProvider && VersionUtilities.isCorePackage(vp.sourcePackage())) {
-        vp.excludeUrls(thoValueSets);
+        vp.keepPinnedOnly(thoValueSets);
       }
     }
     for (const cmp of this.conceptMapProviders) {
       if (cmp instanceof PackageConceptMapProvider && VersionUtilities.isCorePackage(cmp.sourcePackage())) {
-        cmp.excludeUrls(thoConceptMaps);
+        cmp.keepPinnedOnly(thoConceptMaps);
       }
     }
   }
 
+  static #addVersion(map, resource) {
+    if (!resource || !resource.url) {
+      return;
+    }
+    if (!map.has(resource.url)) {
+      map.set(resource.url, new Set());
+    }
+    if (resource.version) {
+      map.get(resource.url).add(resource.version);
+    }
+  }
+
   deleteCodeSystem(cs) {
     this.codeSystems.delete(cs.vurl);
     this.codeSystems.delete(cs.url);
     // If other versions of the SAME url survive, re-point the unversioned [url]
     // entry at the most-recent surviving version. Otherwise leave it deleted.
+    // THO precedence (see addCodeSystem): while a THO version survives, a core one
+    // can't become the default, however recent its (reset) version looks
+    const survivors = [...new Set(this.codeSystems.values())].filter(t => t.url === cs.url);
+    const tho = survivors.filter(t => Provider.#isTHOPackage(t.sourcePackage));
     let existing = null;
-    for (let t of this.codeSystems.values()) {
-      if (t.url === cs.url && (!existing || t.isMoreRecent(existing))) {
+    for (let t of (tho.length > 0 ? tho : survivors)) {
+      if (!existing || t.isMoreRecent(existing)) {
         existing = t;
       }
     }
@@ -726,4 +973,4 @@ class Provider {
 
 }
 
-module.exports = { Provider };
+module.exports = { CodeSystemMap, Provider };