fhirsmith 0.17.0 → 0.18.0

package/tx/vs/vs-vsac.js

@@ -26,7 +26,7 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
    * @param {string} config.apiKey - API key for VSAC authentication
    * @param {string} config.cacheFolder - Local folder for cached database
    * @param {number} [config.refreshIntervalHours=24] - Hours between refresh scans
-   * @param {string} [config.baseUrl='http://cts.nlm.nih.gov/fhir'] - Base URL for VSAC FHIR server
+   * @param {string} [config.baseUrl='https://cts.nlm.nih.gov/fhir'] - Base URL for VSAC FHIR server
    * @param {number} [config.timeoutMs=120000] - HTTP request timeout in milliseconds
    * @param {string} [config.resyncPassword] - If set, enables the operator "resync a ValueSet"
    *   form on the /info page, gated by this password. If unset, the form is not offered.
@@ -43,7 +43,12 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
     // Optional operator password for the on-demand resync form (see info()).
     this.resyncPassword = config.resyncPassword || null;
     this.cacheFolder = folders.ensureFilePath("terminology-cache/vsac");
-    this.baseUrl = config.baseUrl || 'http://cts.nlm.nih.gov/fhir';
+    // https: every request carries the API key. (VSAC redirects http to https, but only after
+    // the key has been sent in the clear)
+    this.baseUrl = (config.baseUrl || 'https://cts.nlm.nih.gov/fhir').replace(/\/+$/, '');
+    if (this.baseUrl.startsWith('http://')) {
+      console.warn(`VSAC: baseUrl ${this.baseUrl} is http, so the API key is sent unencrypted - use https`);
+    }
     this.refreshIntervalHours = config.refreshIntervalHours || 24;
 
     this.dbPath = path.join(this.cacheFolder, 'vsac-valuesets.db');
@@ -397,10 +402,24 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
       return null;
     }
 
-    // Extract relative path from full URL
-    let s = nextLink.url;
-    s = s.replace(this.baseUrl, '');
-    return s;
+    // Followed only on the VSAC server itself, as a path under baseUrl: the request carries the
+    // API key, so a next link must not be able to send it anywhere else. The scheme is ignored -
+    // VSAC's links can say http - and the request goes out on baseUrl's
+    let next;
+    let base;
+    try {
+      base = new URL(this.baseUrl);
+      next = new URL(nextLink.url, this.baseUrl + '/');
+    } catch (e) {
+      console.warn(`VSAC: ignoring next link ${String(nextLink.url).substring(0, 200)}: ${e.message}`);
+      return null;
+    }
+    const basePath = base.pathname.replace(/\/+$/, '');
+    if (next.host !== base.host || !(next.pathname === basePath || next.pathname.startsWith(basePath + '/'))) {
+      console.warn(`VSAC: ignoring next link ${String(nextLink.url).substring(0, 200)}: it is not on ${this.baseUrl}`);
+      return null;
+    }
+    return next.pathname.substring(basePath.length) + next.search;
   }
 
   /**

package/tx/workers/expand.js

@@ -458,6 +458,19 @@ class ValueSetExpander {
           }
         }
 
+        // properties the code system always reports (ICD-11's classificationCode), unless
+        // the request asked for them anyway and the loops here add them
+        if (csProps != null && cs != null && !this.params.properties.includes('*')) {
+          for (const pn of cs.mandatoryExpansionProperties()) {
+            if (!this.params.properties.includes(pn)) {
+              for (const cp of csProps.filter(p => p.code === pn)) {
+                const vn = getValueName(cp);
+                this.defineProperty(expansion, n, this.getPropUrl(cs, pn, cp), pn, vn, cp[vn], true);
+              }
+            }
+          }
+        }
+
         for (const pn of this.params.properties) {
           if (pn === '*') {
             continue; // done above
@@ -890,8 +903,9 @@ class ValueSetExpander {
             const cctxt = await cs.locate(cc.code, this.allAltCodes);
             if (cctxt && cctxt.context && (!this.params.activeOnly || !await cs.isInactive(cctxt.context)) && await this.passesFilters(cs, cctxt.context, prep, filters, 0)) {
               let added;
+              const ecode = await cs.expansionCode(cctxt.context, cc.code);
               if (this.noDetails) {
-                added = await this.includeCode(cs, null, cs.system(), cs.version(), cc.code, await cs.isAbstract(cctxt.context), await cs.isInactive(cctxt.context), null, null, cds,
+                added = await this.includeCode(cs, null, cs.system(), cs.version(), ecode, await cs.isAbstract(cctxt.context), await cs.isInactive(cctxt.context), null, null, cds,
                     null, null, expansion, valueSets, null, null, null, null, excludeInactive, vsSrc.url);
               } else {
                 await this.listDisplaysFromProvider(cds, cs, cctxt.context);
@@ -901,7 +915,7 @@ class ValueSetExpander {
                   if (!ov) {
                     ov = await cs.itemWeight(cctxt.context);
                   }
-                  added = await this.includeCode(cs, null, cs.system(), cs.version(), cc.code, await cs.isAbstract(cctxt.context), await cs.isInactive(cctxt.context), await cs.isDeprecated(cctxt.context), await cs.getStatus(cctxt.context), cds,
+                  added = await this.includeCode(cs, null, cs.system(), cs.version(), ecode, await cs.isAbstract(cctxt.context), await cs.isInactive(cctxt.context), await cs.isDeprecated(cctxt.context), await cs.getStatus(cctxt.context), cds,
                       await cs.definition(cctxt.context), ov, expansion, valueSets, await cs.extensions(cctxt.context), cc.extension, await cs.properties(cctxt.context), null, excludeInactive, vsSrc.url);
                 }
                 if (added) {
@@ -939,16 +953,29 @@ class ValueSetExpander {
           }
 
           const fset = await cs.executeFilters(prep);
-          if (await cs.filtersNotClosed(prep)) {
+          const filtersOpen = await cs.filtersNotClosed(prep);
+          if (filtersOpen) {
             notClosed.value = true;
-          } else if (fset.length === 1 && !excludeInactive && !this.params.activeOnly) {
-            // this.addToTotal(await cs.filterSize(prep, fset[0]));
+          }
+
+          // count=0 asks only for the total. When this include is the whole value set and its one
+          // filter set is the whole answer - nothing else to add, overlap, exclude or skip - the
+          // total is the set's size, and walking it would only end in VALUESET_TOO_COSTLY once
+          // the set is bigger than the limit. Anything less certain is counted by walking, as
+          // before: an earlier version added filterSize *and* walked, and double counted (#91, #93).
+          const countOnly = this.count === 0 && !filtersOpen && fset.length === 1 && filter.isNull
+            && !excludeInactive && !this.params.activeOnly && !this.hasExclusions
+            && !(cset.concept && cset.concept.length) && !(cset.valueSet && cset.valueSet.length)
+            && (compose.include || []).length === 1 && (compose.exclude || []).length === 0;
+          if (countOnly) {
+            this.worker.opContext.log('count only: the filter size is the total');
+            this.addToTotal(await cs.filterSize(prep, fset[0]));
           }
 
           this.worker.opContext.log('iterate filters');
           this.addToTotal(0);
           const cds = new Designations(this.worker.i18n.languageDefinitions);
-          while (await cs.filterMore(prep, fset[0])) {
+          while (!countOnly && await cs.filterMore(prep, fset[0])) {
             await this.worker.checkAndYield('processCodes#5');
             const c = await cs.filterConcept(prep, fset[0]);
             const ok = (!this.params.activeOnly || !await cs.isInactive(c)) && (await this.passesFilters(cs, c, prep, fset, 1));

package/tx/workers/lookup.js

@@ -15,6 +15,7 @@ const {Parameters} = require("../library/parameters");
 const { Issue, OperationOutcome, buildOperationOutcome, outcomeFromError } = require('../library/operation-outcome');
 const {debugLog} = require("../operation-context");
 const {checkAddedParameters} = require("../library/parameter-check");
+const { Languages } = require("../../library/languages");
 
 class LookupWorker extends TerminologyWorker {
   /**
@@ -204,7 +205,7 @@ class LookupWorker extends TerminologyWorker {
       const supplements = this.loadSupplements(codeSystem.url, codeSystem.version, txp.supplements);
 
       // Create a FhirCodeSystemProvider for this CodeSystem
-      const csProvider = new FhirCodeSystemProvider(this.opContext, codeSystem, supplements);
+      const csProvider = FhirCodeSystemProvider.create(this.opContext, codeSystem, supplements);
 
       // Perform the lookup
       const result = await this.doLookup(csProvider, code, txp);
@@ -247,6 +248,11 @@ class LookupWorker extends TerminologyWorker {
       );
     };
 
+    // Explicitly asked for: named in the property list, or '*'. An empty list is not asking for
+    // anything in particular (the server chooses), which is not the same thing for abstract and
+    // notSelectable - see below.
+    const asked = (name) => (params.properties || []).some(p => p === '*' || p.toLowerCase() === name.toLowerCase());
+
     // Locate the code in the code system
     const locateResult = await csProvider.locate(code);
 
@@ -271,7 +277,7 @@ class LookupWorker extends TerminologyWorker {
     // name (required)
     responseParams.push({
       name: 'name',
-      valueString: csProvider.name(params.workingLanguages())
+      valueString: csProvider.lookupName(params.workingLanguages())
     });
     responseParams.push({
       name: 'code',
@@ -303,6 +309,32 @@ class LookupWorker extends TerminologyWorker {
       valueString: disp || display || code
     });
 
+    // A displayLanguage the code system has no display in, for this concept, still gets an
+    // answer - in the default language - but the client is told so: a display in a language it
+    // did not ask for, offered without comment, reads as an answer in the language it did.
+    // Only for an explicit displayLanguage: Accept-Language is a browser's standing preference,
+    // not a request about this code.
+    if (params.hasDisplayLanguages) {
+      // the languages actually asked for: displayLanguage is parsed with a wildcard appended,
+      // and the wildcard matches whatever display there is
+      const langs = new Languages(this.languages);
+      for (const l of params.workingLanguages()) {
+        if (l.code !== '*') langs.add(l);
+      }
+      const covered = langs.length === 0 || (designations.designations.length > 0
+        ? designations.hasAnyDisplays(langs)
+        : csProvider.hasAnyDisplays(langs));
+      if (!covered) {
+        const used = (pd && pd.language) ? String(pd.language) : (csProvider.defLang() || 'en');
+        const csDesc = version ? csProvider.system() + '|' + version : csProvider.system();
+        responseParams.push({
+          name: 'warning',
+          valueString: this.i18n.translate('LOOKUP_DISPLAY_LANGUAGE_FALLBACK', params.HTTPLanguages,
+            [langs.asString(false), code, csDesc, used])
+        });
+      }
+    }
+
     // definition (optional) - top-level parameter
     if (hasProp('definition', true)) {
       const definition = await csProvider.definition(ctxt);
@@ -314,9 +346,12 @@ class LookupWorker extends TerminologyWorker {
       }
     }
 
-    // abstract property (optional)
-    if (hasProp('abstract', true)) {
-      const isAbstract = await csProvider.isAbstract(ctxt);
+    // abstract and notSelectable (agreed for the tx-ecosystem tests, October 2026): a concept that
+    // is not selectable always says so, both ways, whatever properties were asked for - that is
+    // the fact a client most needs about it. A selectable one says neither unless asked.
+    // notSelectable is reconciled after the provider has added its properties, below.
+    const isAbstract = await csProvider.isAbstract(ctxt);
+    if (isAbstract || asked('abstract')) {
       responseParams.push({
         name: 'abstract',
         valueBoolean: isAbstract
@@ -389,7 +424,7 @@ class LookupWorker extends TerminologyWorker {
     // primitive belongs is a provider bug, and shipping it would produce a
     // response that isn't valid FHIR (and, in XML, isn't even parseable).
     const providerParamsFrom = responseParams.length;
-    await csProvider.extendLookup(ctxt, params.properties || [], responseParams);
+    await csProvider.extendLookup(ctxt, params.properties || [], responseParams, params);
     let providerName;
     try {
       providerName = csProvider.system();
@@ -398,6 +433,26 @@ class LookupWorker extends TerminologyWorker {
     }
     checkAddedParameters(responseParams, providerParamsFrom, providerName);
 
+    // notSelectable, the property: present (true) for every concept that is not selectable;
+    // for a selectable one only when asked for (false), and taken out if a provider added it
+    // anyway
+    const isNotSelectable = (p) => p.name === 'property' &&
+      (p.part || []).some(x => x.name === 'code' && x.valueCode === 'notSelectable');
+    const nsAt = responseParams.findIndex(isNotSelectable);
+    if (isAbstract) {
+      if (nsAt < 0) {
+        responseParams.push({ name: 'property', part: [
+          { name: 'code', valueCode: 'notSelectable' }, { name: 'value', valueBoolean: true }] });
+      }
+    } else if (asked('notSelectable')) {
+      if (nsAt < 0) {
+        responseParams.push({ name: 'property', part: [
+          { name: 'code', valueCode: 'notSelectable' }, { name: 'value', valueBoolean: false }] });
+      }
+    } else if (nsAt >= 0) {
+      responseParams.splice(nsAt, 1);
+    }
+
     if (reportedSupplements) {
       for (const supplement of reportedSupplements) {
         responseParams.push({

package/tx/workers/subsumes.js

@@ -177,7 +177,7 @@ class SubsumesWorker extends TerminologyWorker {
     const supplements = this.loadSupplements(codeSystem.url, codeSystem.version, txp.supplements);
 
     // Create a FhirCodeSystemProvider for this CodeSystem
-    const csProvider = new FhirCodeSystemProvider(this.opContext, codeSystem, supplements);
+    const csProvider = FhirCodeSystemProvider.create(this.opContext, codeSystem, supplements);
 
     // Get the codings
     let codingA, codingB;

package/tx/workers/translate.js

@@ -26,6 +26,10 @@ function elementComment(em) {
   return em.comment || Extensions.readString(em, EXT_ELEMENT_COMMENT);
 }
 
+// marks a match whose target came from the code system itself (translateUsingCodeSystem), so
+// checkTargetConcepts leaves it alone. A symbol key, so it never reaches the JSON output
+const TRUSTED_TARGET = Symbol('trustedTarget');
+
 class TranslateWorker extends TerminologyWorker {
   /**
    * @param {OperationContext} opContext - Operation context
@@ -281,10 +285,10 @@ class TranslateWorker extends TerminologyWorker {
     if (conceptMaps.length == 0) {
       if (reverse) {
         await this.findConceptMapsInAdditionalResources(conceptMaps,targetSystem, targetScope, sourceScope, coding.system);
-        await this.provider.findConceptMapForTranslation(this.opContext, conceptMaps, targetSystem, targetScope, sourceScope, coding.system, coding.code);
+        await this.provider.findConceptMapForTranslation(this.opContext, conceptMaps, targetSystem, null, targetScope, sourceScope, coding.system, coding.code);
       } else {
         await this.findConceptMapsInAdditionalResources(conceptMaps, coding.system, sourceScope, targetScope, targetSystem);
-        await this.provider.findConceptMapForTranslation(this.opContext, conceptMaps, coding.system, sourceScope, targetScope, targetSystem, coding.code);
+        await this.provider.findConceptMapForTranslation(this.opContext, conceptMaps, coding.system, coding.version, sourceScope, targetScope, targetSystem, coding.code);
       }
     }
 
@@ -390,7 +394,7 @@ class TranslateWorker extends TerminologyWorker {
     }
 
     // Perform the translation
-    const result = await this.doTranslate(scoped.maps, coding, targetScope, targetSystem, params);
+    const result = await this.doTranslate(scoped.maps, coding, targetScope, targetSystem, txp);
     return res.status(200).json(result);
   }
 
@@ -419,7 +423,11 @@ class TranslateWorker extends TerminologyWorker {
     // speak for another group, and does not suppress another group's fallback.
     for (const g of cm.listGroupsInScope(coding, targetScope, targetSystem)) {
       let groupResult = false;
-      for (const em of (g.element || []).filter(e => e.code === coding.code)) {
+      // the code as given, then its normal form (see sourceCodes): an element written in the
+      // normal form matches a code given in another form, but not the other way round
+      const codes = this.sourceCodes(coding);
+      const elements = codes.flatMap(c => (g.element || []).filter(e => e.code === c));
+      for (const em of elements) {
         // ConceptMap.group.element.noMap says, positively, that this concept has no
         // mapping in this map - which is an answer, not the absence of one. Report it
         // as a match that states the source concept and the map it came from, with no
@@ -871,11 +879,19 @@ class TranslateWorker extends TerminologyWorker {
 
     let translations = await prov.getTranslations(cm, coding, target, reverse);
 
+    // The code system can say why there is no translation (an extension concept whose only
+    // International ancestor is a top-level hierarchy, say): that comes back as an entry
+    // with no code, and goes out as the message in place of the generic one
+    const explanations = translations.filter(t => !t.code && t.message);
+    translations = translations.filter(t => t.code);
+
     if (translations.length > 0) {
       result = true;
+      const usedMaps = new Set();
 
       for (const t of translations) {
-        if (t.map) {
+        if (t.map && !usedMaps.has(t.map)) {
+          usedMaps.add(t.map);
           output.push({
             name: 'used-conceptmap',
             valueUri: t.map
@@ -889,6 +905,15 @@ class TranslateWorker extends TerminologyWorker {
           display: t.display
         };
 
+        const sourceConcept = {
+          system: coding.system,
+          code: coding.code
+        };
+        if (coding.version) {
+          // as given: the version says which edition the code came from, which is
+          // the point of an edition to edition translation
+          sourceConcept.version = coding.version;
+        }
         const matchParts = [];
         matchParts.push({
           name: 'concept',
@@ -896,10 +921,7 @@ class TranslateWorker extends TerminologyWorker {
         });
         matchParts.push({
           name: 'sourceConcept',
-          valueCoding: {
-            system: coding.system,
-            code: coding.code
-          }
+          valueCoding: sourceConcept
         });
         matchParts.push({
           name: 'relationship',
@@ -911,13 +933,26 @@ class TranslateWorker extends TerminologyWorker {
             valueString: t.message
           });
         }
-        matchParts.push({
-          name: 'originMap',
-          valueCanonical: cm.vurl
-        });
+        if (cm.url) {
+          matchParts.push({
+            name: 'originMap',
+            valueCanonical: cm.vurl
+          });
+        }
+        // the code system produced this target itself, from its own content (and for an
+        // edition to edition translation that content may be a version the server does not
+        // hold separately), so there is nothing to check it against: see checkTargetConcepts
         output.push({
           name: 'match',
-          part: matchParts
+          part: matchParts,
+          [TRUSTED_TARGET]: true
+        });
+      }
+    } else {
+      for (const e of explanations) {
+        output.push({
+          name: 'message',
+          valueString: e.message
         });
       }
     }
@@ -1075,6 +1110,9 @@ class TranslateWorker extends TerminologyWorker {
       // across groups - one group having mapped the concept does not stop another group
       // from saying what it does with it.
       const maps = this.sortConceptMaps(conceptMaps);
+      if (!reverse) {
+        await this.noteSourceNormalForm(coding, params);
+      }
       let added = false;
       for (const cm of maps) {
         if (cm.jsonObj.internalSource) {
@@ -1089,12 +1127,13 @@ class TranslateWorker extends TerminologyWorker {
       // or R4's unmatched/disjoint). That is still reported as a match - it is what the map
       // says - but it is not a translation: "The value can only be true if at least one
       // returned match has a relationship other than 'not-related-to'" (tx-ecosystem translate-2b)
+      await this.checkTargetConcepts(result, params);
       const translated = added && result.some(p => p.name === 'match' && this.isPositiveMatch(p));
       result.push({
         name: 'result',
         valueBoolean: translated
       });
-      if (!added) {
+      if (!added && !result.some(p => p.name === 'message')) {
         result.push({
           name: 'message',
           valueString: 'No translations found'
@@ -1125,6 +1164,118 @@ class TranslateWorker extends TerminologyWorker {
     };
   }
 
+  /**
+   * What the server can say about a code: whether its code system is known, whether the code
+   * is valid there (undefined when the code system's content cannot tell - a fragment), and -
+   * when it is - its normal form, as the code system has it (the same
+   * normal form $validate-code reports as normalized-code) and its display. A code system
+   * (or version) the server cannot resolve is not known.
+   *
+   * @returns {Promise<{known: boolean, valid?: boolean, normal?: string, display?: string}>}
+   */
+  async resolveCode(system, version, code, params) {
+    if (!system || !code) {
+      return { known: false };
+    }
+    this.deadCheck('translate-resolve');
+    let cs;
+    try {
+      cs = await this.findCodeSystem(system, version || '', params, ['complete', 'fragment'], null, true);
+    } catch (error) {
+      if (error instanceof Issue) {
+        return { known: false }; // an unknown version, say
+      }
+      throw error;
+    }
+    if (!cs) {
+      return { known: false };
+    }
+    const loc = await cs.locate(code);
+    if (!loc || !loc.context) {
+      // a fragment, or a code system without its content, cannot say a code is not in it
+      const mode = cs.contentMode ? String(cs.contentMode() || 'complete') : 'complete';
+      return mode === 'complete' ? { known: true, valid: false } : { known: true };
+    }
+    const normal = await cs.code(loc.context);
+    return { known: true, valid: true, normal: normal || code, display: await cs.display(loc.context) };
+  }
+
+  /** The normal form of a code, with its display; null when it is the code as given, or unknown. */
+  async normalForm(system, version, code, params) {
+    const r = await this.resolveCode(system, version, code, params);
+    if (!r.valid || r.normal === code) {
+      return null;
+    }
+    return { code: r.normal, display: r.display };
+  }
+
+  /**
+   * The source code may not be in its code system's normal form (an ICD-11 expression written
+   * with short codes, say). A map element written in the normal form still applies to it, so
+   * translation looks for the code as given first, and then for its normal form. The map's own
+   * codes are not normalised: an element written in some other form matches only that code.
+   */
+  async noteSourceNormalForm(coding, params) {
+    const normal = await this.normalForm(coding.system, coding.version, coding.code, params);
+    this.sourceNormal = normal ? { system: coding.system, code: coding.code, normal: normal.code } : null;
+  }
+
+  /** The codes a map element may have for this source: as given, then the normal form. */
+  sourceCodes(coding) {
+    const n = this.sourceNormal;
+    if (n && n.system === coding.system && n.code === coding.code) {
+      return [coding.code, n.normal];
+    }
+    return [coding.code];
+  }
+
+  /**
+   * The target concepts the maps produced are checked against their code systems:
+   *
+   *  - a target in a code system the server does not know gets a `note` part in its match,
+   *    saying that the code has not been validated
+   *  - a target that is not valid in its code system is still reported as the map has it, and
+   *    the response gets an `error` parameter for it (one per invalid target)
+   *  - a valid target that is not in its code system's normal form - an ICD-11 map written with
+   *    short codes, where the normal form is the entity id, say - also carries
+   *    `normalized-concept`: the same concept, as the code system has it
+   *
+   * This applies to any code system. `concept` is always the target as the map states it.
+   */
+  async checkTargetConcepts(result, params) {
+    const errors = [];
+    for (const match of result) {
+      if (match.name !== 'match' || !Array.isArray(match.part) || match[TRUSTED_TARGET]) {
+        continue;
+      }
+      const concept = match.part.find(p => p.name === 'concept')?.valueCoding;
+      if (!concept || !concept.system || !concept.code) {
+        continue;
+      }
+      const r = await this.resolveCode(concept.system, concept.version, concept.code, params);
+      const at = match.part.findIndex(p => p.name === 'concept');
+      const csName = concept.version ? `${concept.system}|${concept.version}` : concept.system;
+      if (!r.known) {
+        match.part.splice(at + 1, 0, { name: 'note', valueString:
+          this.i18n.translate('TRANSLATE_TARGET_CS_UNKNOWN', params.FHTTPLanguages, [csName, concept.code]) });
+      } else if (r.valid === false) {
+        errors.push({ name: 'error', valueString:
+          this.i18n.translate('CONCEPTMAP_GROUP_TARGET_CODE_INVALID', params.FHTTPLanguages, [concept.code, csName]) });
+      } else if (r.valid && r.normal !== concept.code) {
+        const coding = { system: concept.system };
+        if (concept.version) {
+          coding.version = concept.version;
+        }
+        coding.code = r.normal;
+        if (r.display) {
+          coding.display = r.display;
+        }
+        match.part.splice(at + 1, 0, { name: 'normalized-concept', valueCoding: coding });
+      }
+    }
+    result.push(...errors);
+  }
+
   /**
    * Whether a match counts towards result=true. A noMap match says there is no mapping, and
    * an explicit negative relationship says the target is not related, so neither is a