fhirsmith 0.10.1 → 0.11.1

package/tx/operation-context.js

@@ -118,6 +118,19 @@ class ResourceCache {
     return this.cache.has(cacheId);
   }
 
+  /**
+   * Whether a cache is sealed. A sealed cache holds only the resources it was
+   * created with (at $cache-control?mode=start) and does not grow as further
+   * resources are seen on subsequent operations. An unsealed cache accumulates
+   * every resource it sees. Unknown/absent cache-ids report false.
+   * @param {string} cacheId - The cache identifier
+   * @returns {boolean}
+   */
+  isSealed(cacheId) {
+    const entry = this.cache.get(cacheId);
+    return entry ? !!entry.sealed : false;
+  }
+
   /**
    * Add resources to a cache-id (merges with existing)
    * @param {string} cacheId - The cache identifier
@@ -126,6 +139,18 @@ class ResourceCache {
   add(cacheId, resources) {
     if (!resources || resources.length === 0) return;
 
+    // These resources are being retained under a cache-id (they persist across
+    // the batch), so mark them cached: their filter elements survive, and a
+    // provider may memoise resolved filter analysis on them (see filter()/fc).
+    // Inline tx-resources that never enter this pool stay unmarked.
+    for (const resource of resources) {
+      if (resource) {
+        // non-enumerable: internal marker only, must never serialise into a
+        // response or be copied by structuredClone/spread when a VS is cloned.
+        Object.defineProperty(resource, 'isCached', { value: true, enumerable: false, configurable: true, writable: true });
+      }
+    }
+
     const entry = this.cache.get(cacheId) || { resources: [], lastUsed: Date.now(), concepts: 0 };
 
     // Merge resources, avoiding duplicates by url+version. Keep the entry's concept
@@ -158,9 +183,20 @@ class ResourceCache {
    * Set resources for a cache-id (replaces existing)
    * @param {string} cacheId - The cache identifier
    * @param {Array} resources - Resources to set
+   * @param {boolean} [sealed=false] - If true, the cache is fixed at these
+   *   resources and will not grow when further resources are seen later.
    */
-  set(cacheId, resources) {
-    this.log.info(`cache-id '${cacheId}': set (replace all) with ${resources.length} resource(s): ${resources.map(r => this._resourceKey(r)).join(', ')}`);
+  set(cacheId, resources, sealed = false) {
+    this.log.info(`cache-id '${cacheId}': set (replace all, sealed=${!!sealed}) with ${resources.length} resource(s): ${resources.map(r => this._resourceKey(r)).join(', ')}`);
+    // Retained under a cache-id - mark cached so providers may memoise resolved
+    // filter analysis on their filter elements (see filter()/fc).
+    for (const resource of resources) {
+      if (resource) {
+        // non-enumerable: internal marker only, must never serialise into a
+        // response or be copied by structuredClone/spread when a VS is cloned.
+        Object.defineProperty(resource, 'isCached', { value: true, enumerable: false, configurable: true, writable: true });
+      }
+    }
     // Drop the old entry's contribution, then count the replacement.
     const existing = this.cache.get(cacheId);
     if (existing) {
@@ -174,7 +210,8 @@ class ResourceCache {
     this.cache.set(cacheId, {
       resources: [...resources],
       lastUsed: Date.now(),
-      concepts
+      concepts,
+      sealed: !!sealed
     });
     this._trackMax();
   }
@@ -519,6 +556,15 @@ const MEMORY_LIMIT = readMemoryLimit();
 const MEMORY_FRACTION = 0.98;
 const MEMORY_THRESHOLD = MEMORY_LIMIT > 0 ? MEMORY_LIMIT * MEMORY_FRACTION : 0; // 90% of cgroup limit
 const CHECK_FREQUENCY = 100;
+// How long an operation may compute without yielding the event loop (ms).
+// Node runs all JS on one thread: while a long operation executes, no other
+// request is serviced - awaiting in-memory promises does not help, because
+// resolved promises only bounce through the microtask queue, which runs before
+// I/O. checkAndYield() awaits a real setImmediate every YIELD_INTERVAL ms of
+// compute, so pending I/O (new connections, /metadata, timers) runs between
+// slices and one heavy $expand can no longer time out every other client.
+const YIELD_INTERVAL = 25;
+const { setImmediate: yieldToEventLoop } = require('timers/promises');
 
 class OperationContext {
   // Shared counter across all instances — only check RSS every CHECK_FREQUENCY calls
@@ -547,6 +593,24 @@ class OperationContext {
     // Shared by reference with copy()'d contexts so a sub-operation's
     // providers are cleaned up by the parent request's closeProviders().
     this._openProviders = [];
+    // Compute-time accounting for the deadline and for yielding. One object,
+    // shared by reference with copy()'d contexts, so an operation and its
+    // sub-operations (e.g. batch entries) draw down a single budget:
+    //  - compute:    ms of compute consumed in completed slices
+    //  - sliceStart: when the current compute slice began
+    //  - lastYield:  when we last gave the event loop a turn
+    //  - clientGone: set when the client disconnects before the response is sent
+    // The deadline is measured in COMPUTE time, not wall-clock: once operations
+    // time-share the event loop, wall-clock deadlines make concurrency itself
+    // cause aborts (N ops each take N x as long), which turns client retries
+    // into a failure amplifier. An operation that waited is not too costly;
+    // one that consumed its compute budget is.
+    this._clock = {
+      compute: 0,
+      sliceStart: this.startTime,
+      lastYield: this.startTime,
+      clientGone: false
+    };
 
     this.timeTracker.step('tx-op');
   }
@@ -579,6 +643,8 @@ class OperationContext {
     // Share the same provider-cleanup list so providers opened by the copy
     // are released when the parent operation ends.
     newContext._openProviders = this._openProviders;
+    // Share the compute clock: sub-operations spend the parent's budget.
+    newContext._clock = this._clock;
     return newContext;
   }
 
@@ -609,8 +675,10 @@ class OperationContext {
     }
     OperationContext._checkCounter = 0;
 
-    // Time check
-    const elapsed = performance.now() - this.startTime;
+    // Time check - against compute time consumed, not wall clock (see _clock in
+    // the constructor). For operations that never yield the two are identical,
+    // so this changes nothing for paths that don't use checkAndYield().
+    const elapsed = this.computeElapsed();
     if (elapsed > this.timeLimit) {
       const timeInSeconds = Math.round(this.timeLimit / 1000);
       this.log(`Operation took too long @ ${place} (${this.constructor.name})`);
@@ -637,6 +705,64 @@ class OperationContext {
     return false;
   }
 
+  /**
+   * Compute time this operation has consumed (ms): completed slices plus the
+   * slice currently executing. Equals wall-clock elapsed for operations that
+   * never yield.
+   * @returns {number}
+   */
+  computeElapsed() {
+    return this._clock.compute + (performance.now() - this._clock.sliceStart);
+  }
+
+  /**
+   * Mark that the client for this operation disconnected before the response
+   * was sent. The operation is aborted at its next yield point - there is no
+   * one left to send the result to. (Wired up by the request middleware from
+   * the response 'close' event.)
+   */
+  markClientGone() {
+    this._clock.clientGone = true;
+  }
+
+  /**
+   * deadCheck plus cooperative yielding: call this (awaited) from long-running
+   * loops in async code. Runs the normal deadCheck, and after every
+   * YIELD_INTERVAL ms of continuous compute awaits a real setImmediate so the
+   * event loop can service pending I/O - other requests keep getting answered
+   * while this operation works. On resuming from a yield, aborts if the client
+   * has disconnected in the meantime.
+   *
+   * Sync call sites keep using deadCheck(); the async loops that surround them
+   * yield on their behalf, which caps unbroken compute at one loop iteration.
+   *
+   * @param {string} place - Location identifier for debugging
+   */
+  async checkAndYield(place = 'unknown') {
+    this.deadCheck(place);
+    if (this.debugging) {
+      return;
+    }
+    const now = performance.now();
+    if (now - this._clock.lastYield >= YIELD_INTERVAL) {
+      // Close off this compute slice before suspending, so time spent waiting
+      // for our next turn is not charged against the deadline.
+      this._clock.compute += now - this._clock.sliceStart;
+      await yieldToEventLoop();
+      const resumed = performance.now();
+      this._clock.sliceStart = resumed;
+      this._clock.lastYield = resumed;
+      if (this._clock.clientGone) {
+        this.log(`Operation abandoned @ ${place}: client disconnected`);
+        const error = new Issue("error", "too-costly", null,
+            `Operation abandoned at ${place}: the client disconnected before the response was ready`);
+        error.abandoned = true;
+        error.diagnostics = this.diagnostics();
+        throw error;
+      }
+    }
+  }
+
   unSeeAll() {
     this.contexts = [];
   }

package/tx/provider.js

@@ -174,13 +174,7 @@ class Provider {
     for (const resource of resources) {
       const cs = new CodeSystem(await contentLoader.loadFile(resource, contentLoader.fhirVersion()));
       cs.sourcePackage = contentLoader.pid();
-      const existing = this.codeSystems.get(cs.url);
-      if (!existing || cs.isMoreRecent(existing)) {
-        this.codeSystems.set(cs.url, cs);
-      }
-      if (cs.version) {
-        this.codeSystems.set(cs.vurl, cs);
-      }
+      this.addCodeSystem(cs);
     }
     const vs = new PackageValueSetProvider(contentLoader);
     await vs.initialize();
@@ -486,13 +480,48 @@ class Provider {
     }
   }
 
+  /**
+   * True if the source package id (e.g. "hl7.terminology.r4#6.0.2") is a
+   * terminology.hl7.org (THO/UTG) package.
+   */
+  static #isTHOPackage(pid) {
+    return pid != null && pid.startsWith("hl7.terminology");
+  }
+
   addCodeSystem(cs) {
+    // Special case, mirroring CanonicalResourceManager.see() in the Java core:
+    // when content moved from the FHIR core packages to terminology.hl7.org,
+    // the versions went backwards (e.g. coverage-selfpay is 4.0.1 in
+    // hl7.fhir.r4.core (2019) but 1.0.1 in hl7.terminology (2024)), so plain
+    // version precedence would leave the stale core copy as the default. A
+    // resource from an hl7.terminology package displaces any same-URL
+    // resource that came from a FHIR core package, regardless of version.
+    if (Provider.#isTHOPackage(cs.sourcePackage)) {
+      for (const [key, t] of [...this.codeSystems]) {
+        // only the bare-url (default) entry is displaced; explicitly versioned
+        // entries (key = url|version) stay addressable, because the core packages
+        // carry historical versions (e.g. the normalized R4 v2 tables 0006/0360/0391
+        // at 2.1/2.3.1/2.4/2.6/2.7) that hl7.terminology does not publish at all
+        if (key === t.url && t.url === cs.url && VersionUtilities.isCorePackage(t.sourcePackage)) {
+          this.codeSystems.delete(key);
+        }
+      }
+    }
     const existing = this.codeSystems.get(cs.url);
-    if (!existing || cs.isMoreRecent(existing)) {
+    // the reverse guard: a core-package resource never displaces a THO one
+    const yieldsToTHO = existing && Provider.#isTHOPackage(existing.sourcePackage)
+      && VersionUtilities.isCorePackage(cs.sourcePackage);
+    if (!existing || (!yieldsToTHO && cs.isMoreRecent(existing))) {
       this.codeSystems.set(cs.url, cs);
     }
     if (cs.version) {
-      this.codeSystems.set(cs.vurl, cs);
+      // a versioned entry only yields to THO when THO itself provides that exact version
+      const vExisting = this.codeSystems.get(cs.vurl);
+      const vYieldsToTHO = vExisting && Provider.#isTHOPackage(vExisting.sourcePackage)
+        && VersionUtilities.isCorePackage(cs.sourcePackage);
+      if (!vYieldsToTHO) {
+        this.codeSystems.set(cs.vurl, cs);
+      }
     }
   }
 

package/tx/sct/ecl.js

@@ -472,8 +472,21 @@ class ECLParser {
   parseCompoundExpressionConstraint() {
     let left = this.parseRefinedExpressionConstraint();
 
+    // ECL 6.4: conjunction (AND), disjunction (OR) and exclusion (MINUS)
+    // operators must not be mixed within a single ungrouped compound expression;
+    // round brackets are mandatory to disambiguate. Track the operator used in
+    // this compound sequence and reject a mix (e.g. "<< A OR << B AND << C").
+    let compoundOp = null;
+
     while (this.match(ECLTokenType.AND, ECLTokenType.OR, ECLTokenType.MINUS)) {
       const operator = this.current;
+      if (compoundOp !== null && operator.type !== compoundOp) {
+        throw new Error(
+          'Ambiguous ECL expression: conjunction, disjunction and exclusion ' +
+          'operators cannot be mixed without parentheses (ECL 6.4). ' +
+          'Use round brackets to disambiguate.');
+      }
+      compoundOp = operator.type;
       this.advance();
       const right = this.parseRefinedExpressionConstraint();
 
@@ -616,12 +629,19 @@ class ECLParser {
     const attributes = [];
 
     do {
+      // A leading cardinality may prefix either a single attribute
+      // ([m..n] attr = v) or an attribute group ([m..n] { ... }). Parse it once,
+      // then dispatch on whether a group ({) or an attribute follows.
+      let cardinality = null;
+      if (this.match(ECLTokenType.LBRACKET)) {
+        cardinality = this.parseCardinality();
+      }
       if (this.match(ECLTokenType.LBRACE)) {
         // Attribute group
-        attributes.push(this.parseAttributeGroup());
+        attributes.push(this.parseAttributeGroup(cardinality));
       } else {
         // Single attribute
-        attributes.push(this.parseAttribute());
+        attributes.push(this.parseAttribute(cardinality));
       }
     } while (this.match(ECLTokenType.COMMA) && (this.advance(), true));
 
@@ -635,11 +655,10 @@ class ECLParser {
     };
   }
 
-  parseAttributeGroup() {
-    let cardinality = null;
-
-    // Check for cardinality before {
-    if (this.match(ECLTokenType.LBRACKET)) {
+  parseAttributeGroup(cardinality = null) {
+    // Cardinality may have been parsed by the caller (parseAttributeSet); if
+    // not, check for it here (before the opening brace).
+    if (cardinality === null && this.match(ECLTokenType.LBRACKET)) {
       cardinality = this.parseCardinality();
     }
 
@@ -659,12 +678,12 @@ class ECLParser {
     };
   }
 
-  parseAttribute() {
-    let cardinality = null;
+  parseAttribute(cardinality = null) {
     let reverse = false;
 
-    // Check for cardinality first - this must come before attribute name
-    if (this.match(ECLTokenType.LBRACKET)) {
+    // Cardinality may have been parsed by the caller (parseAttributeSet); if
+    // not, check for it here (it must come before the attribute name).
+    if (cardinality === null && this.match(ECLTokenType.LBRACKET)) {
       cardinality = this.parseCardinality();
     }
 

package/tx/sct/expressions.js

@@ -1534,6 +1534,33 @@ class SnomedExpressionServices {
    * List all display names for a concept
    * Equivalent to Pascal ListDisplayNames procedure
    */
+  // Key concept indexes for preferred-term selection (US English is the global
+  // default language reference set). Mirrors the logic in cs-snomed.js so both
+  // display paths agree.
+  _displayConstants() {
+    if (this._dispConst) return this._dispConst;
+    const idx = (id) => { const r = this.concepts.findConcept(id); return r.found ? r.index : -1; };
+    this._dispConst = {
+      usRefset: idx(900000000000509007n),
+      preferred: idx(900000000000548007n),
+      synonym: idx(900000000000013009n)
+    };
+    return this._dispConst;
+  }
+
+  _isPreferredSynonym(description) {
+    const K = this._displayConstants();
+    if (description.kind !== K.synonym) return false;
+    if (!description.valueses) return false;
+    const valueses = this.refs.getReferences(description.valueses);
+    if (!valueses) return false;
+    for (let i = 0; i < valueses.length; i++) {
+      const vals = this.refs.getReferences(valueses[i]); // [acceptabilityIndex, type]
+      if (vals && vals.length > 0 && vals[0] === K.preferred) return true;
+    }
+    return false;
+  }
+
   listDisplayNames(conceptIndex, languageFilter = 0) {
     const designations = [];
 
@@ -1558,7 +1585,7 @@ class SnomedExpressionServices {
             const term = this.strings.getEntry(description.iDesc).trim();
 
             designations.push({
-              isPreferred: i === 0, // First description is considered preferred
+              isPreferred: this._isPreferredSynonym(description), // preferred synonym in US English
               isActive: true,
               languageCode: this.codeForLanguage(description.lang),
               term: term,

package/tx/sct/structures.js

@@ -719,6 +719,11 @@ class SnomedConceptList {
     this.master.writeUInt32LE(value, index + 13);
   }
 
+  setModuleId(index, moduleIndex) {
+    this.#checkPostBuildAccess(index);
+    this.master.writeUInt32LE(moduleIndex, index + 36);
+  }
+
   setInbounds(index, value) {
     this.#checkPostBuildAccess(index);
     this.master.writeUInt32LE(value, index + 17);
@@ -749,11 +754,6 @@ class SnomedConceptList {
     this.master.writeUInt16LE(effectiveTime, index + 34);
   }
 
-  setModuleId(index, value) {
-    this.#checkPostBuildAccess(index);
-    this.master.writeUInt32LE(value, index + 36);
-  }
-
   setStatus(index, value) {
     this.#checkPostBuildAccess(index);
     this.master.writeUInt32LE(value, index + 40);

package/tx/tx-html.js

@@ -64,6 +64,28 @@ function loadTemplate() {
   }
 }
 
+/**
+ * Check if a request wants HTML back (by _format/format param, else Accept header).
+ * Module-level so non-rendering code (e.g. the search worker choosing a default
+ * page size) can make the same call the response renderer will make.
+ */
+function acceptsHtml(req) {
+  let _fmt = req.query._format || req.query.format || req.body?._format;
+  if (_fmt && typeof _fmt !== 'string') {
+    _fmt = null;
+  }
+  if (_fmt && (_fmt == 'html' || _fmt.startsWith('html/'))) {
+    return true;
+  }
+  if (!_fmt) {
+    _fmt = req.headers.accept || '';
+  }
+  if (typeof _fmt !== 'string') {
+    return false;
+  }
+  return _fmt.includes('text/html');
+}
+
 
 class TxHtmlRenderer {
   renderer;
@@ -98,20 +120,7 @@ class TxHtmlRenderer {
    * Check if request accepts HTML
    */
   acceptsHtml(req) {
-    let _fmt = req.query._format || req.query.format || req.body?._format;
-    if (_fmt && typeof _fmt !== 'string') {
-      _fmt = null;
-    }
-    if (_fmt && (_fmt == 'html' || _fmt.startsWith('html/'))) {
-      return true;
-    }
-    if (!_fmt) {
-      _fmt = req.headers.accept || '';
-    }
-    if (typeof _fmt !== 'string') {
-      return false;
-    }
-    return _fmt.includes('text/html');
+    return acceptsHtml(req);
   }
 
   /**
@@ -1362,5 +1371,5 @@ class TxHtmlRenderer {
 }
 
 module.exports = {
-  TxHtmlRenderer, loadTemplate
+  TxHtmlRenderer, loadTemplate, acceptsHtml
 };

package/tx/tx.fhir.org.yml

@@ -18,9 +18,6 @@ sources:
   - unii:unii_20240622.db
   - snomed:sct_intl_20240201.cache
   - snomed!:sct_intl_20250201.cache
-#  - snomed:sct_se_20231130.cache
-#  - snomed:sct_au_20230731.cache
-#  - snomed:sct_be_20231115.cache
   - snomed:sct_ch_20230607.cache
   - snomed:sct_dk_20260331.cache
   - snomed:sct_ips_20241216.cache
@@ -29,7 +26,7 @@ sources:
   - snomed:sct_us_20230301.cache
   - snomed:sct_us_20240301.cache
   - snomed:sct_us_20250901.cache
-  - snomed:sct_test_20250814.cache
+  - snomed:sct_test_20250909.cache
   - cpt:CodeSystem-cpt.db|cpt-2023-fragment-0.1.db
   - omop:omop_v20250227.db
   - npm:hl7.terminology

package/tx/tx.js

@@ -291,9 +291,15 @@ class TXModule {
       // Get Accept-Language header for language preferences
       const acceptLanguage = req.get('Accept-Language') || 'en';
 
-      // Create operation context with language, ID, time limit, and caches
+      // Create operation context with language, ID, time limit, and caches.
+      // The time limit is a compute-time budget (see OperationContext._clock)
+      // and is configurable (config.operationTimeLimit, seconds). The default
+      // sits deliberately under common client timeouts (okhttp reads give up at
+      // 10-30s): if the server's limit is >= the client's, the client hangs up
+      // first and the carefully-built too-costly OperationOutcome is written to
+      // a socket nobody is reading.
       const opContext = new OperationContext(
-        acceptLanguage, this.i18n, requestId, 30,
+        acceptLanguage, this.i18n, requestId, this.config.operationTimeLimit ?? 20,
         endpointInfo.resourceCache, endpointInfo.expansionCache
       );
       opContext.usageTracker = this.usageTracker;
@@ -325,6 +331,16 @@ class TXModule {
       res.on('finish', releaseProviders);
       res.on('close', releaseProviders);
 
+      // If the connection closes before the response was completed, the client
+      // has given up (timeout, cancelled build, dropped connection). Mark the
+      // operation so it aborts at its next yield point instead of computing a
+      // response nobody will read.
+      res.on('close', () => {
+        if (!res.writableEnded) {
+          opContext.markClientGone();
+        }
+      });
+
       // Add X-Request-Id header to response
       res.setHeader('X-Request-Id', requestId);
 

package/tx/vs/vs-package.js

@@ -1,4 +1,5 @@
 const path = require('path');
+const fs = require('fs').promises;
 const { AbstractValueSetProvider } = require('./vs-api');
 const { PackageContentLoader } = require('../../library/package-manager');
 const { ValueSetDatabase } = require('./vs-database');
@@ -51,9 +52,60 @@ class PackageValueSetProvider extends AbstractValueSetProvider {
     }
 
     this.valueSetMap = await this.database.loadAllValueSets(this.sourcePackage());
+
+    // Self-heal caches built before pipe-versioned canonical urls were normalized at
+    // load time (the R4 v2 tables 0006/0360/0391 - see normalizeVersionedCanonicals in
+    // package-manager.js): if any cached value set still carries a "system|version"
+    // include or a pipe in its url, rebuild this cache from the package.
+    if (dbExists && this.#hasPipeVersionedCanonicals()) {
+      await this.database.close();
+      await fs.rm(this.dbPath, { force: true });
+      this.database = new ValueSetDatabase(this.dbPath);
+      await this.database.create();
+      await this._populateDatabase();
+      this.valueSetMap = await this.database.loadAllValueSets(this.sourcePackage());
+    }
+
+    // Mark each value set as cached as it enters the store: it is retained in
+    // memory for the process lifetime, so its compose.include.filter elements
+    // persist and a provider may memoise resolved filter analysis on them
+    // (see CodeSystemProvider.filter()'s `fc` parameter).
+    for (const vs of this.valueSetMap.values()) {
+      if (vs) {
+        // non-enumerable: internal marker only, must never serialise into a
+        // response or be copied by structuredClone/spread when a VS is cloned.
+        Object.defineProperty(vs, 'isCached', { value: true, enumerable: false, configurable: true, writable: true });
+      }
+    }
     this.initialized = true;
   }
 
+  /**
+   * True if any loaded value set still has a pipe-versioned canonical (url or
+   * compose include/exclude system) - the signature of a cache written before
+   * load-time normalization existed.
+   */
+  #hasPipeVersionedCanonicals() {
+    for (const vs of this.valueSetMap.values()) {
+      if (!vs) {
+        continue;
+      }
+      if (typeof vs.url === 'string' && vs.url.includes('|')) {
+        return true;
+      }
+      if (vs.compose) {
+        for (const list of [vs.compose.include, vs.compose.exclude]) {
+          for (const inc of list || []) {
+            if (typeof inc.system === 'string' && inc.system.includes('|')) {
+              return true;
+            }
+          }
+        }
+      }
+    }
+    return false;
+  }
+
   async close() {
     await this.database.close();
   }