fhirsmith 0.9.1 → 0.9.3

package/tx/vs/vs-vsac.js

@@ -1,4 +1,5 @@
 const path = require('path');
+const crypto = require('crypto');
 const axios = require('axios');
 const { AbstractValueSetProvider } = require('./vs-api');
 const { ValueSetDatabase } = require('./vs-database');
@@ -11,6 +12,8 @@ const {debugLog} = require("../operation-context");
  * Fetches and caches ValueSets from the NLM VSAC FHIR server
  */
 class VSACValueSetProvider extends AbstractValueSetProvider {
+  SYNC_AT_START_UP = false;
+
   /**
    * @param {Object} config - Configuration object
    * @param {string} config.apiKey - API key for VSAC authentication
@@ -71,12 +74,11 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
     if (!(await this.database.exists())) {
       await this.database.create();
     } else {
-      // Ensure schema is up to date (e.g. date_first_seen column added after initial deploy)
-      await this.database._migrateIfNeeded(await this.database._getWriteConnection());
-      // Load existing data
+      // Schema migrations are applied lazily by the database layer on first
+      // connection. Just load existing data.
       await this._reloadMap();
     }
-    if (this.valueSetMap.size == 0) {
+    if (this.SYNC_AT_START_UP || this.valueSetMap.size == 0) {
       await this.refreshValueSets();
     }
     // Start periodic refresh
@@ -134,7 +136,7 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
       console.log('Starting VSAC ValueSet refresh...');
 
       // This lists all the currently valid value sets by URL, but not the older versions
-      let url = '/ValueSet?_offset=0&_count=100&_elements=id,url,version,status';
+      let url = '/ValueSet?_offset=0&_count=1000&_elements=id,url,version,status';
 
       let total = undefined;
       let count = 0;
@@ -182,11 +184,11 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
       // deduplicate the queue
       this.queue = [...new Set(this.queue)];
 
-      let tracking = { totalFetched: 0, totalNew: 0, count: 0, newCount : 0 };
+      let tracking = { totalFetched: 0, totalNew: 0, totalUpdated: 0, count: 0, newCount : 0 };
       // phase 2: query for history & content
       this.requeue = [];
       for (let q of this.queue) {
-        this.stats.task('VSAC History for '+q, `running (${tracking.totalFetched} fetched, ${tracking.totalNew} new)`);
+        this.stats.task('VSAC History for '+q, `running (${tracking.totalFetched} fetched, ${tracking.totalNew} new, ${tracking.totalUpdated} updated)`);
         try {
           await this.processContentAndHistory(q, tracking, this.queue.length);
         } catch (error) {
@@ -194,29 +196,27 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
           debugLog(error);
           this.stats.task('VSAC Sync', error.message);
         }
-        // `running (${totalFetched} fetched, ${totalNew} new)`)
         tracking.count++;
       }
       console.log("Requeue");
       for (let q of this.requeue) {
-        this.stats.task('VSAC History for '+q, `running (${tracking.totalFetched} fetched, ${tracking.totalNew} new)`);
+        this.stats.task('VSAC History for '+q, `running (${tracking.totalFetched} fetched, ${tracking.totalNew} new, ${tracking.totalUpdated} updated)`);
         try {
           await this.processContentAndHistory(q, tracking, this.requeue.length);
         } catch (error) {
           debugLog(error);
           this.stats.task('VSAC Sync', error.message);
         }
-        // `running (${totalFetched} fetched, ${totalNew} new)`)
         tracking.count++;
       }
 
       // Reload map with fresh data
       await this._reloadMap();
-      let msg = `VSAC refresh completed. Total: ${tracking.totalFetched} ValueSets, Deleted: ${tracking.deletedCount}`;
+      let msg = `VSAC refresh completed. Total: ${tracking.totalFetched} ValueSets, New: ${tracking.totalNew}, Updated: ${tracking.totalUpdated}`;
       this.stats.taskDone('VSAC Sync', msg);
       console.log(msg);
 
-      await this.database.finishRun(runId, tracking.totalFetched, tracking.totalNew);
+      await this.database.finishRun(runId, tracking.totalFetched, tracking.totalNew, tracking.totalUpdated);
     } catch (error) {
       debugLog(error, 'Error during VSAC refresh:');
       this.stats.taskError('VSAC Sync', `Error (${error.message})`);
@@ -228,30 +228,71 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
   }
 
   /**
-   * Insert multiple ValueSets in a batch operation
+   * Compute a SHA-256 hash of the ValueSet content for change detection.
+   * @param {Object} vs - The ValueSet resource (plain JSON object)
+   * @returns {string} hex-encoded SHA-256
+   * @private
+   */
+  _hashValueSet(vs) {
+    return crypto.createHash('sha256').update(JSON.stringify(vs)).digest('hex');
+  }
+
+  /**
+   * Insert multiple ValueSets in a batch operation.
+   * For each value set: if url|version is already known, compare content hashes.
+   *   - hash unchanged  -> touch last_seen only (seeValueSet)
+   *   - hash changed    -> upsert and record an 'updated' event
+   *   - not seen before -> upsert and record a 'new' event
    * @param {Array<Object>} valueSets - Array of ValueSet resources
-   * @returns {Promise<void>}
+   * @returns {Promise<{newCount: number, updatedCount: number}>}
    */
   async batchUpsertValueSets(valueSets) {
     if (valueSets.length === 0) {
-      return;
+      return { newCount: 0, updatedCount: 0 };
     }
 
-    let count = 0;
+    let newCount = 0;
+    let updatedCount = 0;
+
     // Process sequentially to avoid database locking
     for (const valueSet of valueSets) {
-      let key = valueSet.url+"|"+valueSet.version;
-      let vs = this.valueSetMap.get(key);
-      if (vs) {
-        // we've seen this before, and maybe fetched it's history, so just update
-        // the timestamp
-        await this.database.seeValueSet(valueSet);
+      const key = valueSet.url+"|"+valueSet.version;
+      const existing = this.valueSetMap.get(key);
+      const newHash = this._hashValueSet(valueSet);
+
+      if (existing) {
+        // We've seen this url|version before. Decide whether the content
+        // has actually changed by comparing hashes.
+        //
+        // Note: _reloadMap() mutates the in-memory jsonObj (strips inc.version
+        // from compose.include/exclude), so we cannot reliably recompute a
+        // hash from existing.jsonObj — it would not match the hash of the
+        // original unmutated JSON we stored. For rows predating this feature
+        // (content_hash NULL), we defer update detection until the next cycle:
+        // the upsert below runs only when hashes differ, so on the *next*
+        // sync after migration we'll have a proper baseline.
+        if (existing.contentHash && existing.contentHash === newHash) {
+          // No change - just touch last_seen
+          await this.database.seeValueSet(valueSet);
+        } else if (!existing.contentHash) {
+          // Legacy row without a stored hash - backfill the hash silently
+          // without emitting a spurious 'updated' event. We do a lightweight
+          // touch + hash update rather than a full upsert+event.
+          await this.database.seeValueSet(valueSet);
+          await this.database.setContentHash(valueSet.id, newHash);
+        } else {
+          // Content has changed - treat as update
+          await this.database.upsertValueSet(valueSet, newHash);
+          await this.database.recordEvent('updated', valueSet.url, valueSet.version);
+          updatedCount++;
+        }
       } else {
-        await this.database.upsertValueSet(valueSet);
-        count++;
+        await this.database.upsertValueSet(valueSet, newHash);
+        await this.database.recordEvent('new', valueSet.url, valueSet.version);
+        newCount++;
       }
     }
-    return count;
+    return { newCount, updatedCount };
   }
 
   /**
@@ -511,18 +552,21 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
     const bundle = await this._fetchBundle(url);
 
     let vcount = 0;
+    let perRun = { newCount: 0, updatedCount: 0 };
     if (bundle.entry && bundle.entry.length > 0) {
       // Extract ValueSets from bundle entries
       const valueSets = bundle.entry
           .filter(entry => entry.resource && entry.resource.resourceType === 'ValueSet')
           .map(entry => entry.resource);
       if (valueSets.length > 0) {
-        tracking.totalNew = tracking.totalNew + await this.batchUpsertValueSets(valueSets);
+        perRun = await this.batchUpsertValueSets(valueSets);
+        tracking.totalNew += perRun.newCount;
+        tracking.totalUpdated += perRun.updatedCount;
         tracking.totalFetched += valueSets.length;
         vcount = valueSets.length;
       }
     }
-    let logMsg = `VSAC (${tracking.count} of ${length}) ${q}: ${vcount} versions`;
+    let logMsg = `VSAC (${tracking.count} of ${length}) ${q}: ${vcount} versions (${perRun.newCount} new, ${perRun.updatedCount} updated)`;
     console.log(logMsg);
     this.stats.task('VSAC Sync', logMsg);
   }
@@ -593,36 +637,55 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
 
     const rows = await new Promise((resolve, reject) => {
       db.all(
-          `SELECT 'vs' AS kind,
+          `SELECT 'event' AS kind,
                   url,
                   version,
-                  date_first_seen AS ts,
-                  NULL AS status,
-                  NULL AS error_message,
-                  NULL AS finished_at,
-                  NULL AS total_fetched,
-                  NULL AS total_new
-           FROM valuesets
-           WHERE date_first_seen > 0
+             timestamp AS ts,
+             event_type,
+             NULL AS status,
+             NULL AS error_message,
+             NULL AS finished_at,
+             NULL AS total_fetched,
+             NULL AS total_new,
+             NULL AS total_updated
+           FROM vsac_events
            UNION ALL
-           SELECT 'run' AS kind,
-                  NULL,
-                  NULL,
-                  started_at AS ts,
-                  status,
-                  error_message,
-                  finished_at,
-                  total_fetched,
-                  total_new
-           FROM vsac_runs
-           ORDER BY ts DESC
-             LIMIT 200`,
+          SELECT 'run' AS kind,
+                 NULL,
+                 NULL,
+                 started_at AS ts,
+                 NULL AS event_type,
+                 status,
+                 error_message,
+                 finished_at,
+                 total_fetched,
+                 total_new,
+                 total_updated
+          FROM vsac_runs
+          ORDER BY ts DESC
+            LIMIT 200`,
           [],
           (err, rows) => err ? reject(err) : resolve(rows)
       );
     });
 
-    const fmt = ts => ts
+    // ISO date (YYYY-MM-DD UTC) for grouping
+    const dayKey = ts => ts
+        ? new Date(ts * 1000).toISOString().substring(0, 10)
+        : '';
+    // Human-friendly day heading e.g. "Tuesday, 14 April 2026"
+    const dayLabel = ts => ts
+        ? new Date(ts * 1000).toLocaleDateString('en-GB', {
+          weekday: 'long', year: 'numeric', month: 'long', day: 'numeric',
+          timeZone: 'UTC'
+        })
+        : '—';
+    // HH:MM:SS UTC within a day
+    const timeOnly = ts => ts
+        ? new Date(ts * 1000).toISOString().substring(11, 19) + ' UTC'
+        : '—';
+    // Full timestamp (used in "Running..." detail where context is needed)
+    const fmtFull = ts => ts
         ? new Date(ts * 1000).toISOString().replace('T', ' ').substring(0, 19) + ' UTC'
         : '—';
 
@@ -631,30 +694,59 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
     html += '<thead><tr><th>Time</th><th>Event</th><th>Detail</th></tr></thead>';
     html += '<tbody>';
 
+    let currentDay = null;
     for (const row of rows) {
+      const rowDay = dayKey(row.ts);
+      if (rowDay !== currentDay) {
+        currentDay = rowDay;
+        html += `<tr style="background:#d8d8d8">`;
+        html += `<td colspan="3"><strong>${escape(dayLabel(row.ts))}</strong></td>`;
+        html += `</tr>`;
+      }
+
       if (row.kind === 'run') {
         const duration = row.finished_at ? `${row.finished_at - row.ts}s` : 'in progress';
         let detail, colour;
         if (row.status === 'ok') {
-          detail = `${row.total_fetched} fetched, ${row.total_new} new, ${duration}`;
+          const updated = row.total_updated != null ? `, ${row.total_updated} updated` : '';
+          detail = `${row.total_fetched} fetched, ${row.total_new} new${updated}, ${duration}`;
           colour = 'green';
         } else if (row.status === 'error') {
           detail = `Failed: ${escape(row.error_message || '')} (${duration})`;
           colour = 'red';
         } else {
-          detail = `Running... (started ${fmt(row.ts)})`;
+          detail = `Running... (started ${fmtFull(row.ts)})`;
           colour = 'orange';
         }
         html += `<tr style="background:#f0f0f0">`;
-        html += `<td>${escape(fmt(row.ts))}</td>`;
+        html += `<td>${escape(timeOnly(row.ts))}</td>`;
         html += `<td><strong style="color:${colour}">Sync run</strong></td>`;
         html += `<td>${detail}</td>`;
         html += `</tr>`;
       } else {
+        // Event row: 'new', 'updated', or 'deleted'
+        let label, colour;
+        switch (row.event_type) {
+          case 'new':
+            label = 'New';
+            colour = 'green';
+            break;
+          case 'updated':
+            label = 'Updated';
+            colour = 'blue';
+            break;
+          case 'deleted':
+            label = 'Deleted';
+            colour = 'red';
+            break;
+          default:
+            label = escape(row.event_type || 'Event');
+            colour = 'black';
+        }
         html += `<tr>`;
-        html += `<td>${escape(fmt(row.ts))}</td>`;
-        html += `<td>New value set</td>`;
-        html += `<td>${escape(row.url || '')}#${escape(row.version || '')}</td>`;
+        html += `<td>${escape(timeOnly(row.ts))}</td>`;
+        html += `<td><span style="color:${colour}">${label}</span></td>`;
+        html += `<td>${escape(this.urlTail(row.url) || '')} v <a href="../ValueSet/${escape(this.urlTail(row.url) || '')}-${escape(row.version || '')}">${escape(row.version || '')}</a></td>`;
         html += `</tr>`;
       }
     }
@@ -666,6 +758,10 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
   id() {
     return "vsac";
   }
+
+  urlTail(url) {
+    return url ? url.substring(url.lastIndexOf('/') + 1) : '';
+  }
 }
 
 // Usage examples:

package/tx/workers/expand.js

@@ -22,6 +22,7 @@ const {debugLog} = require("../operation-context");
 
 // Expansion limits (from Pascal constants)
 const EXTERNAL_DEFAULT_LIMIT = 1000;
+const EXTERNAL_TEST_DEFAULT_LIMIT = 3000;
 const INTERNAL_DEFAULT_LIMIT = 10000;
 const EXPANSION_DEAD_TIME_SECS = 30;
 const CACHE_WHEN_DEBUGGING = false;
@@ -508,8 +509,8 @@ class ValueSetExpander {
     this.excluded.add(key);
   }
 
-  async checkCanExpandValueSet(uri, version) {
-    const vs = await this.worker.findValueSet(uri, version);
+  async checkCanExpandValueSet(uri, version, source) {
+    const vs = await this.worker.findValueSet(uri, version, source);
     if (vs == null) {
       if (!version && uri.includes('|')) {
         version = uri.substring(uri.indexOf('|') + 1);
@@ -525,9 +526,8 @@ class ValueSetExpander {
     }
   }
 
-  async expandValueSet(uri, version, filter, notClosed) {
+  async expandValueSet(uri, version, vs, filter, notClosed) {
 
-    let vs = await this.worker.findValueSet(uri, version);
     if (!vs) {
       if (version) {
         throw new Issue('error', 'not-found', null, 'VS_EXP_IMPORT_UNK_PINNED', this.worker.i18n.translate('VS_EXP_IMPORT_UNK_PINNED', this.params.httpLanguages, [uri, version]), "not-found", 422);
@@ -609,14 +609,14 @@ class ValueSetExpander {
     }
   }
 
-  async checkSource(cset, exp, filter, srcURL, ts, vsInfo) {
+  async checkSource(cset, exp, filter, srcURL, ts, vsInfo , source) {
     this.worker.deadCheck('checkSource');
     Extensions.checkNoModifiers(cset, 'ValueSetExpander.checkSource', 'set', srcURL);
     let imp = false;
     for (const u of cset.valueSet || []) {
       this.worker.deadCheck('checkSource');
       const s = this.worker.pinValueSet(u);
-      await this.checkCanExpandValueSet(s, '');
+      await this.checkCanExpandValueSet(s, '', source);
       imp = true;
     }
 
@@ -659,7 +659,7 @@ class ValueSetExpander {
 
         if (!cset.concept && !cset.filter) {
           if (cs.specialEnumeration()) {
-            await this.checkCanExpandValueSet(cs.specialEnumeration(), '');
+            await this.checkCanExpandValueSet(cs.specialEnumeration(), '', null);
           } else if (filter.isNull) {
             if (cs.isNotClosed()) {
               if (cs.specialEnumeration()) {
@@ -704,9 +704,12 @@ class ValueSetExpander {
         this.worker.deadCheck('processCodes#2');
         const s = this.worker.pinValueSet(u);
         this.worker.opContext.log('import value set ' + s);
-        const ivs = new ImportedValueSet(await this.expandValueSet(s, '', filter, notClosed));
-        this.checkResourceCanonicalStatus(expansion, ivs.valueSet, this.valueSet);
-        this.addParamUri(expansion, 'used-valueset', this.worker.makeVurl(ivs.valueSet));
+        let vs = await this.worker.findValueSet(s, '', vsSrc);
+        const ivs = new ImportedValueSet(await this.expandValueSet(s, '', vs, filter, notClosed));
+        this. checkResourceCanonicalStatus(expansion, ivs.valueSet, this.valueSet);
+        if (!vs.isContained) {
+          this.addParamUri(expansion, 'used-valueset', this.worker.makeVurl(ivs.valueSet));
+        }
         valueSets.push(ivs);
       }
       this.addToTotal(await this.importValueSet(valueSets[0].valueSet, expansion, valueSets, 1));
@@ -728,16 +731,20 @@ class ValueSetExpander {
           this.worker.deadCheck('processCodes#2');
           const s = this.worker.pinValueSet(u);
           this.worker.opContext.log('import value set ' + s);
-          const ivs = new ImportedValueSet(await this.expandValueSet(s, '', filter, notClosed));
+          let vs = await this.worker.findValueSet(s, '', vsSrc);
+          const ivs = new ImportedValueSet(await this.expandValueSet(s, '', vs, filter, notClosed));
           this.checkResourceCanonicalStatus(expansion, ivs.valueSet, this.valueSet);
-          this.addParamUri(expansion, 'used-valueset', this.worker.makeVurl(ivs.valueSet));
+          if (!vs.isContained) {
+            this.addParamUri(expansion, 'used-valueset', this.worker.makeVurl(ivs.valueSet));
+          }
           valueSets.push(ivs);
         }
 
         if (!cset.concept && !cset.filter) {
           if (cs.specialEnumeration() && filters.length === 0) {
             this.worker.opContext.log('import special value set ' + cs.specialEnumeration());
-            const base = await this.expandValueSet(cs.specialEnumeration(), '', filter, notClosed);
+            let vs = await this.worker.findValueSet(cs.specialEnumeration(), '', null);
+            const base = await this.expandValueSet(cs.specialEnumeration(), '', vs, filter, notClosed);
             Extensions.addBoolean(expansion, "http://hl7.org/fhir/StructureDefinition/valueset-unclosed", true);
             Extensions.addString(expansion, "http://hl7.org/fhir/StructureDefinition/valueset-unclosed-reason", 'The code System "' + cs.system() + " has a grammar and so has infinite members. This extension is based on " + cs.specialEnumeration());
             await this.importValueSet(base, expansion, valueSets, 0);
@@ -860,7 +867,7 @@ class ValueSetExpander {
               throw new Issue('error', 'invalid', path + ".filter[" + i + "]", 'UNABLE_TO_HANDLE_SYSTEM_FILTER_WITH_NO_VALUE', this.worker.i18n.translate('UNABLE_TO_HANDLE_SYSTEM_FILTER_WITH_NO_VALUE', this.params.httpLanguages, [cs.system(), fc.property, fc.op]), 'vs-invalid', 400);
             }
             Extensions.checkNoModifiers(fc, 'ValueSetExpander.processCodes', 'filter', vsSrc.vurl);
-            await cs.filter(prep, fc.property, fc.op, fc.value);
+            await cs.filter(prep, i == 0, fc.property, fc.op, fc.value);
           }
 
           const fset = await cs.executeFilters(prep);
@@ -871,6 +878,7 @@ class ValueSetExpander {
           }
 
           this.worker.opContext.log('iterate filters');
+          this.addToTotal(0);
           const cds = new Designations(this.worker.i18n.languageDefinitions);
           while (await cs.filterMore(prep, fset[0])) {
             this.worker.deadCheck('processCodes#5');
@@ -937,9 +945,12 @@ class ValueSetExpander {
         for (const u of cset.valueSet) {
           const s = this.worker.pinValueSet(u);
           this.worker.deadCheck('processCodes#2');
-          const ivs = new ImportedValueSet(await this.expandValueSet(s, '', filter, notClosed));
+          let vs = await this.worker.findValueSet(s, '', vsSrc);
+          const ivs = new ImportedValueSet(await this.expandValueSet(s, '',  vs, filter, notClosed));
           this.checkResourceCanonicalStatus(expansion, ivs.valueSet, this.valueSet);
-          this.addParamUri(expansion, 'used-valueset', ivs.valueSet.vurl);
+          if (!vs.isContained) {
+            this.addParamUri(expansion, 'used-valueset', ivs.valueSet.vurl);
+          }
           valueSets.push(ivs);
         }
         this.excludeValueSet(valueSets[0].valueSet, expansion, valueSets, 1);
@@ -959,9 +970,12 @@ class ValueSetExpander {
         this.worker.deadCheck('processCodes#3');
         const s = this.worker.pinValueSet(u);
         this.worker.opContext.log('import value set ' + s);
-        const ivs = new ImportedValueSet(await this.expandValueSet(s, '', filter, notClosed));
+        let vs = await this.worker.findValueSet(s, '', vsSrc);
+        const ivs = new ImportedValueSet(await this.expandValueSet(s, '', vs, filter, notClosed));
         this.checkResourceCanonicalStatus(expansion, ivs.valueSet, this.valueSet);
-        this.addParamUri(expansion, 'used-valueset', this.worker.makeVurl(ivs.valueSet));
+        if (!vs.isContained) {
+          this.addParamUri(expansion, 'used-valueset', this.worker.makeVurl(ivs.valueSet));
+        }
         valueSets.push(ivs);
       }
 
@@ -1039,10 +1053,12 @@ class ValueSetExpander {
           Extensions.addString(expansion, "http://hl7.org/fhir/StructureDefinition/valueset-unclosed-reason", 'The code System "' + cs.system() + " has a grammar and so has infinite members. This extension is based on " + cs.specialEnumeration());
         }
 
+        let first = true;
         for (let fc of cset.filter) {
           this.worker.deadCheck('processCodes#4a');
           Extensions.checkNoModifiers(fc, 'ValueSetExpander.processCodes', 'filter', vsSrc.vurl);
-          await cs.filter(prep, fc.property, fc.op, fc.value);
+          await cs.filter(prep, first, fc.property, fc.op, fc.value);
+          first = false;
         }
 
         this.worker.opContext.log('iterate filters');
@@ -1150,12 +1166,12 @@ class ValueSetExpander {
     const ts = new Map();
     for (const c of source.jsonObj.compose.include || []) {
       this.worker.deadCheck('handleCompose#2');
-      await this.checkSource(c, expansion, filter, source.url, ts, vsInfo);
+      await this.checkSource(c, expansion, filter, source.url, ts, vsInfo, source);
     }
     for (const c of source.jsonObj.compose.exclude || []) {
       this.worker.deadCheck('handleCompose#3');
       this.hasExclusions = true;
-      await this.checkSource(c, expansion, filter, source.url, ts, null);
+      await this.checkSource(c, expansion, filter, source.url, ts, null, source);
     }
 
     this.worker.opContext.log('compose #2');
@@ -1215,6 +1231,7 @@ class ValueSetExpander {
       result.publisher = undefined;
       result.extension = undefined;
       result.text = undefined;
+      result.contained = undefined;
     }
 
     for (let s of this.params.supplements) this.requiredSupplements.add(s);
@@ -1909,7 +1926,7 @@ class ExpandWorker extends TerminologyWorker {
       const url = this.getParameterValue(urlParam);
       const version = versionParam ? this.getParameterValue(versionParam) : null;
 
-      valueSet = await this.findValueSet(url, version);
+      valueSet = await this.findValueSet(url, version, null);
       this.seeSourceVS(valueSet, url);
       if (!valueSet) {
         return res.status(422).json(this.operationOutcome('error', 'not-found',
@@ -2072,8 +2089,8 @@ class ExpandWorker extends TerminologyWorker {
 
     if (params.limit < -1) {
       params.limit = -1;
-    } else if (params.limit > EXTERNAL_DEFAULT_LIMIT) {
-      params.limit = EXTERNAL_DEFAULT_LIMIT; // can't ask for more than this externally, though you can internally
+    } else if (params.limit > this.externalLimit) {
+      params.limit = this.externalLimit; // can't ask for more than this externally, though you can internally
     }
 
     const filter = new SearchFilterText(params.filter);
@@ -2123,6 +2140,7 @@ module.exports = {
   EmptyFilterContext,
   EXTERNAL_DEFAULT_LIMIT,
   INTERNAL_DEFAULT_LIMIT,
+  EXTERNAL_TEST_DEFAULT_LIMIT,
   TotalStatus,
   EXPANSION_DEAD_TIME_SECS
 };

package/tx/workers/related.js

@@ -188,7 +188,7 @@ class RelatedWorker extends TerminologyWorker {
     const url = this.getParameterValue(urlParam);
     const version = versionParam ? this.getParameterValue(versionParam) : null;
 
-    let valueSet = await this.findValueSet(url, version);
+    let valueSet = await this.findValueSet(url, version, null);
     this.seeSourceVS(valueSet, url);
     if (!valueSet) {
       return res.status(404).json(this.operationOutcome('error', 'not-found',

package/tx/workers/validate.js

@@ -351,7 +351,7 @@ class ValueSetChecker {
         let s = this.worker.pinValueSet(u);
         this.worker.deadCheck('prepareConceptSet');
         if (!this.others.has(s)) {
-          let other = await this.worker.findValueSet(s, '');
+          let other = await this.worker.findValueSet(s, '', vs);
           if (other === null) {
             throw new Issue('error', 'not-found', null, 'Unable_to_resolve_value_Set_', this.worker.i18n.translate('Unable_to_resolve_value_Set_', this.params.HTTPLanguages, [s]), 'not-found', 422);
           }
@@ -472,7 +472,7 @@ class ValueSetChecker {
         this.worker.opContext.addNote(this.valueSet, 'Didn\'t find CodeSystem "' + this.worker.renderer.displayCoded(system, version) + '"', this.indentCount);
         result = null;
         cause.value = 'not-found';
-        let vss = await this.worker.findValueSet(system, '');
+        let vss = await this.worker.findValueSet(system, '', null);
         if (vss !== null) {
           vss = null;
           let msg = this.worker.i18n.translate('Terminology_TX_System_ValueSet2', this.params.HTTPLanguages, [system]);
@@ -1154,7 +1154,7 @@ class ValueSetChecker {
         }
         let prov = await this.worker.findCodeSystem(ws, c.version, this.params, ['complete', 'fragment'],  op,true, true, false, this.worker.requiredSupplements);
         if (prov === null) {
-          let vss = await this.worker.findValueSet(ws, '');
+          let vss = await this.worker.findValueSet(ws, '', null);
           if (vss !== null) {
             vss = null;
             let m = this.worker.i18n.translate('Terminology_TX_System_ValueSet2', this.params.HTTPLanguages, [ws]);
@@ -1538,9 +1538,6 @@ class ValueSetChecker {
 
   async checkConceptSet(path, role, cs, cset, code, displays, vs, message, inactive, normalForm, vstatus, op, vcc, messages) {
     this.worker.opContext.addNote(vs, 'check code ' + role + ' ' + this.worker.renderer.displayValueSetInclude(cset) + ' at ' + path, this.indentCount);
-    if (role !== 'not in') {
-      inactive.value = false;
-    }
     let result = false;
     if (!cset.concept && !cset.filter) {
       let loc = await cs.locate(code);
@@ -1683,7 +1680,7 @@ class ValueSetChecker {
         if (!fc.value) {
           throw new Issue('error', 'invalid', null, 'UNABLE_TO_HANDLE_SYSTEM_FILTER_WITH_NO_VALUE', this.worker.i18n.translate('UNABLE_TO_HANDLE_SYSTEM_FILTER_WITH_NO_VALUE', this.params.HTTPLanguages, [cs.system(), fc.property, fc.op]));
         }
-        await cs.filter(prep, fc.property, fc.op, fc.value);
+        await cs.filter(prep, false, fc.property, fc.op, fc.value);
         // if (f === null) {
         //   throw new Issue('error', 'not-supported', null, 'FILTER_NOT_UNDERSTOOD', this.worker.i18n.translate('FILTER_NOT_UNDERSTOOD', this.params.HTTPLanguages, [fc.property, fc.op, fc.value, vs.vurl, cs.system()]) + ' (2)', 'vs-invalid');
         // }
@@ -2232,7 +2229,7 @@ class ValidateWorker extends TerminologyWorker {
       if (csp) {
         return csp;
       } else {
-        let vs = await this.findValueSet(url, version);
+        let vs = await this.findValueSet(url, version, null);
         if (vs) {
           let msg = this.i18n.translate('Terminology_TX_System_ValueSet2', txParams.HTTPLanguages, [url]);
           throw new Issue('error', 'invalid', path, 'Terminology_TX_System_ValueSet2', msg, 'invalid-data');
@@ -2448,22 +2445,22 @@ class ValidateWorker extends TerminologyWorker {
     return defaultValue;
   }
 
-  /**
-   * Find a ValueSet by URL
-   * @param {string} url - ValueSet URL
-   * @param {string} [version] - ValueSet version
-   * @returns {Object|null} ValueSet resource or null
-   */
-  async findValueSet(url, version = null) {
-    // First check additional resources
-    const found = this.findInAdditionalResources(url, version || '', 'ValueSet', false);
-    if (found) {
-      return found;
-    }
-
-    // Then check provider
-    return await this.provider.findValueSet(this.opContext, url, version);
-  }
+  // /**
+  //  * Find a ValueSet by URL
+  //  * @param {string} url - ValueSet URL
+  //  * @param {string} [version] - ValueSet version
+  //  * @returns {Object|null} ValueSet resource or null
+  //  */
+  // async findValueSet(url, version = null) {
+  //   // First check additional resources
+  //   const found = this.findInAdditionalResources(url, version || '', 'ValueSet', false);
+  //   if (found) {
+  //     return found;
+  //   }
+  //
+  //   // Then check provider
+  //   return await this.provider.findValueSet(this.opContext, url, version);
+  // }
 
   /**
    * Get display text for a code (stub implementation for doValidationCS)