fhirsmith 0.9.0 → 0.9.2

package/tx/vs/vs-vsac.js

@@ -1,4 +1,5 @@
 const path = require('path');
+const crypto = require('crypto');
 const axios = require('axios');
 const { AbstractValueSetProvider } = require('./vs-api');
 const { ValueSetDatabase } = require('./vs-database');
@@ -11,12 +12,15 @@ const {debugLog} = require("../operation-context");
  * Fetches and caches ValueSets from the NLM VSAC FHIR server
  */
 class VSACValueSetProvider extends AbstractValueSetProvider {
+  SYNC_AT_START_UP = false;
+
   /**
    * @param {Object} config - Configuration object
    * @param {string} config.apiKey - API key for VSAC authentication
    * @param {string} config.cacheFolder - Local folder for cached database
    * @param {number} [config.refreshIntervalHours=24] - Hours between refresh scans
    * @param {string} [config.baseUrl='http://cts.nlm.nih.gov/fhir'] - Base URL for VSAC FHIR server
+   * @param {number} [config.timeoutMs=120000] - HTTP request timeout in milliseconds
    */
   constructor(config, stats) {
     super();
@@ -43,7 +47,7 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
     const authString = Buffer.from(`apikey:${this.apiKey}`).toString('base64');
     this.httpClient = axios.create({
       baseURL: this.baseUrl,
-      timeout: 30000,
+      timeout: config.timeoutMs || 120000,
       headers: {
         'Accept': 'application/fhir+json',
         'User-Agent': 'FHIR-ValueSet-Provider/1.0',
@@ -70,12 +74,11 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
     if (!(await this.database.exists())) {
       await this.database.create();
     } else {
-      // Ensure schema is up to date (e.g. date_first_seen column added after initial deploy)
-      await this.database._migrateIfNeeded(await this.database._getWriteConnection());
-      // Load existing data
+      // Schema migrations are applied lazily by the database layer on first
+      // connection. Just load existing data.
       await this._reloadMap();
     }
-    if (this.valueSetMap.size == 0) {
+    if (this.SYNC_AT_START_UP || this.valueSetMap.size == 0) {
       await this.refreshValueSets();
     }
     // Start periodic refresh
@@ -181,11 +184,11 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
       // deduplicate the queue
       this.queue = [...new Set(this.queue)];
 
-      let tracking = { totalFetched: 0, totalNew: 0, count: 0, newCount : 0 };
+      let tracking = { totalFetched: 0, totalNew: 0, totalUpdated: 0, count: 0, newCount : 0 };
       // phase 2: query for history & content
       this.requeue = [];
       for (let q of this.queue) {
-        this.stats.task('VSAC History for '+q, `running (${tracking.totalFetched} fetched, ${tracking.totalNew} new)`);
+        this.stats.task('VSAC History for '+q, `running (${tracking.totalFetched} fetched, ${tracking.totalNew} new, ${tracking.totalUpdated} updated)`);
         try {
           await this.processContentAndHistory(q, tracking, this.queue.length);
         } catch (error) {
@@ -193,29 +196,27 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
           debugLog(error);
           this.stats.task('VSAC Sync', error.message);
         }
-        // `running (${totalFetched} fetched, ${totalNew} new)`)
         tracking.count++;
       }
       console.log("Requeue");
       for (let q of this.requeue) {
-        this.stats.task('VSAC History for '+q, `running (${tracking.totalFetched} fetched, ${tracking.totalNew} new)`);
+        this.stats.task('VSAC History for '+q, `running (${tracking.totalFetched} fetched, ${tracking.totalNew} new, ${tracking.totalUpdated} updated)`);
         try {
           await this.processContentAndHistory(q, tracking, this.requeue.length);
         } catch (error) {
           debugLog(error);
           this.stats.task('VSAC Sync', error.message);
         }
-        // `running (${totalFetched} fetched, ${totalNew} new)`)
         tracking.count++;
       }
 
       // Reload map with fresh data
       await this._reloadMap();
-      let msg = `VSAC refresh completed. Total: ${tracking.totalFetched} ValueSets, Deleted: ${tracking.deletedCount}`;
+      let msg = `VSAC refresh completed. Total: ${tracking.totalFetched} ValueSets, New: ${tracking.totalNew}, Updated: ${tracking.totalUpdated}`;
       this.stats.taskDone('VSAC Sync', msg);
       console.log(msg);
 
-      await this.database.finishRun(runId, tracking.totalFetched, tracking.totalNew);
+      await this.database.finishRun(runId, tracking.totalFetched, tracking.totalNew, tracking.totalUpdated);
     } catch (error) {
       debugLog(error, 'Error during VSAC refresh:');
       this.stats.taskError('VSAC Sync', `Error (${error.message})`);
@@ -227,30 +228,71 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
   }
 
   /**
-   * Insert multiple ValueSets in a batch operation
+   * Compute a SHA-256 hash of the ValueSet content for change detection.
+   * @param {Object} vs - The ValueSet resource (plain JSON object)
+   * @returns {string} hex-encoded SHA-256
+   * @private
+   */
+  _hashValueSet(vs) {
+    return crypto.createHash('sha256').update(JSON.stringify(vs)).digest('hex');
+  }
+
+  /**
+   * Insert multiple ValueSets in a batch operation.
+   * For each value set: if url|version is already known, compare content hashes.
+   *   - hash unchanged  -> touch last_seen only (seeValueSet)
+   *   - hash changed    -> upsert and record an 'updated' event
+   *   - not seen before -> upsert and record a 'new' event
    * @param {Array<Object>} valueSets - Array of ValueSet resources
-   * @returns {Promise<void>}
+   * @returns {Promise<{newCount: number, updatedCount: number}>}
    */
   async batchUpsertValueSets(valueSets) {
     if (valueSets.length === 0) {
-      return;
+      return { newCount: 0, updatedCount: 0 };
     }
 
-    let count = 0;
+    let newCount = 0;
+    let updatedCount = 0;
+
     // Process sequentially to avoid database locking
     for (const valueSet of valueSets) {
-      let key = valueSet.url+"|"+valueSet.version;
-      let vs = this.valueSetMap.get(key);
-      if (vs) {
-        // we've seen this before, and maybe fetched it's history, so just update
-        // the timestamp
-        await this.database.seeValueSet(valueSet);
+      const key = valueSet.url+"|"+valueSet.version;
+      const existing = this.valueSetMap.get(key);
+      const newHash = this._hashValueSet(valueSet);
+
+      if (existing) {
+        // We've seen this url|version before. Decide whether the content
+        // has actually changed by comparing hashes.
+        //
+        // Note: _reloadMap() mutates the in-memory jsonObj (strips inc.version
+        // from compose.include/exclude), so we cannot reliably recompute a
+        // hash from existing.jsonObj — it would not match the hash of the
+        // original unmutated JSON we stored. For rows predating this feature
+        // (content_hash NULL), we defer update detection until the next cycle:
+        // the upsert below runs only when hashes differ, so on the *next*
+        // sync after migration we'll have a proper baseline.
+        if (existing.contentHash && existing.contentHash === newHash) {
+          // No change - just touch last_seen
+          await this.database.seeValueSet(valueSet);
+        } else if (!existing.contentHash) {
+          // Legacy row without a stored hash - backfill the hash silently
+          // without emitting a spurious 'updated' event. We do a lightweight
+          // touch + hash update rather than a full upsert+event.
+          await this.database.seeValueSet(valueSet);
+          await this.database.setContentHash(valueSet.id, newHash);
+        } else {
+          // Content has changed - treat as update
+          await this.database.upsertValueSet(valueSet, newHash);
+          await this.database.recordEvent('updated', valueSet.url, valueSet.version);
+          updatedCount++;
+        }
       } else {
-        await this.database.upsertValueSet(valueSet);
-        count++;
+        await this.database.upsertValueSet(valueSet, newHash);
+        await this.database.recordEvent('new', valueSet.url, valueSet.version);
+        newCount++;
       }
     }
-    return count;
+    return { newCount, updatedCount };
   }
 
   /**
@@ -510,18 +552,21 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
     const bundle = await this._fetchBundle(url);
 
     let vcount = 0;
+    let perRun = { newCount: 0, updatedCount: 0 };
     if (bundle.entry && bundle.entry.length > 0) {
       // Extract ValueSets from bundle entries
       const valueSets = bundle.entry
           .filter(entry => entry.resource && entry.resource.resourceType === 'ValueSet')
           .map(entry => entry.resource);
       if (valueSets.length > 0) {
-        tracking.totalNew = tracking.totalNew + await this.batchUpsertValueSets(valueSets);
+        perRun = await this.batchUpsertValueSets(valueSets);
+        tracking.totalNew += perRun.newCount;
+        tracking.totalUpdated += perRun.updatedCount;
         tracking.totalFetched += valueSets.length;
         vcount = valueSets.length;
       }
     }
-    let logMsg = `VSAC (${tracking.count} of ${length}) ${q}: ${vcount} versions`;
+    let logMsg = `VSAC (${tracking.count} of ${length}) ${q}: ${vcount} versions (${perRun.newCount} new, ${perRun.updatedCount} updated)`;
     console.log(logMsg);
     this.stats.task('VSAC Sync', logMsg);
   }
@@ -592,30 +637,33 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
 
     const rows = await new Promise((resolve, reject) => {
       db.all(
-          `SELECT 'vs' AS kind,
+          `SELECT 'event' AS kind,
                   url,
                   version,
-                  date_first_seen AS ts,
-                  NULL AS status,
-                  NULL AS error_message,
-                  NULL AS finished_at,
-                  NULL AS total_fetched,
-                  NULL AS total_new
-           FROM valuesets
-           WHERE date_first_seen > 0
+             timestamp AS ts,
+             event_type,
+             NULL AS status,
+             NULL AS error_message,
+             NULL AS finished_at,
+             NULL AS total_fetched,
+             NULL AS total_new,
+             NULL AS total_updated
+           FROM vsac_events
            UNION ALL
-           SELECT 'run' AS kind,
-                  NULL,
-                  NULL,
-                  started_at AS ts,
-                  status,
-                  error_message,
-                  finished_at,
-                  total_fetched,
-                  total_new
-           FROM vsac_runs
-           ORDER BY ts DESC
-             LIMIT 200`,
+          SELECT 'run' AS kind,
+                 NULL,
+                 NULL,
+                 started_at AS ts,
+                 NULL AS event_type,
+                 status,
+                 error_message,
+                 finished_at,
+                 total_fetched,
+                 total_new,
+                 total_updated
+          FROM vsac_runs
+          ORDER BY ts DESC
+            LIMIT 200`,
           [],
           (err, rows) => err ? reject(err) : resolve(rows)
       );
@@ -635,7 +683,8 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
         const duration = row.finished_at ? `${row.finished_at - row.ts}s` : 'in progress';
         let detail, colour;
         if (row.status === 'ok') {
-          detail = `${row.total_fetched} fetched, ${row.total_new} new, ${duration}`;
+          const updated = row.total_updated != null ? `, ${row.total_updated} updated` : '';
+          detail = `${row.total_fetched} fetched, ${row.total_new} new${updated}, ${duration}`;
           colour = 'green';
         } else if (row.status === 'error') {
           detail = `Failed: ${escape(row.error_message || '')} (${duration})`;
@@ -650,9 +699,28 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
         html += `<td>${detail}</td>`;
         html += `</tr>`;
       } else {
+        // Event row: 'new', 'updated', or 'deleted'
+        let label, colour;
+        switch (row.event_type) {
+          case 'new':
+            label = 'New value set';
+            colour = 'green';
+            break;
+          case 'updated':
+            label = 'Updated value set';
+            colour = 'blue';
+            break;
+          case 'deleted':
+            label = 'Deleted value set';
+            colour = 'red';
+            break;
+          default:
+            label = escape(row.event_type || 'Event');
+            colour = 'black';
+        }
         html += `<tr>`;
         html += `<td>${escape(fmt(row.ts))}</td>`;
-        html += `<td>New value set</td>`;
+        html += `<td><span style="color:${colour}">${label}</span></td>`;
         html += `<td>${escape(row.url || '')}#${escape(row.version || '')}</td>`;
         html += `</tr>`;
       }

package/tx/workers/validate.js

@@ -839,7 +839,7 @@ class ValueSetChecker {
               } else {
                 bAdd = !unknownSystems.has(system + '|' + version);
                 if (bAdd) {
-                  let vl = await this.listVersions(system);
+                  let vl = await this.worker.listVersions(system);
                   if (vl.length == 0) {
                     mid = 'UNKNOWN_CODESYSTEM_VERSION_NONE';
                     vn = system;

package/tx/xversion/xv-valueset.js

@@ -1,5 +1,6 @@
 const {VersionUtilities} = require("../../library/version-utilities");
 const {getValueName} = require("../../library/utilities");
+const {Extensions} = require("../library/extensions");
 
 /**
  * Converts input ValueSet to R5 format (modifies input object for performance)
@@ -13,6 +14,12 @@ function valueSetToR5(jsonObj, sourceVersion) {
   if (VersionUtilities.isR5Ver(sourceVersion)) {
     return jsonObj; // No conversion needed
   }
+  for (const inc of jsonObj.compose.include || []) {
+    valueSetIncludeToR5(inc);
+  }
+  for (const inc of jsonObj.compose.exclude || []) {
+    valueSetIncludeToR5(inc);
+  }
   if (VersionUtilities.isR4Ver(sourceVersion)) {
     return jsonObj; // No conversion needed
   }
@@ -26,6 +33,19 @@ function valueSetToR5(jsonObj, sourceVersion) {
   throw new Error(`Unsupported FHIR version: ${sourceVersion}`);
 }
 
+function valueSetIncludeToR5(inc) {
+  for (const filter of inc.filter || []) {
+    if (filter._op) {
+      let code = Extensions.readString(filter._op, 'http://hl7.org/fhir/5.0/StructureDefinition/extension-ValueSet.compose.include.filter.op');
+      if (code) {
+        filter.op = code;
+        delete filter._op;
+      }
+    }
+  }
+}
+
+
 /**
  * Converts R5 ValueSet to target version format (clones object first)
  * @param {Object} r5Obj - The R5 format ValueSet object
@@ -70,8 +90,8 @@ function valueSetR5ToR4(r5Obj) {
       if (include.filter && Array.isArray(include.filter)) {
         include.filter = include.filter.map(filter => {
           if (filter.op && isR5OnlyFilterOperator(filter.op)) {
-            // Remove R5-only operators
-            return null;
+            filter._op = { "extension": "http://hl7.org/fhir/5.0/StructureDefinition/extension-ValueSet.compose.include.filter.op", "valueCode": filter.op}
+            delete filter.op;
           }
           return filter;
         }).filter(filter => filter !== null);
@@ -85,8 +105,8 @@ function valueSetR5ToR4(r5Obj) {
       if (exclude.filter && Array.isArray(exclude.filter)) {
         exclude.filter = exclude.filter.map(filter => {
           if (filter.op && isR5OnlyFilterOperator(filter.op)) {
-            // Remove R5-only operators
-            return null;
+            filter._op = { "extension": "http://hl7.org/fhir/5.0/StructureDefinition/extension-ValueSet.compose.include.filter.op", "valueCode": filter.op}
+            delete filter.op;
           }
           return filter;
         }).filter(filter => filter !== null);
@@ -135,8 +155,8 @@ function valueSetR5ToR3(r5Obj) {
       if (include.filter && Array.isArray(include.filter)) {
         include.filter = include.filter.map(filter => {
           if (filter.op && !isR3CompatibleFilterOperator(filter.op)) {
-            // Remove non-R3-compatible operators
-            return null;
+            filter._op = { "extension": "http://hl7.org/fhir/5.0/StructureDefinition/extension-ValueSet.compose.include.filter.op", "valueCode": filter.op}
+            delete filter.op;
           }
           return filter;
         }).filter(filter => filter !== null);
@@ -150,8 +170,8 @@ function valueSetR5ToR3(r5Obj) {
       if (exclude.filter && Array.isArray(exclude.filter)) {
         exclude.filter = exclude.filter.map(filter => {
           if (filter.op && !isR3CompatibleFilterOperator(filter.op)) {
-            // Remove non-R3-compatible operators
-            return null;
+            filter._op = { "extension": "http://hl7.org/fhir/5.0/StructureDefinition/extension-ValueSet.compose.include.filter.op", "valueCode": filter.op}
+            delete filter.op;
           }
           return filter;
         }).filter(filter => filter !== null);