fhirsmith 0.7.4 → 0.7.6

package/tx/importers/readme.md

@@ -1,6 +1,8 @@
 # TX-Import: Medical Terminology Import Tool
 
-A comprehensive CLI tool for importing various medical terminology standards into SQLite databases and other formats. The tool supports LOINC, SNOMED CT, UNII, NDC, and provides extensible architecture for additional terminologies.
+A comprehensive CLI tool for importing various medical terminology standards into SQLite databases and other formats
+for use by FHIRsmith. The tool supports LOINC, SNOMED CT, UNII, NDC, and provides extensible architecture for
+additional terminologies.
 
 ## Table of Contents
 

package/tx/library.js

@@ -65,7 +65,8 @@ class Library {
    */
   conceptMapProviders;
 
-  contentSources = [];
+  packageSources = [];
+  externalSources = [];
 
   baseUrl = null;
   cacheFolder = null;
@@ -101,6 +102,7 @@ class Library {
     this.conceptMapProviders = [];
     this.oclProviderSets = new Map();
     this.oclConfig = {};
+    this.ignored = new Set();
 
     // Create package manager for FHIR packages
     const packageServers = ['https://packages2.fhir.org/packages'];
@@ -161,6 +163,7 @@ class Library {
     const config = yaml.parse(yamlContent);
     this.baseUrl = config.base.url;
     this.oclConfig = config.ocl && typeof config.ocl === 'object' ? config.ocl : {};
+    this.ignored = new Set(Array.isArray(config.ignored) ? config.ignored : []);
 
     this.log.info('Fetching Data from '+this.baseUrl);
 
@@ -288,7 +291,7 @@ class Library {
       case 'ocl':
         await this.loadOcl(details, isDefault, mode);
         break;
-        
+
       default:
         throw new Error(`Unknown source type: ${type}`);
     }
@@ -358,6 +361,9 @@ class Library {
       const codeSystemProvider = new OCLCodeSystemProvider(config);
       const valueSetProvider = new OCLValueSetProvider(config);
       const conceptMapProvider = new OCLConceptMapProvider(config);
+      this.externalSources.push(codeSystemProvider);
+      this.externalSources.push(valueSetProvider);
+      this.externalSources.push(conceptMapProvider);
       providerSet = {
         config,
         codeSystemProvider,
@@ -468,6 +474,7 @@ class Library {
         let vsac = new VSACValueSetProvider(this.vsacCfg, this.stats);
         vsac.initialize();
         this.valueSetProviders.push(vsac);
+        this.externalSources.push(vsac);
         //const mem = process.memoryUsage();
         let time = Math.floor(Date.now() - this.lastTime).toString().padStart(5)+" ";
         let system = "vsac".padEnd(50);
@@ -567,6 +574,17 @@ class Library {
     this.registerProvider(omopFN, omop, isDefault);
   }
 
+  /**
+   * Returns true if the given url/version should be excluded from npm/url package loading.
+   * Matches against the ignored list using either plain url or url#version.
+   */
+  #isIgnored(url, version) {
+    if (this.ignored.size === 0) return false;
+    if (this.ignored.has(url)) return true;
+    if (version && this.ignored.has(`${url}#${version}`)) return true;
+    return false;
+  }
+
   async loadNpm(packageManager, details, isDefault, mode, csOnly) {
     // Parse packageId and version from details (e.g., "hl7.terminology.r4#6.0.2")
     let packageId = details;
@@ -584,13 +602,17 @@ class Library {
     const contentLoader = new PackageContentLoader(fullPackagePath);
     await contentLoader.initialize();
 
-    this.contentSources.push(contentLoader.id()+"#"+contentLoader.version());
+    this.packageSources.push(contentLoader.id()+"#"+contentLoader.version());
 
     let cp = new ListCodeSystemProvider();
     const resources = await contentLoader.getResourcesByType("CodeSystem");
     let csc = 0;
     for (const resource of resources) {
       const cs = new CodeSystem(await contentLoader.loadFile(resource, contentLoader.fhirVersion()));
+      if (this.#isIgnored(cs.url, cs.version)) {
+        this.log.info(`Ignoring CodeSystem ${cs.url}${cs.version ? '#' + cs.version : ''} (excluded by config)`);
+        continue;
+      }
       cs.sourcePackage = contentLoader.pid();
       cp.codeSystems.push(cs);
       csc++;
@@ -618,13 +640,17 @@ class Library {
     const contentLoader = new PackageContentLoader(fullPackagePath);
     await contentLoader.initialize();
 
-    this.contentSources.push(contentLoader.id()+"#"+contentLoader.version());
+    this.packageSources.push(contentLoader.id()+"#"+contentLoader.version());
 
     let cp = new ListCodeSystemProvider();
     const resources = await contentLoader.getResourcesByType("CodeSystem");
     let csc = 0;
     for (const resource of resources) {
       const cs = new CodeSystem(await contentLoader.loadFile(resource, contentLoader.fhirVersion()));
+      if (this.#isIgnored(cs.url, cs.version)) {
+        this.log.info(`Ignoring CodeSystem ${cs.url}${cs.version ? '#' + cs.version : ''} (excluded by config)`);
+        continue;
+      }
       cs.sourcePackage = contentLoader.pid();
       cp.codeSystems.set(cs.url, cs);
       cp.codeSystems.set(cs.vurl, cs);
@@ -826,7 +852,8 @@ class Library {
     provider.lastTime = this.lastTime;
     provider.lastMemory = this.lastMemory;
     provider.totalDownloaded = this.totalDownloaded;
-    provider.contentSources = this.contentSources;
+    provider.packageSources = this.packageSources;
+    provider.externalSources = this.externalSources;
 
 
     // Now add the existing value set providers after the FHIR core packages
@@ -905,4 +932,4 @@ class Library {
 
 }
 
-module.exports = { Library };
+module.exports = { Library };

package/tx/provider.js

@@ -49,7 +49,8 @@ class Provider {
    */
   conceptMapProviders;
 
-  contentSources;
+  packageSources;
+  externalSources;
 
   baseUrl = null;
   path;

package/tx/tx-html.js

@@ -234,24 +234,44 @@ class TxHtmlRenderer {
     html += await this.buildSearchForm(req);
 
     // ===== Packages and Factories Section =====
-    html += '<hr/><h3>Content Sources &amp; Code System Factories</h3>';
+    html += '<hr/><h3>Source Content</h3>';
 
-    // List content sources
-    html += '<h6>Content Sources</h6>';
-    if (provider.contentSources && provider.contentSources.length > 0) {
-      const sorted = [...provider.contentSources].sort();
+    // List Packages
+    html += '<h6>FHIR Packages</h6>';
+    if (provider.packageSources && provider.packageSources.length > 0) {
+      const sorted = [...provider.packageSources].sort();
       html += '<ul>';
       for (const source of sorted) {
         html += `<li>${escape(source)}</li>`;
       }
       html += '</ul>';
     } else {
-      html += '<p><em>No content sources available</em></p>';
+      html += '<p><em>No FHIR Packages Loaded</em></p>';
     }
 
-    // Code System Factories table
-// Code System Factories table
-    html += '<h6 class="mt-4">External CodeSystems</h6>';
+    // List Packages
+    html += '<h6>External Sources</h6>';
+    if (provider.externalSources && provider.externalSources.length > 0) {
+      const sorted = [...provider.externalSources].sort();
+      html += '<ul>';
+      for (const source of sorted) {
+        let n = source.name();
+        if (!n) {
+          n = source.sourcePackage();
+        }
+        let ii = source.infoName();
+        if (ii) {
+          html += `<li>${escape(n)} (<a href="info/${source.id()}">${ii}</a>)</li>`;
+        } else {
+          html += `<li>${escape(n)}</li>`;
+        }
+      }
+      html += '</ul>';
+    } else {
+      html += '<p><em>No External Sources Configured</em></p>';
+    }
+
+    html += '<h6 class="mt-4">Special CodeSystems</h6>';
     html += '<table class="grid">';
     html += '<thead><tr><th>Name</th><th>URI</th><th>Version</th><th>Use Count</th></tr></thead>';
     html += '<tbody>';
@@ -1290,6 +1310,13 @@ class TxHtmlRenderer {
     });
   }
 
+  async buildInfoPage(source, req) {
+    let html = '';
+    const infoContent = await source.info(req);
+    html += infoContent;
+    return html;
+  }
+
   buildSourceOptions(provider) {
     let result = '<option value=""></option>';
     result += `<option value="internal">internal</option>`;

package/tx/tx.fhir.org.yml

@@ -40,3 +40,6 @@ sources:
   - npm:us.cdc.phinvads
   - npm:hl7.fhir.uv.sdc
   - internal:vsac
+
+ignored:
+  - urn:iso:std:iso:3166#20210120

package/tx/tx.js

@@ -9,7 +9,7 @@ const express = require('express');
 const path = require('path');
 const Logger = require('../library/logger');
 const { Library } = require('./library');
-const { OperationContext, ResourceCache, ExpansionCache } = require('./operation-context');
+const { OperationContext, ResourceCache, ExpansionCache, debugLog} = require('./operation-context');
 const { LanguageDefinitions } = require('../library/languages');
 const { I18nSupport } = require('../library/i18nsupport');
 const { CodeSystemXML } = require('./xml/codesystem-xml');
@@ -965,6 +965,29 @@ class TXModule {
         this.countRequest('home', Date.now() - start);
       }
     });
+
+    // External source info pages
+    router.get('/info/:id', async (req, res) => {
+      const start = Date.now();
+      try {
+        const source = req.txEndpoint.provider.externalSources.find(s => s.id() === req.params.id);
+        if (!source) {
+          res.status(404).send('Not found');
+          return;
+        }
+        let txhtml = new TxHtmlRenderer(new Renderer(req.txOpContext, req.txEndpoint.provider), this.liquid, this.languages, this.i18n, req.txEndpoint.path);
+        const content = await txhtml.buildInfoPage(source, req);
+        const html = await txhtml.renderPage(source.name(), content, req.txEndpoint, start);
+        res.setHeader('Content-Type', 'text/html');
+        res.send(html);
+      } catch (error) {
+        debugLog(error);
+        this.log.error(`Error rendering info page for ${req.params.id}: ${error.message}`);
+        res.status(500).send('Internal server error');
+      } finally {
+        this.countRequest('info', Date.now() - start);
+      }
+    });
   }
 
   /**
@@ -1153,16 +1176,16 @@ class TXModule {
   ec = 0;
 
   checkProperJson() { // jsonStr) {
-                      //   const errors = [];
-                      //   if (jsonStr.includes("[]")) errors.push("Found [] in json");
-                      //   if (jsonStr.includes('""')) errors.push('Found "" in json');
-                      //
-                      //   if (errors.length > 0) {
-                      //     this.ec++;
-                      //     const filename = `/Users/grahamegrieve/temp/tx-err-log/err${this.ec}.json`;
-                      //     writeFileSync(filename, jsonStr);
-                      //     throw new Error(errors.join('; '));
-                      //   }
+    //   const errors = [];
+    //   if (jsonStr.includes("[]")) errors.push("Found [] in json");
+    //   if (jsonStr.includes('""')) errors.push('Found "" in json');
+    //
+    //   if (errors.length > 0) {
+    //     this.ec++;
+    //     const filename = `/Users/grahamegrieve/temp/tx-err-log/err${this.ec}.json`;
+    //     writeFileSync(filename, jsonStr);
+    //     throw new Error(errors.join('; '));
+    //   }
   }
 
   transformResourceForVersion(data, fhirVersion) {

package/tx/vs/vs-database.js

@@ -22,6 +22,27 @@ class ValueSetDatabase {
     this._writeDb = null;     // Write connection (opened only when needed)
   }
 
+  /**
+   * Apply any pending schema migrations
+   * @param {sqlite3.Database} db
+   * @returns {Promise<void>}
+   * @private
+   */
+  _migrateIfNeeded(db) {
+    return new Promise((resolve, reject) => {
+      db.all("PRAGMA table_info(valuesets)", [], (err, cols) => {
+        if (err) { reject(err); return; }
+        const hasCol = cols.some(c => c.name === 'date_first_seen');
+        if (hasCol) { resolve(); return; }
+        db.run(
+          "ALTER TABLE valuesets ADD COLUMN date_first_seen INTEGER DEFAULT 0",
+          [],
+          (err) => err ? reject(err) : resolve()
+        );
+      });
+    });
+  }
+
   /**
    * Get a read-only database connection (opens lazily if needed)
    * @returns {Promise<sqlite3.Database>}
@@ -62,7 +83,7 @@ class ValueSetDatabase {
           this._writeDb = null;
           reject(new Error(`Failed to open database for writing: ${err.message}`));
         } else {
-          resolve(this._writeDb);
+          this._migrateIfNeeded(this._writeDb).then(() => resolve(this._writeDb)).catch(reject);
         }
       });
     });
@@ -144,7 +165,8 @@ class ValueSetDatabase {
                                          status TEXT,
                                          title TEXT,
                                          content TEXT NOT NULL,
-                                         last_seen INTEGER DEFAULT (strftime('%s', 'now'))
+                                         last_seen INTEGER DEFAULT (strftime('%s', 'now')),
+                                         date_first_seen INTEGER DEFAULT (strftime('%s', 'now'))
               )
           `);
 
@@ -190,6 +212,7 @@ class ValueSetDatabase {
           db.run('CREATE INDEX idx_valuesets_title ON valuesets(title)');
           db.run('CREATE INDEX idx_valuesets_publisher ON valuesets(publisher)');
           db.run('CREATE INDEX idx_valuesets_last_seen ON valuesets(last_seen)');
+          db.run('CREATE INDEX idx_valuesets_date_first_seen ON valuesets(date_first_seen)');
           db.run('CREATE INDEX idx_identifiers_system ON valueset_identifiers(system)');
           db.run('CREATE INDEX idx_identifiers_value ON valueset_identifiers(value)');
           db.run('CREATE INDEX idx_jurisdictions_system ON valueset_jurisdictions(system)');
@@ -246,10 +269,24 @@ class ValueSetDatabase {
             const expansionId = valueSet.expansion?.identifier || null;
 
             db.run(`
-                INSERT OR REPLACE INTO valuesets (
+                INSERT INTO valuesets (
                 id, url, version, date, description, effectivePeriod_start, effectivePeriod_end,
-                expansion_identifier, name, publisher, status, title, content, last_seen
-              ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, strftime('%s', 'now'))
+                expansion_identifier, name, publisher, status, title, content, last_seen, date_first_seen
+              ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, strftime('%s', 'now'), strftime('%s', 'now'))
+              ON CONFLICT(id) DO UPDATE SET
+                url=excluded.url,
+                version=excluded.version,
+                date=excluded.date,
+                description=excluded.description,
+                effectivePeriod_start=excluded.effectivePeriod_start,
+                effectivePeriod_end=excluded.effectivePeriod_end,
+                expansion_identifier=excluded.expansion_identifier,
+                name=excluded.name,
+                publisher=excluded.publisher,
+                status=excluded.status,
+                title=excluded.title,
+                content=excluded.content,
+                last_seen=strftime('%s', 'now')
             `, [
               valueSet.id,
               valueSet.url,

package/tx/vs/vs-vsac.js

@@ -70,6 +70,8 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
     if (!(await this.database.exists())) {
       await this.database.create();
     } else {
+      // Ensure schema is up to date (e.g. date_first_seen column added after initial deploy)
+      await this.database._migrateIfNeeded(await this.database._getWriteConnection());
       // Load existing data
       await this._reloadMap();
     }
@@ -511,6 +513,52 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
     this.stats.task('VSAC Sync', logMsg);
 
   }
+
+  name() {
+    return "VSAC";
+  }
+
+  infoName() {
+    return "history";
+  }
+
+  async info() {
+    const db = await this.database._getReadConnection();
+    const rows = await new Promise((resolve, reject) => {
+      db.all(
+        `SELECT url, version, date_first_seen
+           FROM valuesets
+          WHERE date_first_seen > 0
+          ORDER BY date_first_seen DESC
+          LIMIT 100`,
+        [],
+        (err, rows) => err ? reject(err) : resolve(rows)
+      );
+    });
+
+    const escape = require('escape-html');
+    let html = '<h3>Recently Value Sets Added to VSAC</h3>';
+    html += '<p>The last ' + rows.length + ' value sets found from VSAC, most recent first.</p>';
+    html += '<table class="grid">';
+    html += '<thead><tr><th>URL</th><th>Version</th><th>Date Observed</th></tr></thead>';
+    html += '<tbody>';
+    for (const row of rows) {
+      const date = row.date_first_seen
+        ? new Date(row.date_first_seen * 1000).toISOString().replace('T', ' ').substring(0, 19) + ' UTC'
+        : 'unknown';
+      html += '<tr>';
+      html += `<td>${escape(row.url || '')}</td>`;
+      html += `<td>${escape(row.version || '')}</td>`;
+      html += `<td>${escape(date)}</td>`;
+      html += '</tr>';
+    }
+    html += '</tbody></table>';
+    return html;
+  }
+
+  id() {
+    return "vsac";
+  }
 }
 
 // Usage examples:

package/tx/workers/validate.js

@@ -1389,17 +1389,22 @@ class ValueSetChecker {
     let hd = list.hasDisplay(this.params.workingLanguages(), null, c.display, false, DisplayCheckingStyle.CASE_INSENSITIVE)
     if (!hd.found) {
       let baseMsg;
-      if (hd.difference === DisplayDifference.Normalized) {
+      let severity = this.dispWarning();
+      if (list.userDefined) {
+        baseMsg = 'Display_Name_Not_Fixed_use_Supplement';
+        severity = 'information';
+      } else if (hd.difference === DisplayDifference.Normalized) {
         baseMsg = 'Display_Name_WS_for__should_be_one_of__instead_of';
       } else {
         baseMsg = 'Display_Name_for__should_be_one_of__instead_of';
       }
-      let mid = baseMsg;
       let dc = list.displayCount(this.params.workingLanguages(), null, true);
-      let severity = this.dispWarning();
-      if (dc === 0) {
-        severity = 'warning';
-        dc = list.displayCount(this.params.workingLanguages(), null, false);
+      let mid = baseMsg;
+      if (severity !== 'information') {
+        if (dc === 0) {
+          severity = 'warning';
+          dc = list.displayCount(this.params.workingLanguages(), null, false);
+        }
       }
 
       let m, ds;

package/tx/workers/worker.js

@@ -276,12 +276,9 @@ class TerminologyWorker {
         if (this.hasSupplement(cs, supplements)) {
           continue;
         }
-        // Handle exact URL match (no version specified in supplements)
+        // Handle exact URL match (no version specified in supplements field)
         if (supplementsUrl === url) {
-          // If we're looking for a specific version, only include if no version in supplements URL
-          if (!version) {
-            supplements.push(cs);
-          }
+          supplements.push(cs);
           continue;
         }