fhirsmith 0.5.6 → 0.7.0

package/registry/crawler.js

@@ -8,6 +8,8 @@ const {
   ServerInformation, 
   ServerVersionInformation,
 } = require('./model');
+const {Extensions} = require("../tx/library/extensions");
+const {debugLog} = require("../tx/operation-context");
 
 const MASTER_URL = 'https://fhir.github.io/ig-registry/tx-servers.json';
 
@@ -30,38 +32,13 @@ class RegistryCrawler {
     this.errors = [];
     this.totalBytes = 0;
     this.log = console;
+    this.abortController = null;
   }
 
   useLog(logv) {
     this.log = logv;
   }
 
-  // /**
-  //  * Start the crawler with periodic updates
-  //  */
-  // start() {
-  //   if (this.crawlTimer) {
-  //     return; // Already running
-  //   }
-  //
-  //   // Initial crawl
-  //   this.crawl();
-  //
-  //   // Set up periodic crawling
-  //   this.crawlTimer = setInterval(() => {
-  //     this.crawl();
-  //   }, this.config.crawlInterval);
-  // }
-  //
-  // /**
-  //  * Stop the crawler
-  //  */
-  // stop() {
-  //   if (this.crawlTimer) {
-  //     clearInterval(this.crawlTimer);
-  //     this.crawlTimer = null;
-  //   }
-  // }
 
   /**
    * Main entry point - crawl the registry starting from the master URL
@@ -73,6 +50,7 @@ class RegistryCrawler {
       this.addLogEntry('warn', 'Crawl already in progress, skipping...');
       return this.currentData;
     }
+    this.abortController = new AbortController();
 
     this.isCrawling = true;
     const startTime = new Date();
@@ -100,6 +78,7 @@ class RegistryCrawler {
       // Process each registry
       const registries = masterJson.registries || [];
       for (const registryConfig of registries) {
+        if (this.abortController?.signal.aborted) break;
         const registry = await this.processRegistry(registryConfig);
         if (registry) {
           newData.registries.push(registry);
@@ -111,6 +90,7 @@ class RegistryCrawler {
       // Update the current data
       this.currentData = newData;
     } catch (error) {
+      debugLog(error);
       this.addLogEntry('error', 'Exception Scanning:', error);
       this.currentData.outcome = `Error: ${error.message}`;
       this.errors.push({
@@ -142,7 +122,7 @@ class RegistryCrawler {
     }
     
     if (!registry.address) {
-      this.addLogEntry('error', `No url provided for ${registry.name, registry.name}`, '');
+      this.addLogEntry('error', `No url provided for ${registry.name}`, '');
       return registry;
     }
     
@@ -158,6 +138,7 @@ class RegistryCrawler {
       // Process each server in the registry
       const servers = registryJson.servers || [];
       for (const serverConfig of servers) {
+        if (this.abortController?.signal.aborted) break;
         const server = await this.processServer(serverConfig, registry.address);
         if (server) {
           registry.servers.push(server);
@@ -165,6 +146,7 @@ class RegistryCrawler {
       }
       
     } catch (error) {
+      debugLog(error);
       registry.error = error.message;
       this.addLogEntry('error', `Exception processing registry ${registry.name}: ${error.message}`, registry.address);
     }
@@ -181,7 +163,7 @@ class RegistryCrawler {
     server.name = serverConfig.name;
     server.address = serverConfig.url || '';
     server.accessInfo = serverConfig.access_info || '';
-    
+
     if (!server.name) {
       this.addLogEntry('error', 'No name provided for server', source);
       return server;
@@ -200,7 +182,8 @@ class RegistryCrawler {
     // Process each FHIR version
     const fhirVersions = serverConfig.fhirVersions || [];
     for (const versionConfig of fhirVersions) {
-      const version = await this.processServerVersion(versionConfig, server);
+      if (this.abortController?.signal.aborted) break;
+      const version = await this.processServerVersion(versionConfig, server, serverConfig.exclusions);
       if (version) {
         server.versions.push(version);
       }
@@ -212,7 +195,7 @@ class RegistryCrawler {
   /**
    * Process a single server version
    */
-  async processServerVersion(versionConfig, server) {
+  async processServerVersion(versionConfig, server, exclusions) {
     const version = new ServerVersionInformation();
     version.version = versionConfig.version;
     version.address = versionConfig.url;
@@ -233,20 +216,20 @@ class RegistryCrawler {
       
       switch (majorVersion) {
         case 3:
-          await this.processServerVersionR3(version, server);
+          await this.processServerVersionR3(version, server, exclusions);
           break;
         case 4:
-          await this.processServerVersionR4(version, server);
+          await this.processServerVersionR4or5(version, server, '4.0.1', exclusions);
           break;
         case 5:
-          await this.processServerVersionR5(version, server);
+          await this.processServerVersionR4or5(version, server, '5.0.0', exclusions);
           break;
         default:
           throw new Error(`Version ${version.version} not supported`);
       }
       
       // Sort and deduplicate
-      version.codeSystems = [...new Set(version.codeSystems)].sort();
+      version.codeSystems.sort((a, b) => this.compareCS(a, b));
       version.valueSets = [...new Set(version.valueSets)].sort();
       version.lastSuccess = new Date();
       version.lastTat = `${Date.now() - startTime}ms`;
@@ -254,6 +237,7 @@ class RegistryCrawler {
       this.addLogEntry('info', `  Server ${version.address}: ${version.lastTat} for ${version.codeSystems.length} CodeSystems and ${version.valueSets.length} ValueSets`);
       
     } catch (error) {
+      debugLog(error);
       const elapsed = Date.now() - startTime;
       this.addLogEntry('error', `Server ${version.address}: Error after ${elapsed}ms: ${error.message}`);
       version.error = error.message;
@@ -266,7 +250,7 @@ class RegistryCrawler {
   /**
    * Process an R3 server
    */
-  async processServerVersionR3(version, server) {
+  async processServerVersionR3(version, server, exclusions) {
     // Get capability statement
     const capabilityUrl = `${version.address}/metadata`;
     const capability = await this.fetchJson(capabilityUrl, server.name);
@@ -283,12 +267,12 @@ class RegistryCrawler {
         termCap.parameter.forEach(param => {
           if (param.name === 'system') {
             const uri = param.valueUri || param.valueString;
-            if (uri) {
+            if (uri && !this.isExcluded(uri, exclusions)) {
               version.codeSystems.push(uri);
               // Look for version parts
               if (param.part) {
                 param.part.forEach(part => {
-                  if (part.name === 'version' && part.valueString) {
+                  if (part.name === 'version' && part.valueString && !this.isExcluded(uri+'|'+part.valueString, exclusions)) {
                     version.codeSystems.push(`${uri}|${part.valueString}`);
                   }
                 });
@@ -298,24 +282,28 @@ class RegistryCrawler {
         });
       }
     } catch (error) {
-      this.addLogEntry('error', `Could not fetch terminology capabilities: ${error.message}`);
+      debugLog(error);
+      this.addLogEntry('error', `Could not fetch terminology capabilities from ${version.address}: ${error.message}`);
     }
-    
+
+    if (this.abortController?.signal.aborted) return;
     // Search for value sets
-    await this.fetchValueSets(version, server);
+    await this.fetchValueSets(version, server, exclusions);
   }
 
   /**
    * Process an R4 server
    */
-  async processServerVersionR4(version, server) {
+  async processServerVersionR4or5(version, server, defVersion, exclusions) {
     // Get capability statement
     const capabilityUrl = `${version.address}/metadata`;
     const capability = await this.fetchJson(capabilityUrl, server.code);
     
-    version.version = capability.fhirVersion || '4.0.1';
+    version.version = capability.fhirVersion || defVersion;
     version.software = capability.software ? capability.software.name : "unknown";
-    
+
+    let set = new Set();
+
     // Get terminology capabilities
     try {
       const termCapUrl = `${version.address}/metadata?mode=terminology`;
@@ -323,12 +311,19 @@ class RegistryCrawler {
       
       if (termCap.codeSystem) {
         termCap.codeSystem.forEach(cs => {
-          if (cs.uri) {
-            version.codeSystems.push(cs.uri);
+          let content = cs.content || Extensions.readString(cs, "http://hl7.org/fhir/5.0/StructureDefinition/extension-TerminologyCapabilities.codeSystem.content");
+          if (cs.uri && !this.isExcluded(cs.uri, exclusions)) {
+            if (!set.has(cs.uri)) {
+              set.add(cs.uri);
+              version.codeSystems.push(this.addContent({uri: cs.uri}, content));
+            }
             if (cs.version) {
               cs.version.forEach(v => {
-                if (v.code) {
-                  version.codeSystems.push(`${cs.uri}|${v.code}`);
+                if (v.code && !this.isExcluded(cs.uri+"|"+v.code, exclusions)) {
+                  if (!set.has(cs.uri+"|"+v.code)) {
+                    version.codeSystems.push(this.addContent({uri: cs.uri, version: v.code}, content));
+                    set.add(cs.uri+"|"+v.code);
+                  }
                 }
               });
             }
@@ -336,20 +331,12 @@ class RegistryCrawler {
         });
       }
     } catch (error) {
-      this.addLogEntry('error', `Could not fetch terminology capabilities: ${error.message}`);
+      debugLog(error);
+      this.addLogEntry('error', `Could not fetch terminology capabilities from ${version.address}: ${error.message}`);
     }
     
     // Search for value sets
-    await this.fetchValueSets(version,  server);
-  }
-
-  /**
-   * Process an R5 server
-   */
-  async processServerVersionR5(version, server) {
-    // R5 is essentially the same as R4 for our purposes
-    await this.processServerVersionR4(version, server);
-    version.version = version.version || '5.0.0';
+    await this.fetchValueSets(version, server, exclusions);
   }
 
   /**
@@ -360,16 +347,22 @@ class RegistryCrawler {
    * @param {Object} version - The server version information
    * @param {Object} server - The server information
    */
-  async fetchValueSets(version, server) {
+  async fetchValueSets(version, server, exclusions) {
     // Initial search URL
-    let searchUrl = `${version.address}/ValueSet?_elements=url,version`;
+    let count = 0;
+    let searchUrl = `${version.address}/ValueSet?_elements=url,version`+(version.address.includes("fhir.org") ? "&_count=200" : "");
     try {
       // Set of URLs to avoid duplicates
       const valueSetUrls = new Set();
 
-
       // Continue fetching while we have a URL
       while (searchUrl) {
+        count++;
+        if (count == 1000) {
+          throw new Error(`Fetch ValueSet loop exceeded 1000 iterations - a logic problem on the server? (${version.address})`);
+        }
+
+        if (this.abortController?.signal.aborted) break;
         this.log.debug(`Fetching value sets from ${searchUrl}`);
         const bundle = await this.fetchJson(searchUrl, server.code);
 
@@ -378,9 +371,9 @@ class RegistryCrawler {
           bundle.entry.forEach(entry => {
             if (entry.resource) {
               const vs = entry.resource;
-              if (vs.url) {
+              if (vs.url && !this.isExcluded(vs.url, exclusions)) {
                 valueSetUrls.add(vs.url);
-                if (vs.version) {
+                if (vs.version && !this.isExcluded(vs.url+'|'+vs.version, exclusions)) {
                   valueSetUrls.add(`${vs.url}|${vs.version}`);
                 }
               }
@@ -402,6 +395,7 @@ class RegistryCrawler {
       version.valueSets = Array.from(valueSetUrls).sort();
 
     } catch (error) {
+      debugLog(error);
       this.addLogEntry('error', `Could not fetch value sets: ${error.message} from ${searchUrl}`);
     }
   }
@@ -460,6 +454,7 @@ class RegistryCrawler {
       const response = await axios.get(fetchUrl, {
         timeout: this.config.timeout,
         headers: headers,
+        signal: this.abortController?.signal,
         validateStatus: (status) => status < 500 // Don't throw on 4xx
       });
       
@@ -478,6 +473,7 @@ class RegistryCrawler {
       return response.data;
       
     } catch (error) {
+      debugLog(error);
       if (error.response) {
         throw new Error(`HTTP ${error.response.status}: ${error.response.statusText}`);
       } else if (error.request) {
@@ -621,19 +617,58 @@ class RegistryCrawler {
    * @param {string} level - Filter by log level
    * @returns {Array} Array of log entries
    */
-  getLogs(limit = 100)
+  getLogs(limit = 100, level = null)
   {
     if (!this.logs) {
       return [];
     }
 
     // Filter by level if specified
-    let filteredLogs = this.logs;
+    let filteredLogs = level ? this.logs.filter(entry => entry.level === level) : this.logs;
 
     // Get the latest entries up to the limit
     return filteredLogs.slice(-limit);
   }
 
+  addContent(param, content) {
+    if (content) {
+      param.content = content;
+    }
+    return param;
+  }
+
+  compareCS(a, b) {
+    if (a.version || b.version) {
+      let s = (a.uri+'|'+a.version) || '';
+      return s.localeCompare(b.uri+'|'+b.version);
+    } else {
+      return (a.uri || '').localeCompare(b.uri);
+    }
+  }
+
+  isExcluded(url, exclusions) {
+    for (let exclusion of exclusions || []) {
+      let match = false;
+      if (exclusion.endsWith('*')) {
+        const prefix = exclusion.slice(0, -1);
+        match = url.startsWith(prefix);
+      } else {
+        // Otherwise do exact matching on both full and base URL
+        match = url === exclusion;
+      }
+      if (match) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  shutdown() {
+    if (this.abortController) {
+      this.abortController.abort();
+    }
+  }
+
 }
 
 module.exports = RegistryCrawler;

package/registry/model.js

@@ -37,7 +37,7 @@ class ServerVersionInformation {
   getCsListHtml() {
     if (this.codeSystems.length === 0) return '<ul></ul>';
     return '<ul>' + this.codeSystems.map(cs => 
-      `<li>${escape(cs)}</li>`
+      `<li>${escape(cs.uri+(cs.version ? '|'+cs.version : ''))}</li>`
     ).join('') + '</ul>';
   }
 
@@ -392,7 +392,7 @@ class ServerRegistryUtilities {
     return value === mask;
   }
 
-  static hasMatchingCodeSystem(cs, list, supportMask) {
+  static hasMatchingCodeSystem(cs, list, supportMask, content) {
     if (!cs || list.length === 0) return false;
 
     // Handle URLs with pipes - extract base URL
@@ -403,13 +403,19 @@ class ServerRegistryUtilities {
 
     return list.some(item => {
       // If we support wildcards (masks) and the item ends with "*", do prefix matching
-      if (supportMask && item.endsWith('*')) {
-        const prefix = item.slice(0, -1);
-        return cs.startsWith(prefix) || baseCs.startsWith(prefix);
+      let vurl = item.uri ? item.version ? item.uri+"|"+item.version : item.uri : item;
+      let ok = false;
+      if (supportMask && vurl.endsWith('*')) {
+        const prefix = vurl.slice(0, -1);
+        ok = cs.startsWith(prefix) || baseCs.startsWith(prefix);
+      } else {
+        // Otherwise do exact matching on both full and base URL
+        ok = vurl === cs || vurl === baseCs;
       }
-
-      // Otherwise do exact matching on both full and base URL
-      return item === cs || item === baseCs;
+      if (ok && content) {
+        content.content = item.content;
+      }
+      return ok;
     });
   }
 

package/registry/registry.js

@@ -926,6 +926,9 @@ class RegistryModule {
       this.crawlInterval = null;
     }
 
+    if (this.crawler) {
+      this.crawler.shutdown();
+    }
     // Save current data
     if (this.crawler && this.currentData) {
       await this.saveData();
@@ -1155,6 +1158,7 @@ class RegistryModule {
       html += '<th>URL</th>';
       html += '<th>Security</th>';
       html += '<th>Access Info</th>';
+      html += '<th>Content</th>';
       html += '</tr>';
       html += '</thead>';
       html += '<tbody>';
@@ -1165,6 +1169,7 @@ class RegistryModule {
         html += `<td><a href="${server.url}" target="_blank">${escape(server.url)}</a></td>`;
         html += `<td>${this.renderSecurityTags(server)}</td>`;
         html += `<td>${server.access_info ? escape(server.access_info) : ''}</td>`;
+        html += `<td>${server.content ? escape(server.content) : ''}</td>`;
         html += '</tr>';
       });
 

package/root-template.html

@@ -74,6 +74,7 @@
 
 [%content%]
 
+[%about%]
 
  </div>
 

package/server.js

@@ -395,6 +395,14 @@ process.on('uncaughtException', (error) => {
 });
 
 app.get('/', async (req, res) => {
+  // If an override index.html exists, serve it instead of the FHIRsmith home page
+  if (config.server?.webBase) {
+    const overrideIndex = path.join(path.resolve(config.server.webBase), 'index.html');
+    if (fs.existsSync(overrideIndex)) {
+      return res.sendFile(overrideIndex);
+    }
+  }
+
   // Check if client wants HTML response
   const acceptsHtml = req.headers.accept && req.headers.accept.includes('text/html');
 
@@ -410,69 +418,50 @@ app.get('/', async (req, res) => {
 
       const content = await buildRootPageContent();
 
+      // Load optional about box fragment from data directory
+      let about = '';
+      const aboutPath = path.join(folders.dataDir(), 'about.html');
+      if (fs.existsSync(aboutPath)) {
+        about = fs.readFileSync(aboutPath, 'utf8');
+      }
+
       // Build basic stats for root page
       const stats = {
         version: packageJson.version,
         enabledModules: Object.keys(config.modules).filter(m => config.modules[m].enabled).length,
-        processingTime: Date.now() - startTime
+        processingTime: Date.now() - startTime,
+        about
       };
 
       const html = htmlServer.renderPage('root', escape(config.hostName) || 'FHIRsmith Server', content, stats);
       res.setHeader('Content-Type', 'text/html');
       res.send(html);
+      return;
     } catch (error) {
       serverLog.error('Error rendering root page:', error);
       htmlServer.sendErrorResponse(res, 'root', error);
+      return;
     }
-  } else {
-    // Return JSON response for API clients
-    const enabledModules = {};
-    Object.keys(config.modules).forEach(moduleName => {
-      if (config.modules[moduleName].enabled) {
-        if (moduleName === 'tx') {
-          // TX module has multiple endpoints
-          enabledModules[moduleName] = {
-            enabled: true,
-            endpoints: config.modules.tx.endpoints.map(e => ({
-              path: e.path,
-              fhirVersion: e.fhirVersion,
-              context: e.context || null
-            }))
-          };
-        } else {
-          enabledModules[moduleName] = {
-            enabled: true,
-            endpoint: moduleName === 'vcl' ? '/VCL' : `/${moduleName}`
-          };
-        }
-      }
-    });
-
-    res.json({
-      message: 'FHIR Development Server',
-      version: '1.0.0',
-      modules: enabledModules,
-      endpoints: {
-        health: '/health',
-        ...Object.fromEntries(
-          Object.keys(enabledModules)
-            .filter(m => m !== 'tx')
-            .map(m => [
-              m,
-              m === 'vcl' ? '/VCL' : `/${m}`
-            ])
-        ),
-        // Add TX endpoints separately
-        ...(enabledModules.tx ? {
-          tx: config.modules.tx.endpoints.map(e => e.path)
-        } : {})
-      }
-    });
   }
+  return serveFhirsmithHome(req, res);
 });
 
+app.get('/fhirsmith', (req, res) => serveFhirsmithHome(req, res));
 
 // Serve static files
+if (config.server?.webBase) {
+  const overrideDir = path.resolve(config.server.webBase);
+  app.use((req, res, next) => {
+    const filePath = path.join(overrideDir, req.path);
+    fs.access(filePath, fs.constants.F_OK, (err) => {
+      if (!err) {
+        res.sendFile(filePath);
+      } else {
+        next();
+      }
+    });
+  });
+}
 app.use(express.static(path.join(__dirname, 'static')));
 
 // Health check endpoint
@@ -610,5 +599,84 @@ process.on('SIGINT', async () => {
   process.exit(0);
 });
 
+async function serveFhirsmithHome(req, res) {
+  // Check if client wants HTML response
+  const acceptsHtml = req.headers.accept && req.headers.accept.includes('text/html');
+
+  if (acceptsHtml) {
+    try {
+      const startTime = Date.now();
+
+      // Load template if not already loaded
+      if (!htmlServer.hasTemplate('root')) {
+        const templatePath = path.join(__dirname, 'root-template.html');
+        htmlServer.loadTemplate('root', templatePath);
+      }
+
+      const content = await buildRootPageContent();
+
+      // Build basic stats for root page
+      const stats = {
+        version: packageJson.version,
+        enabledModules: Object.keys(config.modules).filter(m => config.modules[m].enabled).length,
+        processingTime: Date.now() - startTime
+      };
+
+      const html = htmlServer.renderPage('root', escape(config.hostName) || 'FHIRsmith Server', content, stats);
+      res.setHeader('Content-Type', 'text/html');
+      res.send(html);
+      return;
+    } catch (error) {
+      serverLog.error('Error rendering root page:', error);
+      htmlServer.sendErrorResponse(res, 'root', error);
+      return;
+    }
+  } else {
+    // Return JSON response for API clients
+    const enabledModules = {};
+    Object.keys(config.modules).forEach(moduleName => {
+      if (config.modules[moduleName].enabled) {
+        if (moduleName === 'tx') {
+          // TX module has multiple endpoints
+          enabledModules[moduleName] = {
+            enabled: true,
+            endpoints: config.modules.tx.endpoints.map(e => ({
+              path: e.path,
+              fhirVersion: e.fhirVersion,
+              context: e.context || null
+            }))
+          };
+        } else {
+          enabledModules[moduleName] = {
+            enabled: true,
+            endpoint: moduleName === 'vcl' ? '/VCL' : `/${moduleName}`
+          };
+        }
+      }
+    });
+
+    res.json({
+      message: 'FHIR Development Server',
+      version: '1.0.0',
+      modules: enabledModules,
+      endpoints: {
+        health: '/health',
+        ...Object.fromEntries(
+          Object.keys(enabledModules)
+            .filter(m => m !== 'tx')
+            .map(m => [
+              m,
+              m === 'vcl' ? '/VCL' : `/${m}`
+            ])
+        ),
+        // Add TX endpoints separately
+        ...(enabledModules.tx ? {
+          tx: config.modules.tx.endpoints.map(e => e.path)
+        } : {})
+      }
+    });
+  }
+}
+
 // Start the server
 startServer();

package/tx/README.md

@@ -33,22 +33,22 @@ Add the `tx` section to your `config.json`:
       "endpoints": [
         {
           "path": "/tx/r5",
-          "fhirVersion": 5,
+          "fhirVersion": "5.0",
           "context": null
         },
         {
           "path": "/tx/r4",
-          "fhirVersion": 4,
+          "fhirVersion": "4.0",
           "context": null
         },
         {
           "path": "/tx/r3",
-          "fhirVersion": 3,
+          "fhirVersion": "3.0",
           "context": null
         },
         {
           "path": "/tx/r4/demo",
-          "fhirVersion": 4,
+          "fhirVersion": "4.0",
           "context": "demo"
         }
       ]