fhirsmith 0.4.2 → 0.5.0

package/tx/vs/vs-vsac.js

@@ -3,6 +3,8 @@ const axios = require('axios');
 const { AbstractValueSetProvider } = require('./vs-api');
 const { ValueSetDatabase } = require('./vs-database');
 const { VersionUtilities } = require('../../library/version-utilities');
+const folders = require('../../library/folder-setup');
+const ValueSet = require("../library/valueset");
 
 /**
  * VSAC (Value Set Authority Center) ValueSet provider
@@ -16,22 +18,20 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
    * @param {number} [config.refreshIntervalHours=24] - Hours between refresh scans
    * @param {string} [config.baseUrl='http://cts.nlm.nih.gov/fhir'] - Base URL for VSAC FHIR server
    */
-  constructor(config) {
+  constructor(config, stats) {
     super();
+    this.stats = stats;
 
     if (!config.apiKey) {
-      throw new Error('API key is required');
-    }
-    if (!config.cacheFolder) {
-      throw new Error('Cache folder is required');
+      throw new Error('VSAC API key is required');
     }
 
     this.apiKey = config.apiKey;
-    this.cacheFolder = config.cacheFolder;
+    this.cacheFolder = folders.ensureFilePath("terminology-cache/vsac");
     this.baseUrl = config.baseUrl || 'http://cts.nlm.nih.gov/fhir';
     this.refreshIntervalHours = config.refreshIntervalHours || 24;
 
-    this.dbPath = path.join(config.cacheFolder, 'vsac-valuesets.db');
+    this.dbPath = path.join(this.cacheFolder, 'vsac-valuesets.db');
     this.database = new ValueSetDatabase(this.dbPath);
     this.valueSetMap = new Map();
     this.initialized = false;
@@ -60,16 +60,18 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
     if (this.initialized) {
       return;
     }
+    this.stats.addTask('VSAC Sync', `${this.refreshIntervalHours} hours`);
 
     // Create database if it doesn't exist
     if (!(await this.database.exists())) {
       await this.database.create();
-      // Force initial refresh for new database
-      await this.refreshValueSets();
     } else {
       // Load existing data
       await this._reloadMap();
     }
+    if (this.valueSetMap.size == 0) {
+      await this.refreshValueSets();
+    }
 
     // Start periodic refresh
     this._startRefreshTimer();
@@ -110,67 +112,127 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
    * @returns {Promise<void>}
    */
   async refreshValueSets() {
+    this.stats.task('VSAC Sync', 'running');
     if (this.isRefreshing) {
       console.log('Refresh already in progress, skipping');
       return;
     }
+    this.queue = [];
 
     this.isRefreshing = true;
-    const refreshStartTime = Math.floor(Date.now() / 1000);
 
     try {
+      // phase 1: list all value sets
       console.log('Starting VSAC ValueSet refresh...');
 
-      let totalFetched = 0;
-      let url = '/ValueSet?_offset=0&_count=100';
+      // This lists all the currently valid value sets by URL, but not the older versions
+      let url = '/ValueSet?_offset=0&_count=100&_elements=id,url,version,status';
+
+      let total = undefined;
+      let count = 0;
+      let ncount = 0;
 
       while (url) {
-        console.log(`Fetching page: ${url}`);
-        const bundle = await this._fetchBundle(url);
+        console.log(`Sync: ${count} of ${total} - ${ncount} new`);
+        this.stats.task('VSAC Sync', `Sync: ${count} of ${total} - ${ncount} new`);
 
-        if (bundle.entry && bundle.entry.length > 0) {
-          // Extract ValueSets from bundle entries
-          const valueSets = bundle.entry
-            .filter(entry => entry.resource && entry.resource.resourceType === 'ValueSet')
-            .map(entry => entry.resource);
+        const bundle = await this._fetchBundle(url);
 
-          if (valueSets.length > 0) {
-            await this.database.batchUpsertValueSets(valueSets);
-            totalFetched += valueSets.length;
-            console.log(`Processed ${valueSets.length} ValueSets (total: ${totalFetched})`);
+        if (!total) {
+          total = bundle.total;
+        }
+        for (let be of bundle.entry || []) {
+          let vs = be.resource;
+          if (vs) {
+            count++;
+            // if we've seen this value set before, then we've got nothing new here.
+            if (!this.valueSetMap.has(vs.url+"|"+vs.version)) {
+              this.queue.push(vs.url);
+              ncount++;
+            }
           }
         }
-
         // Find next link
         url = this._getNextUrl(bundle);
 
         // Safety check against infinite loops
-        if (bundle.total && totalFetched >= bundle.total) {
-          console.log(`Reached total count (${bundle.total}), stopping`);
+        if (count > total) {
+          console.log(`Reached total count (${total}), stopping`);
           break;
         }
       }
 
-      // Clean up old records
-      const deletedCount = await this.database.deleteOldValueSets(refreshStartTime);
-      if (deletedCount > 0) {
-        console.log(`Deleted ${deletedCount} old ValueSets`);
+      this.lastRefresh = new Date();
+      console.log(`VSAC refresh phase 1 done. Total: ${count} with ${ncount} new items`);
+      this.stats.task('VSAC Sync', `VSAC refresh phase 1 done. Total: ${count} with ${ncount} new items`);
+
+      let tracking = { totalFetched: 0, totalNew: 0, count: 0, newCount : 0 };
+      // phase 2: query for history & content
+      this.requeue = [];
+      for (let q of this.queue) {
+        this.stats.task('VSAC History for '+q, `running (${tracking.totalFetched} fetched, ${tracking.totalNew} new)`);
+        try {
+          await this.processContentAndHistory(q, tracking, this.queue.length);
+        } catch (error) {
+          this.requeue.push(q)
+          console.log(error);
+          this.stats.task('VSAC Sync', error.message);
+        }
+        // `running (${totalFetched} fetched, ${totalNew} new)`)
+        tracking.count++;
+      }
+      console.log("Requeue");
+      for (let q of this.requeue) {
+        this.stats.task('VSAC History for '+q, `running (${tracking.totalFetched} fetched, ${tracking.totalNew} new)`);
+        try {
+          await this.processContentAndHistory(q, tracking, this.requeue.length);
+        } catch (error) {
+          console.log(error);
+          this.stats.task('VSAC Sync', error.message);
+        }
+        // `running (${totalFetched} fetched, ${totalNew} new)`)
+        tracking.count++;
       }
 
       // Reload map with fresh data
       await this._reloadMap();
-
-      this.lastRefresh = new Date();
-      console.log(`VSAC refresh completed. Total: ${totalFetched} ValueSets, Deleted: ${deletedCount}`);
-
+      console.log(`VSAC refresh completed. Total: ${tracking.totalFetched} ValueSets, Deleted: ${tracking.deletedCount}`);
     } catch (error) {
       console.log(error, 'Error during VSAC refresh:');
+      this.stats.task('VSAC Sync', `Error (${error.message})`);
       throw error;
     } finally {
       this.isRefreshing = false;
     }
   }
 
+  /**
+   * Insert multiple ValueSets in a batch operation
+   * @param {Array<Object>} valueSets - Array of ValueSet resources
+   * @returns {Promise<void>}
+   */
+  async batchUpsertValueSets(valueSets) {
+    if (valueSets.length === 0) {
+      return;
+    }
+
+    let count = 0;
+    // Process sequentially to avoid database locking
+    for (const valueSet of valueSets) {
+      let key = valueSet.url+"|"+valueSet.version;
+      let vs = this.valueSetMap.get(key);
+      if (vs) {
+        // we've seen this before, and maybe fetched it's history, so just update
+        // the timestamp
+        await this.database.seeValueSet(valueSet);
+      } else {
+        await this.database.upsertValueSet(valueSet);
+        count++;
+      }
+    }
+    return count;
+  }
+
   /**
    * Fetch a FHIR Bundle from the server
    * @param {string} url - Relative URL to fetch
@@ -184,7 +246,34 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
       if (response.data && response.data.resourceType === 'Bundle') {
         return response.data;
       } else {
-        throw new Error('Response is not a FHIR Bundle');
+        throw new Error('VSAC Response is not a FHIR Bundle');
+      }
+    } catch (error) {
+      if (error.response) {
+        throw new Error(`HTTP ${error.response.status}: ${error.response.statusText}`);
+      } else if (error.request) {
+        throw new Error('Network error: No response received');
+      } else {
+        throw new Error(`Request error: ${error.message}`);
+      }
+    }
+  }
+
+
+  /**
+   * Fetch a FHIR Bundle from the server
+   * @param {string} url - Relative URL to fetch
+   * @returns {Promise<Object>} FHIR Bundle
+   * @private
+   */
+  async _fetchValueSet(id) {
+    try {
+      const response = await this.httpClient.get("/ValueSet/"+id);
+
+      if (response.data && response.data.resourceType === 'ValueSet') {
+        return response.data;
+      } else {
+        throw new Error('VSAC Response is not a FHIR ValueSet');
       }
     } catch (error) {
       if (error.response) {
@@ -244,7 +333,7 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
     // Try exact match first: url|version
     let key = `${url}|${version}`;
     if (this.valueSetMap.has(key)) {
-      return this.valueSetMap.get(key);
+      return await this.checkFullVS(this.valueSetMap.get(key));
     }
 
     // If version is semver, try url|major.minor
@@ -254,7 +343,7 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
         if (majorMinor) {
           key = `${url}|${majorMinor}`;
           if (this.valueSetMap.has(key)) {
-            return this.valueSetMap.get(key);
+            return await this.checkFullVS(this.valueSetMap.get(key));
           }
         }
       }
@@ -264,12 +353,15 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
 
     // Finally try just the URL
     if (this.valueSetMap.has(url)) {
-      return this.valueSetMap.get(url);
+      return await this.checkFullVS(this.valueSetMap.get(url));
     }
 
-    throw new Error(`Value set not found: ${url} version ${version}`);
+    return null;
   }
 
+  async fetchValueSetById(id) {
+    return await this.checkFullVS(this.valueSetMap.get(id));
+  }
   /**
    * Searches for value sets based on criteria
    * @param {Array<{name: string, value: string}>} searchParams - Search criteria
@@ -358,6 +450,50 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
     await this.database.close();
   }
 
+  // eslint-disable-next-line no-unused-vars
+  assignIds(ids) {
+    // nothing?
+  }
+
+  // when we get a valueset from vsac via search, the compose is not
+  // populated. We don't load all the composes. Instead, when value sets
+  // are fetched, we check to see if we've got the compose, and if we
+  // haven't, then we fetch it and store it
+  async checkFullVS(vs) {
+    if (!vs) {
+      return null;
+    }
+    if (vs.jsonObj.compose) {
+      return vs;
+    }
+    console.log('get a full copy for the ValueSet '+vs.url+'|'+vs.version);
+    let vsNew = await this._fetchValueSet(vs.id);
+    await this.database.upsertValueSet(vsNew);
+    this.database.addToMap(this.valueSetMap, vsNew.id, vsNew.url, vsNew.version, vsNew);
+    return new ValueSet(vsNew);
+  }
+
+  async processContentAndHistory(q, tracking, length) {
+    let url = `/ValueSet?url=${q}`;
+    const bundle = await this._fetchBundle(url);
+
+    let vcount = 0;
+    if (bundle.entry && bundle.entry.length > 0) {
+      // Extract ValueSets from bundle entries
+      const valueSets = bundle.entry
+        .filter(entry => entry.resource && entry.resource.resourceType === 'ValueSet')
+        .map(entry => entry.resource);
+      if (valueSets.length > 0) {
+        tracking.totalNew = tracking.totalNew + await this.batchUpsertValueSets(valueSets);
+        tracking.totalFetched += valueSets.length;
+        vcount = valueSets.length;
+      }
+    }
+    let logMsg = `VSAC (${tracking.count} of ${length}) ${q}: ${vcount} versions`;
+    console.log(logMsg);
+    this.stats.task('VSAC Sync', logMsg);
+
+  }
 }
 
 // Usage examples:

package/tx/workers/batch-validate.js

@@ -82,6 +82,7 @@ class BatchValidateWorker extends TerminologyWorker {
             output.push({name: "validation", resource : p});
           } catch (error) {
             this.log.error(error);
+            this.debugLog(error);
             if (error instanceof Issue) {
               let op = new OperationOutcome();
               op.addIssue(error);
@@ -97,6 +98,7 @@ class BatchValidateWorker extends TerminologyWorker {
       return res.json(result);
     } catch (error) {
       this.log.error(error);
+      this.debugLog(error);
       return res.status(error.statusCode || 500).json(this.operationOutcome(
         'error', error.issueCode || 'exception', error.message));
     }

package/tx/workers/batch.js

@@ -40,6 +40,7 @@ class BatchWorker extends TerminologyWorker {
       await this.handleBatch(req, res);
     } catch (error) {
       this.log.error(error);
+      this.debugLog(error);
       if (error instanceof Issue) {
         const oo = new OperationOutcome();
         oo.addIssue(error);
@@ -159,6 +160,7 @@ class BatchWorker extends TerminologyWorker {
 
     } catch (error) {
       this.log.error(error);
+      this.debugLog(error);
       const statusCode = error.statusCode || 500;
       const issueCode = error.issueCode || 'exception';