fhirsmith 0.18.2 → 1.0.0

package/tx/usage-tracker.js

@@ -1,22 +1,46 @@
 
+const { setImmediate: yieldToEventLoop } = require('timers/promises');
+
+// The startup scan reads every value set (thousands with the npm packages, VSAC), about a second
+// of work: give requests a turn every YIELD_INTERVAL_MS of it.
+const YIELD_INTERVAL_MS = 10;
+
 class ConceptUsageTracker {
 
-  constructor() {
+  // Defaults for usageTrackerMaxCodes and usageTrackerMaxCodeLength: most codes
+  // tracked, 2x the about 500,000 codes the value sets of a full library list, and
+  // the longest code, 2.5x the longest of those (40 characters). Codes beyond either
+  // limit aren't tracked, so validating made-up codes can't grow the map forever.
+  static MAX_CODES = 1000000;
+  static MAX_CODE_LENGTH = 100;
+
+  /**
+   * @param {number} [maxCodes] - most codes tracked; Infinity = no limit
+   * @param {number} [maxCodeLength] - longest code tracked; Infinity = no limit
+   */
+  constructor(maxCodes = ConceptUsageTracker.MAX_CODES, maxCodeLength = ConceptUsageTracker.MAX_CODE_LENGTH) {
     this.map = new Map();
+    this.codes = 0;
+    this.maxCodes = maxCodes;
+    this.maxCodeLength = maxCodeLength;
   }
 
   async scanValueSets(library) {
     let c = 0;
+    let sliceStart = Date.now();
     for (let vsp of library.valueSetProviders) {
-      let list = await vsp.listAllValueSets();
-      for (let url of list) {
-        let vs = await vsp.fetchValueSet(url);
-        if (vs && vs.jsonObj.compose) {
+      await vsp.scanAllValueSets(async (vs) => {
+        // in the callback, so it yields however the provider reads its value sets
+        if (Date.now() - sliceStart >= YIELD_INTERVAL_MS) {
+          await yieldToEventLoop();
+          sliceStart = Date.now();
+        }
+        if (vs.jsonObj.compose) {
           if (await this.scanValueSet(vs.jsonObj.compose)) {
             c++;
           }
         }
-      }
+      });
     }
     return c;
   }
@@ -41,14 +65,18 @@ class ConceptUsageTracker {
 
   seeConcept(system, code) {
     let cs = this.map.get(system);
-    if (!cs) {
-      cs = new Map();
-      this.map.set(system, cs);
-    }
-    let ci = cs.get(code);
+    let ci = cs ? cs.get(code) : null;
     if (!ci) {
+      if (this.codes >= this.maxCodes || code.length > this.maxCodeLength) {
+        return;
+      }
+      if (!cs) {
+        cs = new Map();
+        this.map.set(system, cs);
+      }
       ci = { count : 0 }
       cs.set(code, ci);
+      this.codes++;
     }
     ci.count++;
   }

package/tx/vs/vs-api.js

@@ -120,6 +120,25 @@ class AbstractValueSetProvider {
     return [];
   }
 
+  /**
+   * Call fn once for each url in listAllValueSets(), with the latest version of it (what
+   * fetchValueSet(url) returns), for the scans that look at every value set (see
+   * usage-tracker.js, problems.js). listAllValueSets() can list a url once per stored version:
+   * older versions are not scanned, and the latest is scanned only once, so the usage counts
+   * are not inflated by how many versions there are. A provider that doesn't hold its value
+   * sets in memory can do this more cheaply than fetching each one.
+   *
+   * @param {function(ValueSet): Promise<void>} fn
+   */
+  async scanAllValueSets(fn) {
+    for (const url of new Set(await this.listAllValueSets())) {
+      const vs = await this.fetchValueSet(url);
+      if (vs) {
+        await fn(vs);
+      }
+    }
+  }
+
   async close() {
 
   }