fhirsmith 0.18.1 → 1.0.0

package/tx/workers/batch-validate.js

@@ -11,6 +11,11 @@
 // system/codeSystem entry validates against a code system), so both batch routes
 // share one implementation and even a mixed batch is handled correctly.
 //
+// The exception is `url`, which names a value set to ValueSet/$validate-code but a
+// code system to CodeSystem/$validate-code. So on the CodeSystem route `url` is a
+// shared global (the code system), and every entry is a CodeSystem validation;
+// only the ValueSet route dispatches by shape.
+//
 
 const { TerminologyWorker } = require('./worker');
 const { OperationOutcome, Issue, buildOperationOutcome, outcomeFromError } = require('../library/operation-outcome');
@@ -33,14 +38,15 @@ class BatchValidateWorker extends TerminologyWorker {
     super(opContext, log, provider, languages, i18n);
     // "Global" parameters may be supplied once at the top level and are applied to
     // every entry that doesn't override them. tx-resource is always shared. The
-    // primary differs by operation: a ValueSet batch shares url/valueSet, a
-    // CodeSystem batch shares system/codeSystem.
+    // primary differs by operation: a ValueSet batch shares url/valueSet (the value
+    // set), a CodeSystem batch shares url/version/codeSystem (the code system), as
+    // CodeSystem/$validate-code names them - and system, which it also accepts.
     this.valueSetGlobalNames = new Set([
       "tx-resource", "url", "valueSet", "lenient-display-validation",
       "__Accept-Language", "__Content-Language"
     ]);
     this.codeSystemGlobalNames = new Set([
-      "tx-resource", "system", "codeSystem", "lenient-display-validation",
+      "tx-resource", "url", "version", "system", "codeSystem", "lenient-display-validation",
       "__Accept-Language", "__Content-Language"
     ]);
     // Back-compat: `globalNames` was the (ValueSet) set before CodeSystem batches.
@@ -57,12 +63,12 @@ class BatchValidateWorker extends TerminologyWorker {
 
   /** ValueSet/$batch-validate-code: shared globals are url/valueSet. */
   async handleValueSet(req, res) {
-    return this.processBatch(req, res, this.valueSetGlobalNames);
+    return this.processBatch(req, res, this.valueSetGlobalNames, false);
   }
 
-  /** CodeSystem/$batch-validate-code: shared globals are system/codeSystem. */
+  /** CodeSystem/$batch-validate-code: shared globals are url/version/system/codeSystem. */
   async handleCodeSystem(req, res) {
-    return this.processBatch(req, res, this.codeSystemGlobalNames);
+    return this.processBatch(req, res, this.codeSystemGlobalNames, true);
   }
 
   /**
@@ -74,8 +80,11 @@ class BatchValidateWorker extends TerminologyWorker {
    * @param {express.Request} req
    * @param {express.Response} res
    * @param {Set<string>} globalNames - which top-level params are shared globals
+   * @param {boolean} codeSystemRoute - true for CodeSystem/$batch-validate-code: every
+   *   entry is a CodeSystem validation (`url` there is the code system). On the
+   *   ValueSet route, an entry with no url/valueSet is a CodeSystem validation
    */
-  async processBatch(req, res, globalNames) {
+  async processBatch(req, res, globalNames, codeSystemRoute = false) {
     try {
       let params = req.body;
       this.addHttpParams(req, params);
@@ -120,16 +129,32 @@ class BatchValidateWorker extends TerminologyWorker {
             : p.resource.parameter;
           op.jsonObj.parameter.push(...entryParams);
 
-          let worker = new ValidateWorker(this.opContext.copy(), this.log, this.provider, this.languages, this.i18n);
+          // Each entry gets its own provider list, closed as soon as the entry is
+          // answered. Sharing the batch's list would keep every entry's providers
+          // (each with an open sqlite connection, and a reference to the entry's
+          // context and its log) alive until the whole batch ends.
+          const entryContext = this.opContext.copy({ ownProviders: true });
+          let worker = new ValidateWorker(entryContext, this.log, this.provider, this.languages, this.i18n);
           try {
+            // Validating an entry never yields the event loop by itself, and IG builds send batches
+            // of up to 1,000 entries (~1 s of compute): give other requests a turn between entries
+            // (checkAndYield yields only after YIELD_INTERVAL ms of compute, so small batches don't),
+            // and see a disconnect. Inside the try: the time limit tripping here is this entry's
+            // too-costly, and the entries already answered keep their answers; a memory abort or
+            // a disconnect is rethrown below and stops the whole batch.
+            await this.opContext.checkAndYield('batch-validate');
             let p;
-            if (this.hasValueSet(op.jsonObj.parameter)) {
+            if (!codeSystemRoute && this.hasValueSet(op.jsonObj.parameter)) {
               p = await worker.handleValueSetInner(op.jsonObj);
             } else {
               p = await worker.handleCodeSystemInner(op.jsonObj);
             }
             output.push({name: "validation", resource : p});
           } catch (error) {
+            // the client has gone, or memory is short: the whole batch stops, not this entry
+            if (error.abandoned || error.memoryAbort) {
+              throw error;
+            }
             this.log.error(error);
             debugLog(error);
             if (error instanceof Issue) {
@@ -139,6 +164,8 @@ class BatchValidateWorker extends TerminologyWorker {
             } else {
               output.push({name: "validation", resource : outcomeFromError(error) } );
             }
+          } finally {
+            await entryContext.closeProviders();
           }
         }
       }
@@ -209,7 +236,7 @@ class BatchValidateWorker extends TerminologyWorker {
     const { txResources, primaryResources } = this.collectSuppliedResources({ parameter: allParams });
     const pool = txResources.concat(primaryResources);
     if (pool.length > 0) {
-      cache.add(cacheId, pool);
+      this.addToClientCache(cacheId, pool);
     }
     return true;
   }

package/tx/workers/batch.js

@@ -79,7 +79,8 @@ class BatchWorker extends TerminologyWorker {
     const responseEntries = [];
 
     for (let i = 0; i < entries.length; i++) {
-      this.deadCheck(`batch-entry-${i}`);
+      // a turn for the event loop between entries, so a disconnect is seen
+      await this.checkAndYield(`batch-entry-${i}`);
       const entry = entries[i];
       const responseEntry = await this.processEntry(entry, i);
       responseEntries.push(responseEntry);
@@ -159,6 +160,10 @@ class BatchWorker extends TerminologyWorker {
       };
 
     } catch (error) {
+      // the client has gone, or memory is short: the whole batch stops, not this entry
+      if (error.abandoned || error.memoryAbort) {
+        throw error;
+      }
       this.log.error(error);
       debugLog(error);
       const statusCode = error.statusCode || 500;

package/tx/workers/cache-control.js

@@ -97,7 +97,8 @@ class CacheControlWorker extends TerminologyWorker {
 
   /**
    * mode=start: mint a server-issued cache-id, create the (per-endpoint) cache,
-   * front-load any supplied resources into it, and return the id in the body.
+   * front-load any supplied resources into it, and return the id in the body
+   * (and, for proxy request logs, in the `${CACHE_ID_HEADER}` response header).
    *
    * The id is returned in the body rather than a header to keep it readable by
    * browser clients without CORS expose-header configuration; subsequent calls
@@ -134,9 +135,19 @@ class CacheControlWorker extends TerminologyWorker {
     // Flip this to default-true once all clients send an explicit value.
     const sealed = this.readSealed(params.jsonObj);
 
-    // "<instanceCode>.<uuid>" when this server has an instance code, else a bare UUID
-    const cacheId = cache.newCacheId();
-    cache.set(cacheId, resources, sealed);
+    // "<instanceCode>.<...>" when this server has an instance code, else a bare UUID
+    let cacheId;
+    try {
+      cacheId = cache.start(resources, sealed);
+    } catch (e) {
+      throw this.translateCacheIssue(e);
+    }
+
+    // Also echoed as a response header, so that a request log at the proxy (nginx
+    // $sent_http_x_cache_id) can tie each issued id to the start request that got
+    // it - which is what replaying a captured request log needs, because a replay
+    // target issues different ids. The body stays the authoritative carrier (see above).
+    res.setHeader(CACHE_ID_HEADER, cacheId);
 
     return res.status(200).json({
       resourceType: 'Parameters',

package/tx/workers/closure.js

@@ -35,6 +35,13 @@
 // name can read, add to, or reset that table. On a public server, require UUID names
 // (requireUuidNames) so that a name is effectively a secret held by the client that made it.
 //
+// Shared (server.cluster.sharedClosure): other processes answer $closure on the same tables.
+// The per-table lock still orders this process's calls, but a write may find the table has
+// moved on since it was read: then nothing is written, and the call is worked out again
+// from the table as it is now (see add, and the header of closure-store.js). After
+// writeRetries such goes, the call gets the same 429 as a busy queue. The code lookups and
+// subsumption tests done before are remembered for the retries, so a retry costs reads.
+//
 // Cost: everything a client can make this do is bounded (see LIMITS) - the tables, the
 // concepts and entries in a table, what one call can add, what a table's configuration
 // can store, and how many calls can queue for one table and for how long. The work itself
@@ -68,6 +75,11 @@ const LIMITS = {
 
 const RESYNC_PAGE = 2000; // rows read from the store at a time
 
+// shared: how many times an add is worked out again when another process changed the table
+// first, and the most it waits (with jitter) before a retry
+const WRITE_RETRIES = 10;
+const RETRY_WAIT_MS = 50;
+
 class ClosureWorker extends TerminologyWorker {
   /**
    * @param {OperationContext} opContext - Operation context
@@ -219,7 +231,19 @@ class ClosureWorker extends TerminologyWorker {
         if (!table && this.store.tableCount() >= max) {
           throw this.closureIssue('too-costly', 'CLOSURE_TOO_MANY_TABLES', [String(max)], 'too-costly', 422);
         }
-        table = this.store.createTable(name, config);
+        if (this.store.shared) {
+          // the checks above again, in the same transaction as the write
+          const made = this.store.createTableChecked(name, config, { reset, maxTables: max });
+          if (made.refused === 'exists') {
+            throw this.closureIssue('duplicate', 'CLOSURE_TABLE_EXISTS', [name], 'business-rule', 409);
+          }
+          if (made.refused === 'limit') {
+            throw this.closureIssue('too-costly', 'CLOSURE_TOO_MANY_TABLES', [String(max)], 'too-costly', 422);
+          }
+          table = made.table;
+        } else {
+          table = this.store.createTable(name, config);
+        }
         this.log.info(`closure table '${name}' ${reset ? 'reset' : 'created'}`);
         if (concepts.length === 0) {
           return await this.buildConceptMap(name, table.version, [], fhirVersion);
@@ -297,7 +321,9 @@ class ClosureWorker extends TerminologyWorker {
     let after = 0;
     for (;;) {
       await this.checkAndYield('closure-resync');
-      const page = this.store.entriesPage(table.id, v, after, RESYNC_PAGE);
+      const page = this.store.shared
+        ? this.store.entriesPage(table.id, v, after, RESYNC_PAGE, table.version)
+        : this.store.entriesPage(table.id, v, after, RESYNC_PAGE);
       for (const r of page) {
         entries.push({
           source: { system: r.source_system, code: r.source_code, display: r.source_display },
@@ -310,6 +336,17 @@ class ClosureWorker extends TerminologyWorker {
       }
       after = page[page.length - 1].rowid;
     }
+    if (this.store.shared) {
+      // Reset/dispose deletes the old pages. Never return a truncated snapshot, or
+      // entries from a replacement, as though they belonged to the version we read.
+      const current = this.store.getTable(name);
+      if (!current) {
+        throw this.closureIssue('not-found', 'CLOSURE_TABLE_UNKNOWN', [name], 'not-found', 404);
+      }
+      if (current.id !== table.id) {
+        throw this.closureIssue('conflict', 'CLOSURE_TABLE_CHANGED', [name], 'business-rule', 409);
+      }
+    }
     return await this.buildConceptMap(name, table.version, entries, fhirVersion);
   }
 
@@ -320,6 +357,66 @@ class ClosureWorker extends TerminologyWorker {
    * system or code, or going over a limit, fails the whole call.
    */
   async add(table, name, conceptParams, fhirVersion) {
+    if (this.store.shared) {
+      return await this.addShared(table, name, conceptParams, fhirVersion);
+    }
+    const { fresh, entries, out } = await this.workOutAdd(table, name, conceptParams, null);
+    const version = this.store.recordAdd(table, fresh, entries);
+    return await this.addResult(name, version, fresh, entries, out, fhirVersion);
+  }
+
+  /**
+   * Shared: add, with the write checked against the table as it was read
+   * (recordAddChecked). If another process changed the table first, nothing was written:
+   * read the table again and work the add out again, up to writeRetries times, waiting a
+   * little (with jitter) in between; then 429, as for a busy queue. A table that is gone
+   * by then is unknown (404), as it would be to a call that came after the one that
+   * dropped it.
+   */
+  async addShared(table, name, conceptParams, fhirVersion) {
+    const configured = this.closureConfig.writeRetries;
+    const retries = Number.isInteger(configured) && configured >= 0 ? configured : WRITE_RETRIES;
+    let memo = null;
+    for (let attempt = 0; ; attempt++) {
+      if (!memo || memo.tableId !== table.id) {
+        // what's remembered holds for one table's configuration: a reset starts again
+        memo = { tableId: table.id, providers: new Map(), codes: new Map(), subsumes: new Map() };
+      }
+      const { fresh, entries, out } = await this.workOutAdd(table, name, conceptParams, memo);
+      let version = null;
+      try {
+        version = this.store.recordAddChecked(table, fresh, entries);
+      } catch (error) {
+        // the busy timeout ran out: another process held the write lock throughout
+        if (!String(error.code || '').startsWith('SQLITE_BUSY')) {
+          throw error;
+        }
+      }
+      if (version !== null) {
+        return await this.addResult(name, version, fresh, entries, out, fhirVersion);
+      }
+      if (attempt >= retries) {
+        this.log.info(`closure table '${name}': changed by another process on each of ${attempt + 1} tries; answering busy`);
+        throw this.closureIssue('too-costly', 'CLOSURE_TABLE_BUSY', [name], 'too-costly', 429);
+      }
+      this.log.info(`closure table '${name}': changed by another process since it was read; working the add out again (retry ${attempt + 1} of ${retries})`);
+      const wait = Math.random() * Math.min(RETRY_WAIT_MS, 5 * 2 ** attempt);
+      await this.opContext.waitFor(new Promise(resolve => setTimeout(resolve, wait)), 'closure-retry');
+      table = this.store.getTable(name);
+      if (!table) {
+        throw this.closureIssue('not-found', 'CLOSURE_TABLE_UNKNOWN', [name], 'not-found', 404);
+      }
+    }
+  }
+
+  /**
+   * Work out what adding the concepts to the table, as read, would write: the new
+   * concepts and the entries they make. Writes nothing.
+   * @param {Object} memo - shared only (else null): code lookups and subsumption tests
+   *   already done for this table, kept across retries
+   * @returns {{fresh: Array, entries: Array, out: Array}}
+   */
+  async workOutAdd(table, name, conceptParams, memo) {
     // replay the table's configuration: its code systems, and its version rules
     const stored = { resourceType: 'Parameters', parameter: this.store.getParams(table.id) };
     this.setupAdditionalResources(stored);
@@ -331,7 +428,7 @@ class ClosureWorker extends TerminologyWorker {
     for (const c of existing) {
       known.add(c.system + '|' + c.code);
     }
-    const providers = new Map();
+    const providers = memo ? memo.providers : new Map();
     const fresh = [];
 
     for (const p of conceptParams) {
@@ -346,20 +443,31 @@ class ClosureWorker extends TerminologyWorker {
         this.seeSourceProvider(cs, coding.system);
         providers.set(coding.system, cs);
       }
-      const loc = await cs.locate(coding.code);
-      if (!loc || !loc.context) {
-        throw this.unknownCodeIssue(cs, coding.code, loc ? loc.message : null, txp);
+      const memoKey = memo ? coding.system + '|' + (coding.version || '') + '|' + coding.code : null;
+      let found = memo ? memo.codes.get(memoKey) : null;
+      if (!found) {
+        const loc = await cs.locate(coding.code);
+        if (!loc || !loc.context) {
+          throw this.unknownCodeIssue(cs, coding.code, loc ? loc.message : null, txp);
+        }
+        // the code system's own spelling of the code, so a case-insensitive code system
+        // doesn't end up with the same concept in the table twice
+        found = { context: loc.context, code: (await cs.code(loc.context)) || coding.code };
+        if (memo) {
+          memo.codes.set(memoKey, found);
+        }
       }
-      // the code system's own spelling of the code, so a case-insensitive code system
-      // doesn't end up with the same concept in the table twice
-      const code = (await cs.code(loc.context)) || coding.code;
+      const code = found.code;
       const key = coding.system + '|' + code;
       if (known.has(key)) {
         continue;
       }
       known.add(key);
+      if (found.display === undefined) {
+        found.display = await cs.display(found.context);
+      }
       // ref: how the store refers to it until it has an id (see ClosureStore.recordAdd)
-      fresh.push({ key, ref: key, system: coding.system, code, display: await cs.display(loc.context), cs });
+      fresh.push({ key, ref: key, system: coding.system, code, display: found.display, cs });
     }
 
     const maxConcepts = this.limit('maxConceptsPerTable');
@@ -374,14 +482,36 @@ class ClosureWorker extends TerminologyWorker {
       this.limit('maxEntriesPerTable') - (fresh.length > 0 ? this.store.entryCount(table.id) : 0));
     const entries = [];
     const out = [];
-    const before = existing.map(c => ({ ref: c.id, system: c.system, code: c.code, display: c.display }));
+    const before = existing.map(c => ({ key: c.system + '|' + c.code, ref: c.id, system: c.system, code: c.code, display: c.display }));
+    // Where the provider can list a code's ancestors (CodeSystemProvider.ancestorsOf), a pair
+    // is related from the two ancestor sets, which costs one provider call per concept rather
+    // than a subsumesTest per pair. Each set is asked for once per call, when first needed,
+    // and only for concepts of a system that has new concepts; a concept the provider has no
+    // set for (null, the default) is compared by subsumesTest, as before. The sets are not
+    // kept between calls, so they always come from the code system as it is now.
+    const ancestorSets = new Map(); // key -> Set of codes, or null
+    const ancestorsOf = async (cs, c) => {
+      if (!ancestorSets.has(c.key)) {
+        await this.checkAndYield('closure-ancestors');
+        ancestorSets.set(c.key, await cs.ancestorsOf(c.code));
+      }
+      return ancestorSets.get(c.key);
+    };
     for (const n of fresh) {
+      const an = await ancestorsOf(n.cs, n);
       for (const o of before) {
         if (o.system !== n.system) {
           continue;
         }
         await this.checkAndYield('closure-subsumes');
-        const outcome = await n.cs.subsumesTest(n.code, o.code);
+        const ao = an ? await ancestorsOf(n.cs, o) : null;
+        let outcome;
+        if (ao) {
+          // the same outcome subsumesTest gives (see ancestorsOf): n and o are different codes
+          outcome = ao.has(n.code) ? 'subsumes' : an.has(o.code) ? 'subsumed-by' : 'not-subsumed';
+        } else {
+          outcome = memo ? await this.memoSubsumes(memo, n, o) : await n.cs.subsumesTest(n.code, o.code);
+        }
         let source, target, relationship;
         if (outcome === 'subsumes') {          // the new concept is the broader one
           source = o; target = n; relationship = NARROWER;
@@ -401,7 +531,20 @@ class ClosureWorker extends TerminologyWorker {
       before.push(n);
     }
 
-    const version = this.store.recordAdd(table, fresh, entries);
+    return { fresh, entries, out };
+  }
+
+  /** n.cs.subsumesTest(n.code, o.code), remembered in the memo (shared retries). */
+  async memoSubsumes(memo, n, o) {
+    const key = n.system + '|' + n.code + '|' + o.code;
+    if (!memo.subsumes.has(key)) {
+      memo.subsumes.set(key, await n.cs.subsumesTest(n.code, o.code));
+    }
+    return memo.subsumes.get(key);
+  }
+
+  /** The response to an add, once it is written. */
+  async addResult(name, version, fresh, entries, out, fhirVersion) {
     if (fresh.length > 0) {
       this.log.info(`closure table '${name}': added ${fresh.length} concept(s), ${entries.length} entr${entries.length === 1 ? 'y' : 'ies'}, now version ${version}`);
     }

package/tx/workers/compare.js

@@ -172,18 +172,21 @@ class CompareWorker extends TerminologyWorker {
 
   async doCompare(txp, thisVS, otherVS) {
 
-    // ok, we have to compare the composes. we don't care about anything else
-    const thisC = thisVS.jsonObj.compose;
-    const otherC = otherVS.jsonObj.compose;
+    // ok, we have to compare the composes. we don't care about anything else.
+    // Copies: tidyIncludes merges, removes and sorts includes in place, and these value
+    // sets are shared - cache-id resources, and the server's own - so must not change
+    // (ExpansionCache.computeKey remembers a cached resource's hash, for one).
+    const thisC = thisVS.jsonObj.compose ? structuredClone(thisVS.jsonObj.compose) : thisVS.jsonObj.compose;
+    const otherC = otherVS.jsonObj.compose ? structuredClone(otherVS.jsonObj.compose) : otherVS.jsonObj.compose;
     if (!thisC) {
       return this.makeOutcome("indeterminate", `The ValueSet ${thisVS.vurl} has no compose`);
     }
-    Extensions.checkNoModifiers(thisC, 'CompareWorker.doCompare', 'compose', thisVS.vurl)
+    Extensions.checkNoModifiers(thisC, 'CompareWorker.doCompare', 'ValueSet.compose', thisVS.vurl)
     this.checkNoLockedDate(thisVS.vurl, thisC);
     if (!otherC) {
       return this.makeOutcome("indeterminate", `The ValueSet ${otherVS.vurl} has no compose`);
     }
-    Extensions.checkNoModifiers(otherC, 'CompareWorker.doCompare', 'compose', otherVS.vurl)
+    Extensions.checkNoModifiers(otherC, 'CompareWorker.doCompare', 'ValueSet.compose', otherVS.vurl)
     this.checkNoLockedDate(otherVS.vurl, otherC);
 
     let systems = new Map(); // tracks whether the comparison is version dependent or not