fhirsmith 0.18.1 → 1.0.0

package/tx/tx.js

@@ -14,7 +14,7 @@ const path = require('path');
 const fs = require('fs');
 const Logger = require('../library/logger');
 const { Library } = require('./library');
-const { OperationContext, ResourceCache, ExpansionCache, debugLog} = require('./operation-context');
+const { OperationContext, ResourceCache, ExpansionCache, ExpansionCacheBudget, debugLog} = require('./operation-context');
 const { LanguageDefinitions } = require('../library/languages');
 const { I18nSupport } = require('../library/i18nsupport');
 const { CodeSystemXML } = require('./xml/codesystem-xml');
@@ -25,7 +25,7 @@ const packageJson = require("../package.json");
 // Import workers
 const ReadWorker = require('./workers/read');
 const SearchWorker = require('./workers/search');
-const { ExpandWorker, INTERNAL_DEFAULT_LIMIT, EXTERNAL_TEST_DEFAULT_LIMIT} = require('./workers/expand');
+const { ExpandWorker, INTERNAL_DEFAULT_LIMIT, EXTERNAL_DEFAULT_LIMIT, WALK_LIMIT_FACTOR } = require('./workers/expand');
 const { ValidateWorker } = require('./workers/validate');
 const TranslateWorker = require('./workers/translate');
 const LookupWorker = require('./workers/lookup');
@@ -61,6 +61,7 @@ const { buildOperationOutcome, outcomeFromError } = require('./library/operation
 const txOpenApi = require('./openapi');
 const { SyndicationStore } = require('./syndication/syndication-store');
 const syndicationFeed = require('./syndication/syndication-feed');
+const { EcosystemResolver, FORWARDED_HEADER } = require('./ecosystem-resolver');
 // const {writeFileSync} = require("fs");
 
 class TXModule {
@@ -138,6 +139,62 @@ class TXModule {
     }
   }
 
+  /**
+   * The memory limits in effect (see "Configuration Options" in tx/README.md): each from the tx setting
+   * of the same name, or its default when left out (or null); 0 = no limit (Infinity).
+   * @param {Object} config - Module configuration
+   * @returns {{cacheMaxConcepts: number, cacheMaxTotalConcepts: number, usageTrackerMaxCodes: number,
+   *   usageTrackerMaxCodeLength: number, languageTagCacheSize: number}}
+   */
+  static resourceLimits(config) {
+    const limit = (name, dflt) => {
+      let n = Number(config[name] ?? dflt);
+      if (!Number.isFinite(n) || n < 0) {
+        n = dflt; // not a number: the default, not silently no limit
+      }
+      return n === 0 ? Infinity : n;
+    };
+    return {
+      cacheMaxConcepts: limit('cacheMaxConcepts', ResourceCache.MAX_CONCEPTS_PER_CACHE),
+      cacheMaxTotalConcepts: limit('cacheMaxTotalConcepts', ResourceCache.MAX_TOTAL_CONCEPTS),
+      usageTrackerMaxCodes: limit('usageTrackerMaxCodes', ConceptUsageTracker.MAX_CODES),
+      usageTrackerMaxCodeLength: limit('usageTrackerMaxCodeLength', ConceptUsageTracker.MAX_CODE_LENGTH),
+      languageTagCacheSize: limit('languageTagCacheSize', LanguageDefinitions.MAX_PARSED)
+    };
+  }
+
+  /**
+   * The expansion limits, from the configuration or the defaults: external (the most codes an
+   * $expand returns; externalLimit, default 10000), internal (the most an expansion builds, an
+   * imported value set's for one; internalLimit, default 100000) and walk (the furthest a paged
+   * $expand walks to reach its page; walkLimit, default 10 x internal). Each, when set, must be a
+   * positive whole number, and external <= internal <= walk: anything else stops startup
+   * @returns {{external: number, internal: number, walk: number}}
+   */
+  static expansionLimits(config = {}) {
+    const read = (name, dflt) => {
+      const v = config[name];
+      if (v === undefined || v === null) {
+        return dflt;
+      }
+      const n = typeof v === 'number' || (typeof v === 'string' && /^\s*\d+\s*$/.test(v)) ? Number(v) : NaN;
+      if (!Number.isInteger(n) || n <= 0) {
+        throw new Error(`modules.tx.${name} must be a whole number above 0, not '${v}'`);
+      }
+      return n;
+    };
+    const internal = read('internalLimit', INTERNAL_DEFAULT_LIMIT);
+    const external = read('externalLimit', Math.min(EXTERNAL_DEFAULT_LIMIT, internal));
+    const walk = read('walkLimit', internal * WALK_LIMIT_FACTOR);
+    if (external > internal) {
+      throw new Error(`modules.tx.externalLimit (${external}) can't be more than internalLimit (${internal}): an expansion can't return more codes than it builds`);
+    }
+    if (internal > walk) {
+      throw new Error(`modules.tx.walkLimit (${walk}) can't be less than internalLimit (${internal})`);
+    }
+    return { external, internal, walk };
+  }
+
   /**
    * Initialize the TX module
    * @param {Object} config - Module configuration
@@ -156,9 +213,15 @@ class TXModule {
       consoleErrors: config.consoleErrors,
       telnetErrors: config.telnetErrors
     });
-    this.usageTracker = new ConceptUsageTracker();
+    this.limits = TXModule.resourceLimits(config);
+    this.expansionLimitValues = TXModule.expansionLimits(config);
+    this.usageTracker = new ConceptUsageTracker(this.limits.usageTrackerMaxCodes, this.limits.usageTrackerMaxCodeLength);
 
     this.log.info('Initializing TX module');
+    this.log.info(`Limits: operationTimeLimit=${config.operationTimeLimit ?? 20}s, cacheTimeout=${config.cacheTimeout || 30}min, ` +
+      Object.entries(this.limits).map(([name, value]) => `${name}=${value === Infinity ? 'unlimited' : value}`).join(', '));
+    this.log.info(`Expansion limits: externalLimit=${this.expansionLimitValues.external}, internalLimit=${this.expansionLimitValues.internal}, ` +
+      `walkLimit=${this.expansionLimitValues.walk}`);
 
     // Optional instance code, prefixed onto every cache-id this server issues so a
     // proxy in front of several instances can route by it (see nginx.md). Checked
@@ -188,6 +251,7 @@ class TXModule {
     const langPath = path.join(__dirname, 'data');
     this.log.info(`Loading language definitions from: ${langPath}`);
     this.languages = await LanguageDefinitions.fromFiles(langPath);
+    this.languages.maxParsed = this.limits.languageTagCacheSize;
     this.log.info('Language definitions loaded');
 
     // Initialize i18n support
@@ -202,12 +266,19 @@ class TXModule {
     // the database can't be opened, that stops startup - silently running without it
     // would be worse.
     this.closureStore = null;
-    if (config.closure && config.closure.enabled) {
-      this.closureStore = new ClosureStore(config.closure, this.log);
+    // a tx-worker (cluster.md) opens neither the closure tables nor the syndication store:
+    // the master owns them, and $closure and the feed are routed to it - unless
+    // server.cluster.sharedClosure is on (set only with a role), when every process
+    // answers $closure on the one database (see closure-store.js)
+    const worker = config.clusterRole === 'tx-worker';
+    const sharedClosure = config.sharedClosure === true;
+    if (config.closure && config.closure.enabled && (!worker || sharedClosure)) {
+      this.closureStore = new ClosureStore(config.closure, this.log, { shared: sharedClosure });
       this.closureStore.open();
-      this.log.info(`$closure enabled: tables in ${this.closureStore.path}`);
+      this.log.info(`$closure enabled: tables in ${this.closureStore.path}${sharedClosure ? ', shared with the other processes' : ''}`);
       const days = config.closure.retentionDays;
-      if (days) {
+      // with the tables shared, the master alone drops the stale ones
+      if (days && !worker) {
         const prune = async () => {
           try {
             const dropped = await this.closureStore.pruneUnused(days);
@@ -225,7 +296,7 @@ class TXModule {
 
     // Initialize metadata handler with config
     this.metadataHandler = new MetadataHandler({
-      closure: this.closureStore != null,
+      closure: this.closureStore != null || (worker && !!(config.closure && config.closure.enabled)),
       baseUrl: config.baseUrl,
       serverVersion: packageJson.version,
       txVersion: packageJson.txVersion,
@@ -243,14 +314,31 @@ class TXModule {
 
     // Load the library from YAML
     this.log.info(`Loading library from: ${config.librarySource}`);
-    this.library = new Library(config.librarySource, config.vsacCfg, this.log, this.stats);
+    this.library = new Library(config.librarySource, config.vsacCfg, this.log, this.stats, config.snomed);
+    if (config.pruneTerminologyCache !== undefined && typeof config.pruneTerminologyCache !== 'boolean') {
+      throw new Error(`modules.tx.pruneTerminologyCache must be true or false, not '${config.pruneTerminologyCache}'`);
+    }
+    // delete the .db and .cache files and package folders in terminology-cache that aren't used
+    // (once the endpoints are set up: see below)
+    this.library.pruneTerminologyCacheFiles = config.pruneTerminologyCache === true;
+    if (config.clusterRole) {
+      // the folder:/file: source caches are the master's, under one name every process shares
+      this.library.localSourceTag = 'master';
+      // the master builds the package caches; a worker only reads them
+      this.library.packageCachesReadOnly = config.clusterRole === 'tx-worker';
+      // the master refreshes the IANA media types registry; a worker reads its copy
+      this.library.mimeTypesReadOnly = config.clusterRole === 'tx-worker';
+      // the LOINC, RxNorm, NDC, UNII, CPT and OMOP databases are the master's; a worker opens
+      // them read-only
+      this.library.terminologyDatabasesReadOnly = config.clusterRole === 'tx-worker';
+    }
 
     // The syndication feed: content is recorded in the store as the library loads it, from
     // the time the administrator turns it on (see tx/syndication). One store serves every
     // endpoint, since the library is shared. If it is turned on and the store can't be
     // opened, that stops startup.
     this.syndicationStore = null;
-    if (config.syndication && config.syndication.enabled) {
+    if (config.syndication && config.syndication.enabled && !worker) {
       this.syndicationStore = new SyndicationStore(path.join(this.library.cacheFolder, 'syndication'), this.log).open();
       this.library.syndication = this.syndicationStore;
       this.log.info(`Syndication feed enabled: store in ${this.syndicationStore.folder}`);
@@ -263,6 +351,9 @@ class TXModule {
     for (const endpoint of config.endpoints) {
       await this.setupEndpoint(endpoint, app);
     }
+    // everything the library and the endpoints use is loaded: with pruneTerminologyCache, the
+    // rest of terminology-cache can go
+    await this.library.pruneUnusedCaches();
 
     this.log.info(`TX module initialized with ${config.endpoints.length} endpoint(s)`);
 
@@ -300,17 +391,38 @@ class TXModule {
     const cacheTimeoutMinutes = this.config.cacheTimeout || 30;
     const expansionCacheSize = this.config.expansionCacheSize || 1000;
     const expansionCacheMemoryThreshold = this.config.expansionCacheMemoryThreshold || 0;
+    // one budget of codes for the expansion caches of all the endpoints
+    if (!this.expansionCacheBudget) {
+      this.expansionCacheBudget = new ExpansionCacheBudget(this.config.expansionCacheMaxCodes || 0);
+      this.log.info(`Expansion caches: at most ${this.expansionCacheBudget.maxCodes} codes in all` +
+        (this.config.expansionCacheMaxCodes ? '' : ' (a quarter of the heap limit, at 1 KB a code)'));
+    }
 
     // Store endpoint info for provider creation
     const endpointInfo = {
       path: endpointPath,
       fhirVersion,
       context: context || null,
-      resourceCache: new ResourceCache(this.stats),
-      expansionCache: new ExpansionCache(this.stats, expansionCacheSize, expansionCacheMemoryThreshold)
+      // server.cluster.sharedCache (cluster.md, "Options"; set only with a role): one SQLite
+      // file that every process shares. Otherwise this process's memory, as ever
+      resourceCache: this.config.sharedCache ? this.createSharedResourceCache(endpointPath) : new ResourceCache(this.stats),
+      expansionCache: new ExpansionCache(this.stats, expansionCacheSize, expansionCacheMemoryThreshold, this.expansionCacheBudget)
     };
     // Create the provider once for this endpoint
     endpointInfo.provider = await this.library.cloneWithFhirVersion(fhirVersion, context, endpointPath);
+    for (const old of ['ecosystemResolution', 'ecosystemExcludedServers']) {
+      if (this.config[old] !== undefined) {
+        throw new Error(`modules.tx.${old} has been replaced by modules.tx.ecosystem (see tx/README.md)`);
+      }
+    }
+    const ecosystem = EcosystemResolver.checkConfig(this.config.ecosystem);
+    if (ecosystem) {
+      if (!this.ecosystemResolver) {
+        this.ecosystemResolver = new EcosystemResolver(ecosystem);
+        this.log.info(`Code systems not held here are looked for in the ecosystem (${this.ecosystemResolver.registry})`);
+      }
+      endpointInfo.provider.ecosystem = this.ecosystemResolver;
+    }
 
     // Set up periodic pruning of the resource cache
     // cacheTimeout is in minutes, default to 30 minutes
@@ -321,6 +433,7 @@ class TXModule {
     // than guessing at a number it has no way to see.
     endpointInfo.resourceCache.setIdleTimeout(cacheTimeoutMs);
     endpointInfo.resourceCache.setInstanceCode(this.config.instanceCode);
+    endpointInfo.resourceCache.setLimits(this.limits.cacheMaxConcepts, this.limits.cacheMaxTotalConcepts);
     if (this.stats) {
       this.stats.addTask("Client Cache", "5 min");
     }
@@ -382,6 +495,9 @@ class TXModule {
       // others read req.body / query directly). An explicit cache-id parameter
       // on the request still wins (see setupAdditionalResources).
       opContext.cacheId = req.get('X-Cache-Id') || null;
+      // a request another server sent on while resolving a code system through the ecosystem
+      // isn't resolved through the ecosystem again (see EcosystemResolver)
+      opContext.ecosystemForwarded = !!req.get(FORWARDED_HEADER);
 
       // Attach everything to request
       req.txProvider = endpointInfo.provider;
@@ -465,7 +581,8 @@ class TXModule {
             res.setHeader('Content-Type', jsonFmt);
             this.checkProperJson(jsonStr);
             responseSize = Buffer.byteLength(jsonStr, 'utf8');
-            result = originalJson(data);
+            // send the string built above: res.json(data) would serialise the resource again
+            result = res.send(jsonStr);
           }
 
           // Log the request with request ID
@@ -494,6 +611,12 @@ class TXModule {
     router.use((req, res, next) => {
       const contentType = req.get('Content-Type') || '';
 
+      // The body was bigger than the server accepts (server.maxBodyMB; see below)
+      if (req.bodyTooLarge) {
+        return res.status(413).json(buildOperationOutcome('error', 'too-costly',
+          `The request body is bigger than this server accepts (${req.bodyTooLarge.limit} bytes)`));
+      }
+
       // Only process POST/PUT
       if (req.method !== 'POST' && req.method !== 'PUT') {
         return next();
@@ -568,6 +691,17 @@ class TXModule {
       next();
     });
 
+    // A body over the size limit is refused by the body parsers, which run before this
+    // router, and Express would answer with its HTML error page. Note it and carry on, so
+    // the router answers with an OperationOutcome, in the format the client asked for
+    app.use(endpointPath, (err, req, res, next) => {
+      if (err.type !== 'entity.too.large') {
+        return next(err);
+      }
+      req.bodyTooLarge = err;
+      next();
+    });
+
     // Register the router with the app
     app.use(endpointPath, router);
     this.routers.set(endpointPath, router);
@@ -645,7 +779,7 @@ class TXModule {
         let worker = new LookupWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handle(req, res);
       } finally {
-        this.countRequest(endpointPath, '$lookup', Date.now() - start);
+        this.countRequest(endpointPath, '$lookup', Date.now() - start, req);
       }
     });
     router.post('/CodeSystem/\\$lookup', async (req, res) => {
@@ -654,7 +788,7 @@ class TXModule {
         let worker = new LookupWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handle(req, res);
       } finally {
-        this.countRequest(endpointPath, '$lookup', Date.now() - start);
+        this.countRequest(endpointPath, '$lookup', Date.now() - start, req);
       }
     });
 
@@ -665,7 +799,7 @@ class TXModule {
         let worker = new SubsumesWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handle(req, res);
       } finally {
-        this.countRequest(endpointPath, '$subsumes', Date.now() - start);
+        this.countRequest(endpointPath, '$subsumes', Date.now() - start, req);
       }
     });
     router.post('/CodeSystem/\\$subsumes', async (req, res) => {
@@ -674,7 +808,7 @@ class TXModule {
         let worker = new SubsumesWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handle(req, res);
       } finally {
-        this.countRequest(endpointPath, '$subsumes', Date.now() - start);
+        this.countRequest(endpointPath, '$subsumes', Date.now() - start, req);
       }
     });
 
@@ -685,7 +819,7 @@ class TXModule {
         let worker = new ValidateWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handleCodeSystem(req, res);
       } finally {
-        this.countRequest(endpointPath, '$validate', Date.now() - start);
+        this.countRequest(endpointPath, '$validate', Date.now() - start, req);
       }
     });
     router.post('/CodeSystem/\\$validate-code', async (req, res) => {
@@ -694,7 +828,7 @@ class TXModule {
         let worker = new ValidateWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handleCodeSystem(req, res);
       } finally {
-        this.countRequest(endpointPath, '$validate', Date.now() - start);
+        this.countRequest(endpointPath, '$validate', Date.now() - start, req);
       }
     });
 
@@ -705,7 +839,7 @@ class TXModule {
         let worker = new BatchValidateWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handleCodeSystem(req, res);
       } finally {
-        this.countRequest(endpointPath, '$batch', Date.now() - start);
+        this.countRequest(endpointPath, '$batch', Date.now() - start, req);
       }
     });
     router.post('/CodeSystem/\\$batch-validate-code', async (req, res) => {
@@ -714,7 +848,7 @@ class TXModule {
         let worker = new BatchValidateWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handleCodeSystem(req, res);
       } finally {
-        this.countRequest(endpointPath, '$batch', Date.now() - start);
+        this.countRequest(endpointPath, '$batch', Date.now() - start, req);
       }
     });
     // ValueSet/$validate-code (GET and POST)
@@ -724,7 +858,7 @@ class TXModule {
         let worker = new ValidateWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handleValueSet(req, res);
       } finally {
-        this.countRequest(endpointPath, '$validate', Date.now() - start);
+        this.countRequest(endpointPath, '$validate', Date.now() - start, req);
       }
     });
     router.post('/ValueSet/\\$validate-code', async (req, res) => {
@@ -733,7 +867,7 @@ class TXModule {
         let worker = new ValidateWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handleValueSet(req, res);
       } finally {
-        this.countRequest(endpointPath, '$validate', Date.now() - start);
+        this.countRequest(endpointPath, '$validate', Date.now() - start, req);
       }
     });
 
@@ -744,7 +878,7 @@ class TXModule {
         let worker = new CompareWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handle(req, res);
       } finally {
-        this.countRequest(endpointPath, '$compare', Date.now() - start);
+        this.countRequest(endpointPath, '$compare', Date.now() - start, req);
       }
     });
     router.post('/ValueSet/\\$compare', async (req, res) => {
@@ -753,7 +887,7 @@ class TXModule {
         let worker = new CompareWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handle(req, res);
       } finally {
-        this.countRequest(endpointPath, '$compare', Date.now() - start);
+        this.countRequest(endpointPath, '$compare', Date.now() - start, req);
       }
     });
 
@@ -764,7 +898,7 @@ class TXModule {
         let worker = new BatchValidateWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handleValueSet(req, res);
       } finally {
-        this.countRequest(endpointPath, '$batch', Date.now() - start);
+        this.countRequest(endpointPath, '$batch', Date.now() - start, req);
       }
     });
     router.post('/ValueSet/\\$batch-validate-code', async (req, res) => {
@@ -773,7 +907,7 @@ class TXModule {
         let worker = new BatchValidateWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handleValueSet(req, res);
       } finally {
-        this.countRequest(endpointPath, 'validate', Date.now() - start);
+        this.countRequest(endpointPath, 'validate', Date.now() - start, req);
       }
     });
 
@@ -781,19 +915,19 @@ class TXModule {
     router.get('/ValueSet/\\$expand', async (req, res) => {
       const start = Date.now();
       try {
-        let worker = new ExpandWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n, this.internalLimit(req), this.externalLimit(req));
+        let worker = new ExpandWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n, this.internalLimit(req), this.externalLimit(req), this.walkLimit(req));
         await worker.handle(req, res, this.log);
       } finally {
-        this.countRequest(endpointPath, '$expand', Date.now() - start);
+        this.countRequest(endpointPath, '$expand', Date.now() - start, req);
       }
     });
     router.post('/ValueSet/\\$expand', async (req, res) => {
       const start = Date.now();
       try {
-        let worker = new ExpandWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n, this.internalLimit(req), this.externalLimit(req));
+        let worker = new ExpandWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n, this.internalLimit(req), this.externalLimit(req), this.walkLimit(req));
         await worker.handle(req, res, this.log);
       } finally {
-        this.countRequest(endpointPath, '$expand', Date.now() - start);
+        this.countRequest(endpointPath, '$expand', Date.now() - start, req);
       }
     });
 
@@ -804,7 +938,7 @@ class TXModule {
         let worker = new CacheControlWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handle(req, res, this.log);
       } finally {
-        this.countRequest(endpointPath, '$cache-control', Date.now() - start);
+        this.countRequest(endpointPath, '$cache-control', Date.now() - start, req);
       }
     });
     router.post('/\\$cache-control', async (req, res) => {
@@ -813,7 +947,7 @@ class TXModule {
         let worker = new CacheControlWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handle(req, res, this.log);
       } finally {
-        this.countRequest(endpointPath, '$cache-control', Date.now() - start);
+        this.countRequest(endpointPath, '$cache-control', Date.now() - start, req);
       }
     });
 
@@ -824,7 +958,7 @@ class TXModule {
         let worker = new TranslateWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handle(req, res, this.log);
       } finally {
-        this.countRequest(endpointPath, '$translate', Date.now() - start);
+        this.countRequest(endpointPath, '$translate', Date.now() - start, req);
       }
     });
     router.post('/ConceptMap/\\$translate', async (req, res) => {
@@ -833,7 +967,7 @@ class TXModule {
         let worker = new TranslateWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handle(req, res, this.log);
       } finally {
-        this.countRequest(endpointPath, '$translate', Date.now() - start);
+        this.countRequest(endpointPath, '$translate', Date.now() - start, req);
       }
     });
 
@@ -845,7 +979,7 @@ class TXModule {
         let worker = new ClosureWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n, this.closureStore, this.config.closure);
         await worker.handle(req, res, this.log);
       } finally {
-        this.countRequest(endpointPath, '$closure', Date.now() - start);
+        this.countRequest(endpointPath, '$closure', Date.now() - start, req);
       }
     });
     router.post('/\\$closure', async (req, res) => {
@@ -854,7 +988,7 @@ class TXModule {
         let worker = new ClosureWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n, this.closureStore, this.config.closure);
         await worker.handle(req, res, this.log);
       } finally {
-        this.countRequest(endpointPath, '$closure', Date.now() - start);
+        this.countRequest(endpointPath, '$closure', Date.now() - start, req);
       }
     });
 
@@ -867,7 +1001,7 @@ class TXModule {
         let worker = new LookupWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handleInstance(req, res);
       } finally {
-        this.countRequest(endpointPath, '$lookup', Date.now() - start);
+        this.countRequest(endpointPath, '$lookup', Date.now() - start, req);
       }
     });
     router.post('/CodeSystem/:id/\\$lookup', async (req, res) => {
@@ -876,7 +1010,7 @@ class TXModule {
         let worker = new LookupWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handleInstance(req, res);
       } finally {
-        this.countRequest(endpointPath, '$lookup', Date.now() - start);
+        this.countRequest(endpointPath, '$lookup', Date.now() - start, req);
       }
     });
 
@@ -887,7 +1021,7 @@ class TXModule {
         let worker = new SubsumesWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handleInstance(req, res);
       } finally {
-        this.countRequest(endpointPath, '$subsumes', Date.now() - start);
+        this.countRequest(endpointPath, '$subsumes', Date.now() - start, req);
       }
     });
     router.post('/CodeSystem/:id/\\$subsumes', async (req, res) => {
@@ -896,7 +1030,7 @@ class TXModule {
         let worker = new SubsumesWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handleInstance(req, res);
       } finally {
-        this.countRequest(endpointPath, '$subsumes', Date.now() - start);
+        this.countRequest(endpointPath, '$subsumes', Date.now() - start, req);
       }
     });
 
@@ -907,7 +1041,7 @@ class TXModule {
         let worker = new ValidateWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handleCodeSystemInstance(req, res, this.log);
       } finally {
-        this.countRequest(endpointPath, '$validate', Date.now() - start);
+        this.countRequest(endpointPath, '$validate', Date.now() - start, req);
       }
     });
     router.post('/CodeSystem/:id/\\$validate-code', async (req, res) => {
@@ -916,7 +1050,7 @@ class TXModule {
         let worker = new ValidateWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handleCodeSystemInstance(req, res, this.log);
       } finally {
-        this.countRequest(endpointPath, '$validate', Date.now() - start);
+        this.countRequest(endpointPath, '$validate', Date.now() - start, req);
       }
 
     });
@@ -928,7 +1062,7 @@ class TXModule {
         let worker = new ValidateWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handleValueSetInstance(req, res, this.log);
       } finally {
-        this.countRequest(endpointPath, '$validate', Date.now() - start);
+        this.countRequest(endpointPath, '$validate', Date.now() - start, req);
       }
     });
     router.post('/ValueSet/:id/\\$validate-code', async (req, res) => {
@@ -937,7 +1071,7 @@ class TXModule {
         let worker = new ValidateWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handleValueSetInstance(req, res, this.log);
       } finally {
-        this.countRequest(endpointPath, '$validate', Date.now() - start);
+        this.countRequest(endpointPath, '$validate', Date.now() - start, req);
       }
     });
 
@@ -949,7 +1083,7 @@ class TXModule {
         let worker = new CompareWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handleInstance(req, res, this.log);
       } finally {
-        this.countRequest(endpointPath, '$compare', Date.now() - start);
+        this.countRequest(endpointPath, '$compare', Date.now() - start, req);
       }
     });
     router.post('/ValueSet/:id/\\$compare', async (req, res) => {
@@ -958,7 +1092,7 @@ class TXModule {
         let worker = new CompareWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handleInstance(req, res, this.log);
       } finally {
-        this.countRequest(endpointPath, '$compare', Date.now() - start);
+        this.countRequest(endpointPath, '$compare', Date.now() - start, req);
       }
     });
 
@@ -966,19 +1100,19 @@ class TXModule {
     router.get('/ValueSet/:id/\\$expand', async (req, res) => {
       const start = Date.now();
       try {
-        let worker = new ExpandWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n, this.internalLimit(req), this.externalLimit(req));
+        let worker = new ExpandWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n, this.internalLimit(req), this.externalLimit(req), this.walkLimit(req));
         await worker.handleInstance(req, res, this.log);
       } finally {
-        this.countRequest(endpointPath, '$expand', Date.now() - start);
+        this.countRequest(endpointPath, '$expand', Date.now() - start, req);
       }
     });
     router.post('/ValueSet/:id/\\$expand', async (req, res) => {
       const start = Date.now();
       try {
-        let worker = new ExpandWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n, this.internalLimit(req), this.externalLimit(req));
+        let worker = new ExpandWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n, this.internalLimit(req), this.externalLimit(req), this.walkLimit(req));
         await worker.handleInstance(req, res, this.log);
       } finally {
-        this.countRequest(endpointPath, '$expand', Date.now() - start);
+        this.countRequest(endpointPath, '$expand', Date.now() - start, req);
       }
     });
 
@@ -989,7 +1123,7 @@ class TXModule {
         let worker = new TranslateWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handleInstance(req, res, this.log);
       } finally {
-        this.countRequest(endpointPath, '$translate', Date.now() - start);
+        this.countRequest(endpointPath, '$translate', Date.now() - start, req);
       }
     });
     router.post('/ConceptMap/:id/\\$translate', async (req, res) => {
@@ -998,7 +1132,7 @@ class TXModule {
         let worker = new TranslateWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handleInstance(req, res, this.log);
       } finally {
-        this.countRequest(endpointPath, '$translate', Date.now() - start);
+        this.countRequest(endpointPath, '$translate', Date.now() - start, req);
       }
     });
 
@@ -1020,7 +1154,7 @@ class TXModule {
           let worker = new ReadWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
           await worker.handle(req, res, resourceType, (req, res, resourceType, id, matches) => this.renderIdChoices(req, res, resourceType, id, matches));
         } finally {
-          this.countRequest(endpointPath, 'read', Date.now() - start);
+          this.countRequest(endpointPath, 'read', Date.now() - start, req);
         }
       });
     }
@@ -1033,7 +1167,7 @@ class TXModule {
           let worker = new SearchWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
           await worker.handle(req, res, resourceType);
         } finally {
-          this.countRequest(endpointPath, 'search', Date.now() - start);
+          this.countRequest(endpointPath, 'search', Date.now() - start, req);
         }
       });
       router.post(`/${resourceType}/_search`, async (req, res) => {
@@ -1042,7 +1176,7 @@ class TXModule {
           let worker = new SearchWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
           await worker.handle(req, res, resourceType);
         } finally {
-          this.countRequest(endpointPath, 'search', Date.now() - start);
+          this.countRequest(endpointPath, 'search', Date.now() - start, req);
         }
       });
     }
@@ -1068,7 +1202,7 @@ class TXModule {
           // anything else falls through to the not-found fallback rather than hanging
           return next();
         } finally {
-          this.countRequest(endpointPath, '$read', Date.now() - start);
+          this.countRequest(endpointPath, '$read', Date.now() - start, req);
         }
       });
     }
@@ -1079,7 +1213,7 @@ class TXModule {
         let worker = new OperationsWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n);
         await worker.handle(req, res);
       } finally {
-        this.countRequest(endpointPath, '$op', Date.now() - start);
+        this.countRequest(endpointPath, '$op', Date.now() - start, req);
       }
     });
 
@@ -1095,7 +1229,7 @@ class TXModule {
         res.setHeader('Content-Type', 'text/html');
         res.send(html);
       } finally {
-        this.countRequest(endpointPath, 'ecl', Date.now() - start);
+        this.countRequest(endpointPath, 'ecl', Date.now() - start, req);
       }
     });
 
@@ -1109,7 +1243,7 @@ class TXModule {
         res.setHeader('Content-Type', 'text/html');
         res.send(html);
       } finally {
-        this.countRequest(endpointPath, 'problems', Date.now() - start);
+        this.countRequest(endpointPath, 'problems', Date.now() - start, req);
       }
     });
 
@@ -1147,7 +1281,7 @@ class TXModule {
             res.send(source);
           }
         } finally {
-          this.countRequest(endpointPath, 'library', Date.now() - start);
+          this.countRequest(endpointPath, 'library', Date.now() - start, req);
         }
       });
     }
@@ -1167,7 +1301,7 @@ class TXModule {
           res.status(500).json(this.operationOutcome('error', 'exception', error.message));
         }
       } finally {
-        this.countRequest(endpointPath, 'metadata', Date.now() - start);
+        this.countRequest(endpointPath, 'metadata', Date.now() - start, req);
       }
     });
 
@@ -1182,7 +1316,7 @@ class TXModule {
           res.status(500).json(this.operationOutcome('error', 'exception', error.message));
         }
       } finally {
-        this.countRequest(endpointPath, '$versions', Date.now() - start);
+        this.countRequest(endpointPath, '$versions', Date.now() - start, req);
       }
     });
 
@@ -1192,7 +1326,7 @@ class TXModule {
       try {
         await res.json(buildOperationOutcome('information', 'informational', `FHIR Terminology Server - FHIR v${req.txEndpoint.fhirVersion}`));
       } finally {
-        this.countRequest(endpointPath, 'home', Date.now() - start);
+        this.countRequest(endpointPath, 'home', Date.now() - start, req);
       }
     });
 
@@ -1217,7 +1351,7 @@ class TXModule {
         this.log.error(`Error rendering info page for ${req.params.id}: ${error.message}`);
         res.status(500).send('Internal server error');
       } finally {
-        this.countRequest(endpointPath, 'info', Date.now() - start);
+        this.countRequest(endpointPath, 'info', Date.now() - start, req);
       }
     };
     router.get('/info/:id', infoHandler);
@@ -1236,7 +1370,7 @@ class TXModule {
         res.status(404).json(this.operationOutcome('error', 'not-found',
           `Unknown ${what} on this server: ${req.method} ${req.baseUrl}${req.path}`, 'not-found'));
       } finally {
-        this.countRequest(endpointPath, 'unknown', Date.now() - start);
+        this.countRequest(endpointPath, 'unknown', Date.now() - start, req);
       }
     });
 
@@ -1380,6 +1514,39 @@ class TXModule {
     return `${counts.total.toLocaleString('en-US')} requests were served${where}: ${list}.`;
   }
 
+  /**
+   * An endpoint's $cache-control cache in the shared SQLite file (databases/resource-cache.db),
+   * for server.cluster.sharedCache. Endpoints share the file but not caches.
+   * @param {string} endpointPath - e.g. /r4
+   * @returns {SqliteResourceCache}
+   */
+  createSharedResourceCache(endpointPath) {
+    const { SqliteResourceCache } = require('./resource-cache-sqlite');
+    const { newCacheGeneration } = require('../library/cache-generation');
+    const folders = require('../library/folder-setup');
+    const file = path.join(folders.databasesDir(), 'resource-cache.db');
+    // from server.cluster.sharedCacheMaxMB, by applyRole (library/role.js, where the default
+    // is); 0: no cap
+    const maxBytes = this.config.sharedCacheMaxBytes ?? 0;
+    if (this.sharedCacheGeneration === undefined) {
+      // Caches don't survive a server restart: each time the master starts (under the
+      // launcher or by hand) it begins a new generation, and every live cache becomes a
+      // "lost to a server restart" tombstone. A tx-worker joins the current generation,
+      // so one restarting while the master carries on keeps every cache. One token for
+      // all of this process's endpoints
+      this.sharedCacheGeneration = this.config.clusterRole === 'master' ? newCacheGeneration() : null;
+    }
+    this.log.info(`Resource cache for ${endpointPath}: shared with the other processes, in ${file}, ` +
+      (maxBytes ? `client resources capped at ${+(maxBytes / 2 ** 20).toFixed(1)} MB for every endpoint together` : 'no size cap') +
+      ' (server.cluster.sharedCacheMaxMB)');
+    const cache = new SqliteResourceCache(this.stats, { path: file, scope: endpointPath, maxBytes,
+      generation: this.sharedCacheGeneration });
+    if (cache.lostToRestart) {
+      this.log.info(`Resource cache: ${cache.lostToRestart} cache(s) from before this start are now tombstones (lost to a server restart)`);
+    }
+    return cache;
+  }
+
   /**
    * Shutdown the module
    */
@@ -1389,6 +1556,13 @@ class TXModule {
       clearInterval(timer);
     }
     this.timers = [];
+    if (this.config && this.config.sharedCache) {
+      for (const ep of this.endpoints || []) {
+        if (ep.resourceCache) {
+          ep.resourceCache.close();
+        }
+      }
+    }
     if (this.syndicationStore) {
       // let an induction in progress finish, so the store isn't left with a half-copied file
       await this.syndicationStore.whenIdle();
@@ -1460,7 +1634,7 @@ class TXModule {
         this.log.error(`Error rendering syndication feed: ${error.message}`);
         res.status(500).json(this.operationOutcome('error', 'exception', 'Unable to render the syndication feed'));
       } finally {
-        this.countRequest(endpointPath, 'syndication', Date.now() - start);
+        this.countRequest(endpointPath, 'syndication', Date.now() - start, req);
       }
     };
 
@@ -1543,9 +1717,20 @@ class TXModule {
     return data;
   }
 
-  countRequest(endpoint, name, tat) {
+  /**
+   * Count a request. One that had to ask other servers (ecosystem resolution: the operation
+   * context counts the requests it sent) is counted as an external request: those take
+   * seconds, and are graphed separately (ServerStats.countExternalRequest)
+   *
+   * @param {Object} [req] - the request, for its operation context
+   */
+  countRequest(endpoint, name, tat, req = null) {
     if (this.stats) {
-      this.stats.countRequest(name, tat, endpoint);
+      if (req && req.txOpContext && req.txOpContext.externalRequests > 0) {
+        this.stats.countExternalRequest(name, tat, endpoint);
+      } else {
+        this.stats.countRequest(name, tat, endpoint);
+      }
     }
   }
 
@@ -1644,23 +1829,40 @@ class TXModule {
     }
   }
 
+  // the configured expansion limits, checked at startup (see expansionLimits). Every client gets
+  // the same: the tx-ecosystem test runner (User-Agent Tools/Java) used to be held to the
+  // defaults, and now asks for a lower limit where a test depends on one (x-too-costly-threshold)
+  currentExpansionLimits() {
+    return this.expansionLimitValues || TXModule.expansionLimits(this.config || {});
+  }
+
+  // eslint-disable-next-line no-unused-vars
   internalLimit(req) {
-    let isTest = req.header("User-Agent") == 'Tools/Java';
-    if (this.config.internalLimit && !isTest) return this.config.internalLimit; else return INTERNAL_DEFAULT_LIMIT;
+    return this.currentExpansionLimits().internal;
+  }
+
+  // How far a paged expansion (count and offset) may walk to reach its page: offset + count. Only
+  // the page is built, the codes before it are counted, so this can be well past internalLimit
+  // eslint-disable-next-line no-unused-vars
+  walkLimit(req) {
+    return this.currentExpansionLimits().walk;
   }
 
   externalLimit(req) {
-    let hdr = req.headers["x-too-costly-threshold"];
-    if (hdr) {
-      return parseInt(hdr);
+    let limit = this.currentExpansionLimits().external;
+    // x-too-costly-threshold sets the limit for this request - mostly so test cases can ask for
+    // a lower one. It can't be more than the internal limit, and a value that isn't a positive
+    // whole number is ignored: it never means "no limit"
+    const hdr = req.headers["x-too-costly-threshold"];
+    if (hdr !== undefined) {
+      const n = Number(hdr);
+      if (Number.isInteger(n) && n > 0) {
+        limit = Math.min(n, this.internalLimit(req));
+      }
     }
-    // the tx-ecosystem test runner (Tools/Java) always gets the test limit, so every server
-    // gives it the same answer. Otherwise the configured externalLimit - it used to be read
-    // only when internalLimit was configured too, so setting it alone did nothing
-    let isTest = req.header("User-Agent") == 'Tools/Java';
-    if (this.config.externalLimit && !isTest) return this.config.externalLimit; else return EXTERNAL_TEST_DEFAULT_LIMIT;
+    return limit;
   }
 
 }
 
-module.exports = TXModule;
+module.exports = TXModule;