fhirsmith 0.18.2 → 1.0.0

package/tx/operation-context.js

@@ -3,8 +3,15 @@ const inspector = require("inspector");
 const crypto = require("crypto");
 const {Languages} = require("../library/languages");
 const {Issue} = require("./library/operation-outcome");
+const {CanonicalResource} = require("./library/canonical-resource");
 const Logger = require("../library/logger");
-const { readCgroupMemoryLimit } = require('../library/cgroup-memory');
+const memoryGuard = require('../library/memory-guard');
+
+// When this process started, in ms, and the compact form of it that cache-ids carry when
+// the server has an instance code ("<code>.<start>.<uuid>"): base 36, 8 characters until
+// 2059. An id with this server's code and an earlier start was issued before a restart.
+const PROCESS_START = Math.round(performance.timeOrigin);
+const PROCESS_START_TOKEN = PROCESS_START.toString(36);
 
 /**
  * Check if running under a debugger
@@ -112,10 +119,71 @@ class ResourceCache {
     // to poll instead of guessing. Null = not advertised.
     this.idleTimeoutMs = null;
     // This server instance's code (modules.tx.instanceCode), prefixed onto every
-    // cache-id it issues as "<code>.<uuid>". A proxy in front of several instances
+    // cache-id it issues as "<code>.<start>.<uuid>" (the shared store's ids are
+    // "<code>.<generation>.<random>"). A proxy in front of several instances
     // routes on the prefix to get a cached request back to the one instance that
     // holds the cache (see nginx.md). Null = unprefixed ids (a single server).
     this.instanceCode = null;
+    // Most concepts one cache may hold (a request that would take it over drops it), and most
+    // all the caches may hold between them (the least recently used caches are
+    // dropped to stay under it). Set from cacheMaxConcepts / cacheMaxTotalConcepts.
+    this.maxConceptsPerCache = ResourceCache.MAX_CONCEPTS_PER_CACHE;
+    this.maxTotalConcepts = ResourceCache.MAX_TOTAL_CONCEPTS;
+  }
+
+  // Defaults for cacheMaxConcepts and cacheMaxTotalConcepts. 250,000 is about 10x the
+  // largest IG Publisher session measured on build.fhir.org (Nov 2026: 24,048 concepts);
+  // the total is 4 such full caches, about 250 MB of heap at about 250 bytes a concept.
+  static MAX_CONCEPTS_PER_CACHE = 250000;
+  static MAX_TOTAL_CONCEPTS = 1000000;
+
+  /**
+   * Set the concept limits (see the constructor). Infinity = no limit.
+   * @param {number} perCache
+   * @param {number} total
+   */
+  setLimits(perCache, total) {
+    this.maxConceptsPerCache = perCache;
+    this.maxTotalConcepts = total;
+  }
+
+  /**
+   * Refuse a change that would take one cache over its limit.
+   * @param {string} cacheId
+   * @param {number} concepts - what the cache would hold
+   */
+  _checkLimit(cacheId, concepts) {
+    if (concepts > this.maxConceptsPerCache) {
+      // (in English: the worker translates it into the request's language)
+      const err = new Issue('error', 'too-costly', null, 'CACHE_START_TOO_BIG',
+        `The cache '${cacheId}' would hold ${concepts} concepts, more than this server allows in one cache (${this.maxConceptsPerCache}, cacheMaxConcepts)`, 'too-costly', 422).handleAsOO(422);
+      err.msgParams = [cacheId, concepts, this.maxConceptsPerCache];
+      throw err;
+    }
+  }
+
+  /**
+   * Drop the least recently used caches (never the one just changed) while the
+   * caches hold more concepts than allowed between them.
+   * @param {string} keep - the cache-id just changed
+   */
+  _evict(keep) {
+    while (this.totalConcepts > this.maxTotalConcepts) {
+      let oldest = null;
+      for (const [cacheId, entry] of this.cache.entries()) {
+        if (cacheId !== keep && (!oldest || entry.lastUsed < oldest[1].lastUsed)) {
+          oldest = [cacheId, entry];
+        }
+      }
+      if (!oldest) {
+        return;
+      }
+      const [cacheId, entry] = oldest;
+      this.totalConcepts -= entry.concepts || 0;
+      this._recordTombstone(cacheId, entry, 'evicted');
+      this.log.info(`cache-id '${cacheId}': dropped to make room (held ${entry.concepts} concepts; the caches may hold ${this.maxTotalConcepts} in all)`);
+      this.cache.delete(cacheId);
+    }
   }
 
   /**
@@ -141,8 +209,9 @@ class ResourceCache {
       return null;
     }
     // At most 16 characters: the cache-id goes back to the client as a FHIR id
-    // (valueId), which is capped at 64 - and "<code>." plus a 36-character UUID
-    // has to fit.
+    // (valueId), which is capped at 64 - and "<code>." plus the 8-character start,
+    // a '.' and a 36-character UUID (or the shared store's two 22-character tokens)
+    // have to fit.
     if (typeof code !== 'string' || !/^[A-Za-z0-9]{1,16}$/.test(code)) {
       throw new Error(`tx instanceCode '${code}' is not valid: it must be 1-16 letters and digits`);
     }
@@ -150,18 +219,51 @@ class ResourceCache {
   }
 
   /**
-   * Mint a new cache-id: a UUID, prefixed with "<instanceCode>." when this server
-   * has an instance code. Minting only - the caller creates the entry with set().
+   * Mint a new cache-id: a UUID, or "<instanceCode>.<start>.<uuid>" when this server
+   * has an instance code, where <start> is when this process started (base 36), so a
+   * later process can tell an id issued before a restart from one never issued.
+   * Minting only - the caller creates the entry with set(), or uses start().
    * @returns {string}
    */
   newCacheId() {
     const uuid = crypto.randomUUID();
-    return this.instanceCode ? `${this.instanceCode}.${uuid}` : uuid;
+    return this.instanceCode ? `${this.instanceCode}.${PROCESS_START_TOKEN}.${uuid}` : uuid;
+  }
+
+  /**
+   * Create a cache ($cache-control?mode=start): mint its id, store the resources.
+   * @param {Array} resources
+   * @param {boolean} sealed
+   * @returns {string} the new cache-id
+   */
+  start(resources, sealed = false) {
+    const cacheId = this.newCacheId();
+    this.set(cacheId, resources, sealed);
+    return cacheId;
+  }
+
+  /**
+   * When the process that issued a cache-id with this server's instance code started
+   * (ms), or null if the id doesn't say (no code, another code, or the older
+   * "<code>.<uuid>" form).
+   * @param {string} cacheId
+   * @returns {number|null}
+   */
+  issuedStartOf(cacheId) {
+    if (!this.instanceCode || typeof cacheId !== 'string') {
+      return null;
+    }
+    const parts = cacheId.split('.');
+    if (parts.length !== 3 || parts[0] !== this.instanceCode || !/^[0-9a-z]{1,10}$/.test(parts[1])) {
+      return null;
+    }
+    return parseInt(parts[1], 36);
   }
 
   /**
    * The instance code in a cache-id, or null if it has none. Ids this server mints
-   * are either a bare UUID (no '.') or "<code>.<uuid>".
+   * are either a bare UUID (no '.') or "<code>.<...>": "<code>.<start>.<uuid>" from the
+   * memory store, "<code>.<generation>.<random>" from the shared one.
    * @param {string} cacheId
    * @returns {string|null}
    */
@@ -185,10 +287,12 @@ class ResourceCache {
    *
    * @param {string} cacheId - The cache identifier
    * @param {Object} entry - The entry being removed (for its timings)
-   * @param {'closed'|'expired'|'cleared'} reason - How it went away:
+   * @param {'closed'|'expired'|'cleared'|'evicted'|'discarded'} reason - How it went away:
    *   `closed` = the client released it with $cache-control?mode=end;
    *   `expired` = it went unused for longer than the idle timeout;
-   *   `cleared` = the server dropped every cache at once.
+   *   `cleared` = the server dropped every cache at once;
+   *   `evicted` = it was dropped, least recently used, to make room for others.
+   *   `discarded` = a request would have taken it over cacheMaxConcepts.
    * @param {Object} [extra] - Reason-specific detail (e.g. idleMs, maxAgeMs)
    */
   _recordTombstone(cacheId, entry, reason, extra = {}) {
@@ -235,6 +339,15 @@ class ResourceCache {
   describeMissing(cacheId) {
     const t = this.tombstone(cacheId);
     if (!t) {
+      // Our own code: the id says when the process that issued it started. An earlier
+      // start is a cache lost to a restart; this process's start, one it never issued
+      const issuedStart = this.issuedStartOf(cacheId);
+      if (issuedStart !== null && issuedStart < PROCESS_START) {
+        return { messageId: 'CACHE_ID_LOST_TO_RESTART', params: [cacheId, formatDuration(Date.now() - PROCESS_START)] };
+      }
+      if (issuedStart !== null) {
+        return { messageId: 'CACHE_ID_NEVER_ISSUED', params: [cacheId, formatDuration(Date.now() - PROCESS_START), this.maxTombstones] };
+      }
       // No record of it here. If its prefix names a different instance, say so:
       // that's a routing problem (proxy misconfigured, or the owning instance is
       // down and the request fell through to a backup), not a lifecycle one.
@@ -255,6 +368,10 @@ class ResourceCache {
           messageId: 'CACHE_ID_EXPIRED',
           params: [cacheId, formatDuration(t.idleMs), formatDuration(t.maxAgeMs), ago]
         };
+      case 'evicted':
+        return { messageId: 'CACHE_ID_EVICTED', params: [cacheId, ago] };
+      case 'discarded':
+        return { messageId: 'CACHE_ID_DISCARDED', params: [cacheId, ago, t.limit] };
       default:
         return { messageId: 'CACHE_ID_CLEARED', params: [cacheId, ago] };
     }
@@ -400,35 +517,77 @@ class ResourceCache {
       }
     }
 
-    const entry = this.cache.get(cacheId) || { resources: [], createdAt: Date.now(), lastUsed: Date.now(), concepts: 0 };
+    // What this request brings on its own (a resource sent twice counted once). If that alone is
+    // more than one cache may hold, no cache can take it - a new one would be dropped too, and the
+    // client would retry in vain - so the request is refused as too costly (422), and the cache,
+    // which isn't the problem, is kept
+    const own = new Map();
+    for (const resource of resources) {
+      own.set(this._resourceKey(resource), this._conceptsOf(resource));
+    }
+    let ownConcepts = 0;
+    for (const n of own.values()) {
+      ownConcepts += n;
+    }
+    if (ownConcepts > this.maxConceptsPerCache) {
+      // (in English: the worker translates it into the request's language - see addToClientCache)
+      const err = new Issue('error', 'too-costly', null, 'CACHE_REQUEST_TOO_BIG',
+        `This request's resources hold ${ownConcepts} concepts, more than this server allows in one cache (${this.maxConceptsPerCache}, cacheMaxConcepts), so no cache can hold them`, 'too-costly', 422).handleAsOO(422);
+      err.msgParams = [ownConcepts, this.maxConceptsPerCache];
+      throw err;
+    }
 
-    // Merge resources, avoiding duplicates by url+version. Keep the entry's concept
-    // subtotal and the cache-wide total in step with each insertion/replacement.
+    const entry = this.cache.get(cacheId) || { resources: [], createdAt: Date.now(), lastUsed: Date.now(), concepts: 0 };
+    // Merge resources, avoiding duplicates by url+version, into a copy: a change that would
+    // take the cache over its limit discards the cache instead.
+    const merged = [...entry.resources];
+    let concepts = entry.concepts;
     for (const resource of resources) {
       const key = this._resourceKey(resource);
-      const newConcepts = this._conceptsOf(resource);
-      const existingIndex = entry.resources.findIndex(r => this._resourceKey(r) === key);
+      const existingIndex = merged.findIndex(r => this._resourceKey(r) === key);
       if (existingIndex >= 0) {
-        // Replace existing: adjust by the difference
-        const delta = newConcepts - this._conceptsOf(entry.resources[existingIndex]);
-        entry.resources[existingIndex] = resource;
-        entry.concepts += delta;
-        this.totalConcepts += delta;
+        // Replace existing: count the difference
+        concepts += this._conceptsOf(resource) - this._conceptsOf(merged[existingIndex]);
+        merged[existingIndex] = resource;
         if (this.logCacheOps) {
           this.log.info(`cache-id '${cacheId}': replaced ${key}`);
         }
       } else {
-        entry.resources.push(resource);
-        entry.concepts += newConcepts;
-        this.totalConcepts += newConcepts;
+        concepts += this._conceptsOf(resource);
+        merged.push(resource);
         if (this.logCacheOps) {
           this.log.info(`cache-id '${cacheId}': added ${key}`);
         }
       }
     }
+    if (concepts > this.maxConceptsPerCache) {
+      // The client can't do anything with a cache it can't add to, so drop it, and don't
+      // perform this request: the next request with this id is told it was discarded
+      // (CACHE_ID_DISCARDED).
+      if (this.cache.has(cacheId)) {
+        this.totalConcepts -= entry.concepts || 0;
+        this.cache.delete(cacheId);
+      }
+      this._recordTombstone(cacheId, entry, 'discarded', { limit: this.maxConceptsPerCache });
+      this.log.info(`cache-id '${cacheId}': discarded (would hold ${concepts} concepts; one cache may hold ${this.maxConceptsPerCache})`);
+      // Answered as every other cache that is gone is (404, tx-issue-type cache-id-unknown), so a
+      // client needs one rule: forget what it thinks the server has, start a new cache, and send
+      // the request again (it has not been performed). The message id says why it is gone.
+      const params = [cacheId, this.maxConceptsPerCache];
+      const err = new Issue('error', 'not-found', null, 'CACHE_ID_DISCARDED_TOO_BIG',
+        `The cache '${cacheId}' grew too large: this request would have taken it over ${this.maxConceptsPerCache} concepts, ` +
+        'the most this server allows in one cache (cacheMaxConcepts), so the server has dropped the cache. ' +
+        'This operation has not been performed. Start a new cache, and send the request again', 'cache-id-unknown', 404).handleAsOO(404);
+      err.msgParams = params;
+      throw err;
+    }
+    this.totalConcepts += concepts - entry.concepts;
+    entry.resources = merged;
+    entry.concepts = concepts;
 
     entry.lastUsed = Date.now();
     this.cache.set(cacheId, entry);
+    this._evict(cacheId);
     this._trackMax();
   }
 
@@ -452,15 +611,16 @@ class ResourceCache {
         Object.defineProperty(resource, 'isCached', { value: true, enumerable: false, configurable: true, writable: true });
       }
     }
+    let concepts = 0;
+    for (const resource of resources) {
+      concepts += this._conceptsOf(resource);
+    }
+    this._checkLimit(cacheId, concepts);
     // Drop the old entry's contribution, then count the replacement.
     const existing = this.cache.get(cacheId);
     if (existing) {
       this.totalConcepts -= existing.concepts || 0;
     }
-    let concepts = 0;
-    for (const resource of resources) {
-      concepts += this._conceptsOf(resource);
-    }
     this.totalConcepts += concepts;
     this.cache.set(cacheId, {
       resources: [...resources],
@@ -471,6 +631,7 @@ class ResourceCache {
     });
     // This id is alive again, so any record of its previous death is now wrong.
     this.tombstones.delete(cacheId);
+    this._evict(cacheId);
     this._trackMax();
   }
 
@@ -592,10 +753,97 @@ class ResourceCache {
   }
 }
 
+/**
+ * How many codes the expansion caches may hold between them - one budget for the server, shared
+ * by the caches of all its endpoints. An expansion's cost in memory is roughly proportional to the
+ * number of codes in it (about 0.8 KB each, for SNOMED CT pages with displays), so counting codes
+ * bounds the memory the caches use without measuring anything. When a new entry takes the total
+ * over the budget, the least recently used entries - from whichever cache - are dropped.
+ */
+class ExpansionCacheBudget {
+  /**
+   * Assumed memory per cached code, used to derive the default budget from the heap size
+   */
+  static BYTES_PER_CODE = 1024;
+
+  /**
+   * Share of the heap the default budget allows the caches
+   */
+  static HEAP_SHARE = 0.25;
+
+  /**
+   * The default budget: a quarter of the heap limit (node's --max-old-space-size, or its default),
+   * at BYTES_PER_CODE a code
+   * @returns {number}
+   */
+  static defaultMaxCodes() {
+    const heapLimit = require('v8').getHeapStatistics().heap_size_limit;
+    return Math.floor(heapLimit * ExpansionCacheBudget.HEAP_SHARE / ExpansionCacheBudget.BYTES_PER_CODE);
+  }
+
+  /**
+   * @param {number} [maxCodes] - how many codes the caches may hold; 0 or missing = the default
+   */
+  constructor(maxCodes = 0) {
+    this.maxCodes = maxCodes > 0 ? maxCodes : ExpansionCacheBudget.defaultMaxCodes();
+    this.total = 0;
+    // every entry of every cache sharing this budget, least recently used first
+    this.lru = new Map();
+  }
+
+  /**
+   * Add an entry (already in its cache) and drop the least recently used entries while over budget
+   */
+  add(entry) {
+    this.lru.set(entry, true);
+    this.total += entry.codes;
+    for (const old of this.lru.keys()) {
+      if (this.total <= this.maxCodes) {
+        break;
+      }
+      old.owner.evictEntry(old);
+    }
+  }
+
+  touch(entry) {
+    if (this.lru.delete(entry)) {
+      this.lru.set(entry, true);
+    }
+  }
+
+  remove(entry) {
+    if (this.lru.delete(entry)) {
+      this.total -= entry.codes;
+    }
+  }
+}
+
+/**
+ * The number of codes in an expansion (all levels of a hierarchical one)
+ * @param {Object} expansion - an expanded ValueSet (wrapper or JSON)
+ * @returns {number}
+ */
+function countExpansionCodes(expansion) {
+  const json = expansion && expansion.jsonObj ? expansion.jsonObj : expansion;
+  let count = 0;
+  const walk = (list) => {
+    for (const c of list || []) {
+      count++;
+      if (c.contains) {
+        walk(c.contains);
+      }
+    }
+  };
+  walk(json && json.expansion ? json.expansion.contains : null);
+  return count;
+}
+
 /**
  * Cache for expanded ValueSets
  * Stores expansions keyed by hash of (valueSet, params, additionalResources)
  * Only caches expansions that took longer than the minimum cache time
+ * Bounded by an entry count, and by a budget of codes (see ExpansionCacheBudget), which may be
+ * shared with other caches
  */
 class ExpansionCache {
   /**
@@ -611,9 +859,12 @@ class ExpansionCache {
   /**
    * @param {number} maxSize - Maximum number of entries to keep (default 1000)
    * @param {number} memoryThresholdMB - Heap usage in MB that triggers dropping oldest half (0 = disabled)
+   * @param {ExpansionCacheBudget} [budget] - the budget of codes, shared with other caches (default: one of its own)
    */
-  constructor(stats, maxSize = ExpansionCache.DEFAULT_MAX_SIZE, memoryThresholdMB = 0) {
+  constructor(stats, maxSize = ExpansionCache.DEFAULT_MAX_SIZE, memoryThresholdMB = 0, budget = null) {
     this.stats = stats;
+    this.budget = budget || new ExpansionCacheBudget();
+    // least recently used first (get moves an entry to the end)
     this.cache = new Map();
     this.maxSize = maxSize;
     this.memoryThresholdBytes = memoryThresholdMB * 1024 * 1024;
@@ -631,9 +882,12 @@ class ExpansionCache {
    * @param {Object|ValueSet} valueSet - The ValueSet to expand (wrapper or JSON)
    * @param {Object} params - Parameters resource (tx-resource and valueSet params excluded)
    * @param {Array} additionalResources - Additional resources in scope (CodeSystem/ValueSet wrappers)
+   * @param {number} [limit] - the most codes the response may hold (the request's
+   *   x-too-costly-threshold, else the configured externalLimit): the same request under a lower
+   *   limit can be a 422, or lose its total, so it is a different answer
    * @returns {string} Hash key
    */
-  computeKey(valueSet, params, additionalResources) {
+  computeKey(valueSet, params, additionalResources, limit = null) {
     const keyParts = [];
 
     // ValueSet content - always hash the full JSON content
@@ -645,19 +899,22 @@ class ExpansionCache {
     if (params) {
       keyParts.push(`params:`+params.hashSource());
     }
+    if (limit != null) {
+      keyParts.push(`limit:${limit}`);
+    }
 
     // Additional resources - hash the full content of each resource
     // Resources are now CodeSystem/ValueSet wrappers, not raw JSON
     if (additionalResources && additionalResources.length > 0) {
       const resourceHashes = additionalResources
           .map(r => {
-            // Get the JSON object from wrapper or use directly
-            const json = r.jsonObj || r;
-            // Create a content hash for this resource
-            return crypto.createHash('sha256')
-                .update(JSON.stringify(json))
-                .digest('hex')
-                .substring(0, 16); // Use first 16 chars for brevity
+            // A wrapper remembers its hash (CanonicalResource.expansionKeyHash): a cache-id's
+            // resources come with every $expand, and serialising and hashing them all
+            // each time was most of the cost of a cached request. Raw JSON is hashed the
+            // same way, so the key is the same either way.
+            return typeof r.expansionKeyHash === 'function'
+              ? r.expansionKeyHash()
+              : CanonicalResource.computeExpansionKeyHash(r.jsonObj || r);
           })
           .sort();
       keyParts.push(`additional:${resourceHashes.join(',')}`);
@@ -679,6 +936,9 @@ class ExpansionCache {
     if (entry) {
       entry.lastUsed = Date.now();
       entry.hitCount++;
+      this.cache.delete(key);
+      this.cache.set(key, entry);
+      this.budget.touch(entry);
       return entry.expansion;
     }
     return null;
@@ -707,21 +967,44 @@ class ExpansionCache {
       return false;
     }
 
-    // Enforce max size before adding - evict oldest (by lastUsed) if needed
+    // an expansion bigger than the whole budget would only empty the caches
+    const codes = countExpansionCodes(expansion);
+    if (codes > this.budget.maxCodes) {
+      return false;
+    }
+
+    this.clear(key);
+    // Enforce max size before adding - evict the least recently used if needed
     if (this.cache.size >= this.maxSize) {
       this.evictOldest(1);
     }
 
-    this.cache.set(key, {
+    const entry = {
+      owner: this,
+      key: key,
       expansion: expansion,
+      codes: codes,
       createdAt: Date.now(),
       lastUsed: Date.now(),
       durationMs: durationMs,
       hitCount: 0
-    });
+    };
+    this.cache.set(key, entry);
+    this.budget.add(entry);
     return true;
   }
 
+  /**
+   * Drop an entry (the budget drops this cache's entries through this)
+   * @param {Object} entry
+   */
+  evictEntry(entry) {
+    if (this.cache.get(entry.key) === entry) {
+      this.cache.delete(entry.key);
+    }
+    this.budget.remove(entry);
+  }
+
   /**
    * Evict the N oldest entries by lastUsed time
    * @param {number} count - Number of entries to evict
@@ -730,15 +1013,12 @@ class ExpansionCache {
   evictOldest(count) {
     if (this.cache.size === 0 || count <= 0) return 0;
 
-    // Get entries sorted by lastUsed (oldest first)
-    const entries = Array.from(this.cache.entries())
-        .sort((a, b) => a[1].lastUsed - b[1].lastUsed);
-
-    const toEvict = Math.min(count, entries.length);
-    for (let i = 0; i < toEvict; i++) {
-      this.cache.delete(entries[i][0]);
+    // the map is in least recently used order
+    const toEvict = [...this.cache.values()].slice(0, count);
+    for (const entry of toEvict) {
+      this.evictEntry(entry);
     }
-    return toEvict;
+    return toEvict.length;
   }
 
   /**
@@ -780,14 +1060,19 @@ class ExpansionCache {
    * @param {string} key - Hash key
    */
   clear(key) {
-    this.cache.delete(key);
+    const entry = this.cache.get(key);
+    if (entry) {
+      this.evictEntry(entry);
+    }
   }
 
   /**
    * Clear all cached entries
    */
   clearAll() {
-    this.cache.clear();
+    for (const entry of [...this.cache.values()]) {
+      this.evictEntry(entry);
+    }
   }
 
   /**
@@ -799,12 +1084,17 @@ class ExpansionCache {
    */
   getStats() {
     let totalHits = 0;
+    let codes = 0;
     for (const entry of this.cache.values()) {
       totalHits += entry.hitCount;
+      codes += entry.codes;
     }
     return {
       size: this.cache.size,
       maxSize: this.maxSize,
+      codes,
+      budgetCodes: this.budget.total,
+      budgetMaxCodes: this.budget.maxCodes,
       memoryThresholdMB: this.memoryThresholdBytes > 0 ? this.memoryThresholdBytes / (1024 * 1024) : 0,
       totalHits
     };
@@ -816,17 +1106,6 @@ class ExpansionCache {
 }
 
 
-/**
- * Read the cgroup memory limit once at startup.
- * Returns the byte limit, or 0 if unavailable (disables the check).
- */
-function readMemoryLimit() {
-  return readCgroupMemoryLimit().limit;
-}
-
-const MEMORY_LIMIT = readMemoryLimit();
-const MEMORY_FRACTION = 0.98;
-const MEMORY_THRESHOLD = MEMORY_LIMIT > 0 ? MEMORY_LIMIT * MEMORY_FRACTION : 0; // MEMORY_FRACTION of cgroup limit
 const CHECK_FREQUENCY = 100;
 // How long an operation may compute without yielding the event loop (ms).
 // Node runs all JS on one thread: while a long operation executes, no other
@@ -835,7 +1114,7 @@ const CHECK_FREQUENCY = 100;
 // I/O. checkAndYield() awaits a real setImmediate every YIELD_INTERVAL ms of
 // compute, so pending I/O (new connections, /metadata, timers) runs between
 // slices and one heavy $expand can no longer time out every other client.
-const YIELD_INTERVAL = 25;
+const YIELD_INTERVAL = 10;
 const { setImmediate: yieldToEventLoop } = require('timers/promises');
 
 class OperationContext {
@@ -859,12 +1138,22 @@ class OperationContext {
     // Parameters (buildParameters, raw req.body, query/form). An explicit
     // cache-id *parameter* still takes precedence over this.
     this.cacheId = null;
+    this.ecosystemForwarded = false; // see EcosystemResolver
+    // how many requests answering this one sent to other servers (EcosystemResolver): if any,
+    // it's counted as an external request (ServerStats.countExternalRequest). Shared with the
+    // copies (a batch's entries), so what they send counts against the request
+    this._external = { requests: 0 };
     this.debugging = isDebugging();
     // Providers opened during this operation that need their underlying
     // resources (sqlite connections, etc.) released when the operation ends.
     // Shared by reference with copy()'d contexts so a sub-operation's
     // providers are cleaned up by the parent request's closeProviders().
-    this._openProviders = [];
+    // closed: closeProviders() has run, so one registered later (a build that
+    // finished after the client went) is closed at once - nothing else will.
+    // built: the providers built so far, factory -> variant -> provider (see
+    // buildProviderOnce), kept with the list that
+    // closes them, so whatever shares one shares the other.
+    this._providers = { open: [], closed: false, built: new Map() };
     // Compute-time accounting for the deadline and for yielding. One object,
     // shared by reference with copy()'d contexts, so an operation and its
     // sub-operations (e.g. batch entries) draw down a single budget:
@@ -896,11 +1185,27 @@ class OperationContext {
     return 'op_' + Math.random().toString(36).substr(2, 9) + '_' + Date.now();
   }
 
+  /** the number of requests sent to other servers for this operation (see EcosystemResolver) */
+  get externalRequests() {
+    return this._external.requests;
+  }
+
+  /** another request sent to another server for this operation */
+  countExternalRequest() {
+    this._external.requests++;
+  }
+
   /**
    * Create a copy of this operation context
+   * @param {Object} [options]
+   * @param {boolean} [options.ownProviders=false] - give the copy its own provider
+   *   list instead of sharing this one. The caller must then call closeProviders()
+   *   on the copy when its sub-operation is done (a batch does this per entry, so
+   *   an entry's providers - and through them the entry's context and log - are
+   *   released when the entry is answered, not held until the whole batch ends).
    * @returns {OperationContext}
    */
-  copy() {
+  copy(options = {}) {
     const newContext = new OperationContext(
         this.langs, this.i18n, this.id, this.timeLimit / 1000,
         this.resourceCache, this.expansionCache
@@ -912,9 +1217,16 @@ class OperationContext {
     newContext.debugging = this.debugging;
     newContext.cacheId = this.cacheId;
     newContext.usageTracker = this.usageTracker;
+    newContext.ecosystemForwarded = this.ecosystemForwarded;
+    newContext._external = this._external;
     // Share the same provider-cleanup list so providers opened by the copy
-    // are released when the parent operation ends.
-    newContext._openProviders = this._openProviders;
+    // are released when the parent operation ends - unless the caller will
+    // release them itself, with a list of its own.
+    if (options.ownProviders) {
+      newContext._providers = { open: [], closed: false, built: new Map() };
+    } else {
+      newContext._providers = this._providers;
+    }
     // Share the compute clock: sub-operations spend the parent's budget.
     newContext._clock = this._clock;
     return newContext;
@@ -940,6 +1252,8 @@ class OperationContext {
     if (this.debugging) {
       return false;
     }
+    // a flag, so cheap: once the client has gone, stop at the next check, not the next yield
+    this.abandonIfClientGone(place);
 
     OperationContext._checkCounter++;
     if (OperationContext._checkCounter < CHECK_FREQUENCY) {
@@ -960,18 +1274,20 @@ class OperationContext {
       throw error;
     }
 
-    // Memory check (piggyback on same sample)
-    if (MEMORY_THRESHOLD > 0) {
-      const rss = process.memoryUsage.rss();
-      if (rss > MEMORY_THRESHOLD) {
-        const usedGB = (rss / 1024 / 1024 / 1024).toFixed(1);
-        const limitGB = (MEMORY_LIMIT / 1024 / 1024 / 1024).toFixed(1);
-        this.log(`Memory Limit: ${usedGB} GB of ${limitGB} GB limit @ ${place}`);
-        const error = new Issue("error", "too-costly", null, null,
-            `Operation aborted: server memory usage (${usedGB} GB) exceeds safe threshold (${MEMORY_FRACTION * 100}% of ${limitGB} GB limit) at ${place}`, "too-costly", 422);
-        error.diagnostics = this.diagnostics();
-        throw error;
-      }
+    // Memory check (piggyback on same sample). Threshold and limit: library/memory-guard.js
+    const rss = memoryGuard.overAbortThreshold(place);
+    if (rss) {
+      const usedGB = memoryGuard.gb(rss);
+      const limitGB = memoryGuard.gb(memoryGuard._state.limit);
+      const pct = Math.round(memoryGuard._state.abortFraction * 100);
+      this.log(`Memory Limit: ${usedGB} GB of ${limitGB} GB limit @ ${place}`);
+      const error = new Issue("error", "too-costly", null, null,
+          `Operation aborted: server memory usage (${usedGB} GB) exceeds safe threshold (${pct}% of ${limitGB} GB limit) at ${place}`, "too-costly", 422).handleAsOO(422);
+      // the whole operation stops: as an OperationOutcome, not a validation's result, and a
+      // batch doesn't go on to its next entry
+      error.memoryAbort = true;
+      error.diagnostics = this.diagnostics();
+      throw error;
     }
 
     return false;
@@ -989,9 +1305,9 @@ class OperationContext {
 
   /**
    * Mark that the client for this operation disconnected before the response
-   * was sent. The operation is aborted at its next yield point - there is no
-   * one left to send the result to. (Wired up by the request middleware from
-   * the response 'close' event.)
+   * was sent. The operation is aborted at its next check (deadCheck,
+   * checkAndYield, waitFor) - there is no one left to send the result to.
+   * (Wired up by the request middleware from the response 'close' event.)
    */
   markClientGone() {
     this._clock.clientGone = true;
@@ -1024,14 +1340,7 @@ class OperationContext {
       const resumed = performance.now();
       this._clock.sliceStart = resumed;
       this._clock.lastYield = resumed;
-      if (this._clock.clientGone) {
-        this.log(`Operation abandoned @ ${place}: client disconnected`);
-        const error = new Issue("error", "too-costly", null, null,
-            `Operation abandoned at ${place}: the client disconnected before the response was ready`, "too-costly", 422);
-        error.abandoned = true;
-        error.diagnostics = this.diagnostics();
-        throw error;
-      }
+      this.abandonIfClientGone(place);
     }
   }
 
@@ -1057,15 +1366,25 @@ class OperationContext {
       this._clock.sliceStart = resumed;
       this._clock.lastYield = resumed;
     }
+    this.abandonIfClientGone(place);
+    return result;
+  }
+
+  /**
+   * Throw (422, abandoned) if the client has disconnected (markClientGone). The error is
+   * handled as an OperationOutcome (Issue.handleAsOO), so a validation doesn't report it as
+   * its result, and a batch stops at it rather than carry on with the next entry.
+   * @param {string} place - Location identifier for debugging
+   */
+  abandonIfClientGone(place) {
     if (this._clock.clientGone) {
       this.log(`Operation abandoned @ ${place}: client disconnected`);
       const error = new Issue("error", "too-costly", null, null,
-          `Operation abandoned at ${place}: the client disconnected before the response was ready`, "too-costly", 422);
+          `Operation abandoned at ${place}: the client disconnected before the response was ready`, "too-costly", 422).handleAsOO(422);
       error.abandoned = true;
       error.diagnostics = this.diagnostics();
       throw error;
     }
-    return result;
   }
 
   unSeeAll() {
@@ -1180,21 +1499,60 @@ class OperationContext {
    * @param {Object} provider - The provider returned from factory.build()
    */
   registerProvider(provider) {
-    if (provider && typeof provider.close === 'function') {
-      this._openProviders.push(provider);
+    // once: a provider can be shared within an operation (Provider.buildOnce)
+    if (provider && typeof provider.close === 'function' && !this._providers.open.includes(provider)) {
+      this._providers.open.push(provider);
+      if (this._providers.closed) {
+        this.closeProviders().catch(() => {});
+      }
     }
   }
 
   /**
-   * Close every provider registered during this operation. Safe to call
-   * multiple times — the list is cleared after the first call. Errors
+   * A provider built once for this operation (and the copies that share its providers) and
+   * reused after that: a value set with an include per code asks for the same code system
+   * once per include, and each build of a database-backed one is a connection held until
+   * the operation ends. The promise is kept, so calls that overlap share one build; a failed
+   * build is not kept, so the next call tries again. The caller registers the provider for
+   * closing (registerProvider), as for any other.
+   *
+   * @param {Object} factory - the factory that builds it (the same factory object is the same
+   *   code system and version, however it was looked up)
+   * @param {string} variant - what else makes a different provider (see Provider.buildOnce:
+   *   the supplements)
+   * @param {function(): Promise<Object>} build
+   * @returns {Promise<Object>}
+   */
+  buildProviderOnce(factory, variant, build) {
+    let built = this._providers.built.get(factory);
+    if (!built) {
+      built = new Map();
+      this._providers.built.set(factory, built);
+    }
+    let provider = built.get(variant);
+    if (!provider) {
+      provider = Promise.resolve(build());
+      provider.catch(() => {
+        if (built.get(variant) === provider) {
+          built.delete(variant);
+        }
+      });
+      built.set(variant, provider);
+    }
+    return provider;
+  }
+
+  /**
+   * Close every provider registered during this operation, including through
+   * a copy. Safe to call multiple times — the list is emptied each time. Errors
    * from individual close() calls are swallowed so one bad provider can't
    * prevent the others from releasing their resources.
    */
   async closeProviders() {
-    if (!this._openProviders || this._openProviders.length === 0) return;
-    const providers = this._openProviders;
-    this._openProviders = [];
+    this._providers.closed = true;
+    // nothing closed is handed out again
+    this._providers.built.clear();
+    const providers = this._providers.open.splice(0);
     for (const p of providers) {
       try {
         await p.close();
@@ -1225,7 +1583,10 @@ module.exports = {
   TimeTracker,
   ResourceCache,
   ExpansionCache,
+  ExpansionCacheBudget,
+  countExpansionCodes,
   isDebugging,
   debugLog,
-  formatDuration
-};
+  formatDuration,
+  PROCESS_START
+};