fhirsmith 0.18.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. package/ACCESSIBILITY.md +119 -0
  2. package/CHANGELOG.md +149 -0
  3. package/CONFIGURATION.md +305 -0
  4. package/LICENSE +29 -0
  5. package/README.md +29 -122
  6. package/cluster.js +1220 -0
  7. package/cluster.md +707 -0
  8. package/config-template.json +77 -8
  9. package/extension-tracker/extension-tracker-template.html +5 -8
  10. package/extension-tracker/extension-tracker.js +0 -2
  11. package/folder/folder.js +7 -6
  12. package/ips/analysis.js +560 -0
  13. package/ips/ips-template.html +5 -8
  14. package/ips/ips.js +132 -59
  15. package/ips/nationality.js +120 -0
  16. package/ips/pipeline.js +74 -21
  17. package/ips/presenter.js +1 -1
  18. package/ips/readme.md +132 -25
  19. package/ips/render.js +466 -77
  20. package/ips/store.js +79 -22
  21. package/ips/structure.js +247 -0
  22. package/ips/terminology.js +1 -1
  23. package/library/body-limit.js +22 -0
  24. package/library/cache-generation.js +15 -0
  25. package/library/cgroup-memory.js +133 -1
  26. package/library/cluster-counts.js +121 -0
  27. package/library/cluster-events.js +153 -0
  28. package/library/cluster-metrics.js +193 -0
  29. package/library/cluster-status.js +468 -0
  30. package/library/config-reader.js +96 -0
  31. package/library/daily-report.js +100 -2
  32. package/library/html-server.js +27 -1
  33. package/library/i18nsupport.js +23 -1
  34. package/library/keep-alive.js +18 -0
  35. package/library/languages.js +17 -3
  36. package/library/logger.js +31 -0
  37. package/library/memory-guard.js +428 -0
  38. package/library/native-memory.js +52 -0
  39. package/library/npm-audit.js +51 -25
  40. package/library/openapi-doc.js +7 -5
  41. package/library/package-manager.js +8 -0
  42. package/library/role.js +197 -0
  43. package/library/server-credentials.js +169 -0
  44. package/library/sqlite-utilities.js +59 -0
  45. package/library/ssrf-guard.js +100 -0
  46. package/library/static-response-stats.js +16 -0
  47. package/library/version-utilities.js +50 -0
  48. package/library/worker-count.js +192 -0
  49. package/nginx.md +30 -11
  50. package/npmprojector/indexer.js +18 -10
  51. package/package.json +11 -8
  52. package/packages/feed-check-page.js +117 -0
  53. package/packages/feed-check.js +279 -0
  54. package/packages/npm-check.js +57 -3
  55. package/packages/npm-signing.js +219 -0
  56. package/packages/openapi.yaml +207 -1
  57. package/packages/package-crawler.js +322 -41
  58. package/packages/packages-template.html +6 -8
  59. package/packages/packages.js +604 -57
  60. package/packages/readme.md +113 -1
  61. package/publisher/publisher-template.html +2 -5
  62. package/publisher/publisher.js +121 -110
  63. package/publisher/readme.md +75 -0
  64. package/registry/crawler.js +84 -108
  65. package/registry/documentation/configuration.md +34 -0
  66. package/registry/registry-template.html +5 -8
  67. package/registry/registry.js +5 -4
  68. package/root-bare-template.html +7 -4
  69. package/root-template.html +5 -8
  70. package/server.js +398 -198
  71. package/shl/shl.js +38 -37
  72. package/static/a11y/a11y-check.js +832 -0
  73. package/static/a11y/axe.min.js +12 -0
  74. package/static/assets/css/bootstrap-fhir.css +15 -15
  75. package/static/assets/images/flags/au.svg +18 -0
  76. package/static/assets/images/flags/unknown.svg +11 -0
  77. package/static/assets/images/flags/us.svg +63 -0
  78. package/static/assets/images/icon_audit.png +0 -0
  79. package/static/assets/images/icon_clinical.png +0 -0
  80. package/static/assets/images/icon_document.png +0 -0
  81. package/static/assets/images/icon_external.png +0 -0
  82. package/static/assets/images/icon_medication.png +0 -0
  83. package/static/assets/images/icon_organization.png +0 -0
  84. package/static/assets/images/icon_other.png +0 -0
  85. package/static/assets/images/icon_person.png +0 -0
  86. package/static/assets/images/icon_section.png +0 -0
  87. package/stats-db.js +45 -0
  88. package/stats.js +197 -10
  89. package/testing/testing-template.html +5 -8
  90. package/token/nginx_passport_setup.md +1 -1
  91. package/token/security_guide.md +1 -1
  92. package/token/token-template.html +2 -5
  93. package/token/token.js +40 -36
  94. package/translations/Messages.properties +19 -0
  95. package/tx/README.md +154 -4
  96. package/tx/closure/closure-store.js +126 -3
  97. package/tx/cm/cm-database.js +69 -57
  98. package/tx/cm/cm-package.js +29 -1
  99. package/tx/cs/cs-api.js +284 -19
  100. package/tx/cs/cs-api.md +41 -4
  101. package/tx/cs/cs-areacode.js +1 -1
  102. package/tx/cs/cs-country.js +17 -5
  103. package/tx/cs/cs-cpt.js +33 -16
  104. package/tx/cs/cs-cs.js +148 -14
  105. package/tx/cs/cs-currency.js +1 -1
  106. package/tx/cs/cs-icd11.js +21 -13
  107. package/tx/cs/cs-icf.js +22 -3
  108. package/tx/cs/cs-lang.js +2 -2
  109. package/tx/cs/cs-loinc.js +185 -66
  110. package/tx/cs/cs-mimetypes.js +12 -2
  111. package/tx/cs/cs-ndc.js +40 -25
  112. package/tx/cs/cs-omop.js +54 -29
  113. package/tx/cs/cs-rxnorm.js +300 -196
  114. package/tx/cs/cs-snomed.js +608 -64
  115. package/tx/cs/cs-ucum.js +14 -4
  116. package/tx/cs/cs-unii.js +18 -14
  117. package/tx/cs/cs-virtual.js +210 -0
  118. package/tx/ecosystem-resolver.js +413 -0
  119. package/tx/html/cluster-dash-metrics.liquid +26 -0
  120. package/tx/html/cluster-home-metrics.liquid +31 -0
  121. package/tx/html/cluster-metrics-script.liquid +251 -0
  122. package/tx/html/codesystem-operations.liquid +3 -3
  123. package/tx/html/ecl-panel.liquid +1 -1
  124. package/tx/html/external-metrics.liquid +17 -0
  125. package/tx/html/operations-form.liquid +17 -17
  126. package/tx/html/search-form.liquid +11 -11
  127. package/tx/html/status-line.liquid +58 -0
  128. package/tx/html/status-panel.liquid +140 -0
  129. package/tx/html/status-tasks.liquid +25 -0
  130. package/tx/html/tx-template.html +2 -5
  131. package/tx/importers/atc-to-fhir.js +17 -16
  132. package/tx/importers/export-icd11-codesystem.js +3 -2
  133. package/tx/importers/import-icd11.js +4 -3
  134. package/tx/importers/import-loinc.module.js +118 -56
  135. package/tx/importers/import-ndc.module.js +56 -44
  136. package/tx/importers/import-rxnorm.module.js +118 -79
  137. package/tx/importers/import-sct.module.js +57 -58
  138. package/tx/importers/import-unii.module.js +37 -36
  139. package/tx/importers/subset-loinc.module.js +49 -48
  140. package/tx/importers/subset-rxnorm.module.js +29 -28
  141. package/tx/importers/terminal.js +18 -0
  142. package/tx/importers/tx-import-base.js +13 -12
  143. package/tx/importers/tx-import-settings.js +5 -4
  144. package/tx/importers/tx-import.js +53 -52
  145. package/tx/library/canonical-resource.js +49 -0
  146. package/tx/library/codesystem.js +94 -5
  147. package/tx/library/designations.js +45 -2
  148. package/tx/library/extensions.js +13 -7
  149. package/tx/library/implicit-valuesets.js +66 -0
  150. package/tx/library/renderer.js +77 -21
  151. package/tx/library/sqlite-utilities.js +2 -0
  152. package/tx/library/ucum-parsers.js +11 -1
  153. package/tx/library/wait-for-cache.js +43 -0
  154. package/tx/library.js +321 -22
  155. package/tx/openapi-operations.js +6 -4
  156. package/tx/operation-context.js +460 -99
  157. package/tx/params.js +25 -0
  158. package/tx/problems.js +3 -5
  159. package/tx/provider.js +278 -42
  160. package/tx/resource-cache-sqlite.js +1050 -0
  161. package/tx/sct/ecl.js +47 -6
  162. package/tx/sct/editions.js +39 -0
  163. package/tx/sct/expressions.js +10 -1
  164. package/tx/sct/load-config.js +107 -0
  165. package/tx/sct/mmap.js +109 -0
  166. package/tx/sct/structures.js +166 -13
  167. package/tx/sct/text-index.js +5 -1
  168. package/tx/syndication/syndication-feed.js +2 -2
  169. package/tx/tx-html.js +44 -8
  170. package/tx/tx.fhir.org.yml +8 -8
  171. package/tx/tx.js +282 -80
  172. package/tx/usage-tracker.js +39 -11
  173. package/tx/vs/vs-api.js +19 -0
  174. package/tx/vs/vs-database.js +386 -62
  175. package/tx/vs/vs-package.js +27 -4
  176. package/tx/vs/vs-vsac.js +648 -100
  177. package/tx/workers/batch-validate.js +37 -10
  178. package/tx/workers/batch.js +6 -1
  179. package/tx/workers/cache-control.js +15 -4
  180. package/tx/workers/closure.js +156 -13
  181. package/tx/workers/compare.js +8 -5
  182. package/tx/workers/expand.js +1173 -289
  183. package/tx/workers/metadata.js +35 -0
  184. package/tx/workers/search.js +53 -1
  185. package/tx/workers/supplement-filters.js +145 -0
  186. package/tx/workers/translate.js +169 -4
  187. package/tx/workers/validate.js +223 -119
  188. package/tx/workers/worker.js +123 -31
  189. package/tx/xversion/xv-valueset.js +7 -4
  190. package/utilities/submit-test-report.js +3 -1
  191. package/utilities/tx-stress-test.js +482 -0
  192. package/vcl/readme.md +37 -0
  193. package/vcl/vcl-parser.js +15 -0
  194. package/xig/readme.md +52 -0
  195. package/xig/xig-template.html +5 -8
  196. package/xig/xig.js +238 -422
  197. package/changelog-template.md +0 -22
  198. package/registry/test-runner.js +0 -237
  199. package/release.sh +0 -80
  200. package/test-scripts/repro-re2-wasm-leak.js +0 -34
  201. package/tsconfig.eslint.json +0 -9
  202. package/tx/html/dash-metrics.liquid +0 -147
  203. package/tx/html/home-metrics.liquid +0 -225
@@ -19,12 +19,17 @@ const crypto = require('crypto');
19
19
  const ValueSet = require("../library/valueset");
20
20
  const {VersionUtilities} = require("../../library/version-utilities");
21
21
  const {debugLog} = require("../operation-context");
22
-
23
- // Expansion limits (from Pascal constants)
24
- const EXTERNAL_DEFAULT_LIMIT = 1000;
25
- const EXTERNAL_TEST_DEFAULT_LIMIT = 3000;
26
- const INTERNAL_DEFAULT_LIMIT = 10000;
27
- const EXPANSION_DEAD_TIME_SECS = 30;
22
+ const { splitFilters, supplementFilter } = require('./supplement-filters');
23
+
24
+ // Expansion limits, when the configuration doesn't set them (modules.tx.externalLimit and
25
+ // internalLimit; see TXModule.expansionLimits): the most codes an $expand returns, and the most
26
+ // an expansion builds (an imported value set's, for one)
27
+ const EXTERNAL_DEFAULT_LIMIT = 10000;
28
+ const INTERNAL_DEFAULT_LIMIT = 100000;
29
+ // How far a paged expansion (count and offset) may walk to reach its page, as a multiple of the
30
+ // internal limit when not configured (modules.tx.walkLimit). Only the page is built: the codes
31
+ // before it are kept as keys, so walking costs time (bounded by the operation limit), not memory
32
+ const WALK_LIMIT_FACTOR = 10;
28
33
  const CACHE_WHEN_DEBUGGING = false;
29
34
 
30
35
  /**
@@ -186,6 +191,13 @@ class EmptyFilterContext {
186
191
  }
187
192
  }
188
193
 
194
+ /**
195
+ * The codes of a value set expanded for an import, added with their keys only (see
196
+ * ValueSetExpander.deferDetails): for each, how to read its details when the importing expansion
197
+ * returns it - {expander, cs, expansion, read}
198
+ */
199
+ const DEFERRED_DETAILS = new WeakMap();
200
+
189
201
  class ValueSetCounter {
190
202
  constructor() {
191
203
  this.count = 0;
@@ -201,6 +213,7 @@ class ValueSetExpander {
201
213
  params;
202
214
  doingVersion = true;
203
215
  excludedSystems = new Set();
216
+ excludedProviders = []; // {system, cs} - see checkForExclusionVersionSpecialCase
204
217
  excluded = new Set();
205
218
  hasExclusions = false;
206
219
  requiredSupplements = new Set();
@@ -209,12 +222,40 @@ class ValueSetExpander {
209
222
  internalLimit = INTERNAL_DEFAULT_LIMIT;
210
223
  externalLimit = EXTERNAL_DEFAULT_LIMIT;
211
224
  noDetails = false;
225
+ // count=0: the codes counted (see #countListedCodes) but not built, by keyS
226
+ countedKeys = new Set();
227
+ composeOnlyListsCodes = false;
228
+ // A paged expansion (count > 0) walks to its page keeping only the keys of the codes before
229
+ // it, and counts those after it (see #beforePage): paging is whether this one does, walked how many codes
230
+ // it has passed, walkedKeys their keys (for deduplication), walkLimit how far it may walk
231
+ paging = false;
232
+ walked = 0;
233
+ // a code system's cap (expandLimitation) kept codes off the page asked for: {system, limit}, for
234
+ // the expansion-issue that explains it (see #noteCapCut)
235
+ capCut = null;
236
+ walkedKeys = new Set();
237
+ // and after the page, how many more codes it has counted for the total (see #finishIfPageFilled)
238
+ counted = 0;
239
+ // a paged expansion of one selection whose size the provider vouches for, as count=0 trusts it
240
+ // (one include of a whole code system, or of one filter set, and nothing else): that size, which
241
+ // is the total, so the walk stops once the page is full (see #finishIfPageFilled). null: the walk counts
242
+ providerSize = null;
243
+ walkLimit = INTERNAL_DEFAULT_LIMIT * WALK_LIMIT_FACTOR;
244
+ // a provider applied offset and count to the compose itself (processSelection): every code it
245
+ // returns is on the page
246
+ providerPaged = false;
247
+ // An imported value set (include.valueSet, exclude.valueSet) whose expansion the importing
248
+ // expansion uses for its codes, and from which it returns only some codes, or none (its page):
249
+ // each code is added with its key, and what its details are read from is kept with it, for the
250
+ // importing expansion to build the ones it returns (#buildDeferred). Same codes, same order
251
+ deferDetails = false;
212
252
 
213
253
  constructor(worker, params) {
214
254
  this.worker = worker;
215
255
  this.params = params;
216
256
  this.internalLimit = worker.internalLimit;
217
257
  this.externalLimit = worker.externalLimit;
258
+ this.walkLimit = worker.walkLimit || (this.internalLimit || INTERNAL_DEFAULT_LIMIT) * WALK_LIMIT_FACTOR;
218
259
 
219
260
  this.csCounter = new Map();
220
261
  }
@@ -271,11 +312,21 @@ class ValueSetExpander {
271
312
  return true;
272
313
  }
273
314
  for (const s of this.params.designations) {
274
- const [l, r] = s.split('|');
275
- if (cd.use != null && cd.use.system === l && cd.use.code === r) {
315
+ // a token: system|code, or a bare code (or |code), which matches a language or a use
316
+ // code from any system. designation=nl-NL is the same as urn:ietf:bcp:47|nl-NL
317
+ const bar = s.indexOf('|');
318
+ const system = bar < 0 ? '' : s.substring(0, bar);
319
+ const code = bar < 0 ? s : s.substring(bar + 1);
320
+ if (!system) {
321
+ if ((cd.language != null && cd.language.code === code) || (cd.use != null && cd.use.code === code)) {
322
+ return true;
323
+ }
324
+ continue;
325
+ }
326
+ if (cd.use != null && cd.use.system === system && cd.use.code === code) {
276
327
  return true;
277
328
  }
278
- if (cd.language != null && l === 'urn:ietf:bcp:47' && r === cd.language.code) {
329
+ if (cd.language != null && system === 'urn:ietf:bcp:47' && code === cd.language.code) {
279
330
  return true;
280
331
  }
281
332
  }
@@ -300,7 +351,33 @@ class ValueSetExpander {
300
351
  }
301
352
  }
302
353
 
303
- includeCode(cs, parent, system, version, code, isAbstract, isInactive, deprecated, status, displays, definition, itemWeight, expansion, imports, csExtList, vsExtList, csProps, expProps, excludeInactive, srcURL) {
354
+ /**
355
+ * The response limit (limitCount: the request's, or for an imported value set, the internal
356
+ * limit), checked before a code is added to the expansion, by every path that adds one
357
+ * (includeCode, importValueSetItem). At the limit: when the page asked for (count and offset)
358
+ * is full, the expansion is done, without a total; otherwise the expansion is too costly
359
+ * (VALUESET_TOO_COSTLY). Exclusions are no exception: they are all processed before any
360
+ * include, so the codes already in the list stay in it. Only counting (count=0), nothing is
361
+ * returned, so the limit is how far a walk may go (walkLimit)
362
+ * @param {string} srcURL - the value set, for the message
363
+ */
364
+ #checkLimit(srcURL) {
365
+ const limit = this.count === 0 ? this.walkLimit : this.limitCount;
366
+ if (limit > 0 && this.fullList.length >= limit) {
367
+ if (this.count > -1 && this.offset > -1 && this.count + this.offset > 0 && this.walked + this.fullList.length >= this.count + this.offset) {
368
+ this.noTotal();
369
+ throw new Issue('information', 'informational', null, null, null, null).setFinished();
370
+ } else {
371
+ throw new Issue("error", "too-costly", null, 'VALUESET_TOO_COSTLY', this.tooCostlyMessage(srcURL, '>' + limit), null, 422).withDiagnostics(this.worker.opContext.diagnostics());
372
+ }
373
+ }
374
+ }
375
+
376
+ /**
377
+ * @param {function(): Promise<Object>} [deferredRead] - the code's details are not added now, but
378
+ * read with this when an importing expansion returns the code (see deferDetails, #readDetails)
379
+ */
380
+ async includeCode(cs, parent, system, version, code, isAbstract, isInactive, deprecated, status, displays, definition, itemWeight, expansion, imports, csExtList, vsExtList, csProps, expProps, excludeInactive, srcURL, deferredRead = null) {
304
381
  let result = null;
305
382
  this.worker.deadCheck('processCode');
306
383
 
@@ -310,39 +387,15 @@ class ValueSetExpander {
310
387
  if (isInactive && excludeInactive) {
311
388
  return null;
312
389
  }
313
- if (this.isExcluded(system, version, code)) {
390
+ if (await this.isExcluded(system, version, code)) {
314
391
  return null;
315
392
  }
316
393
 
317
- // expandLimitation() caps how many codes a single code system may contribute
318
- // (e.g. CPT limits enumeration to 1000 codes by agreement with the AMA).
319
- // NB: expandLimitation and system are methods - reading them as properties
320
- // (cs.expandLimitation > 0) compares the function object and is always false,
321
- // which silently disabled this cap.
322
- const expandLimit = cs != null ? cs.expandLimitation() : 0;
323
- if (expandLimit > 0) {
324
- let cnt = this.csCounter.get(cs.system());
325
- if (cnt == null) {
326
- cnt = new ValueSetCounter();
327
- this.csCounter.set(cs.system(), cnt);
328
- }
329
- cnt.increment();
330
- if (cnt.count > expandLimit) {
331
- return null;
332
- }
394
+ if (this.#overSystemCap(cs)) {
395
+ return null;
333
396
  }
334
397
 
335
- if (this.limitCount > 0 && this.fullList.length >= this.limitCount && !this.hasExclusions) {
336
- if (this.count > -1 && this.offset > -1 && this.count + this.offset > 0 && this.fullList.length >= this.count + this.offset) {
337
- this.noTotal();
338
- throw new Issue('information', 'informational', null, null, null, null).setFinished();
339
- } else {
340
- if (!srcURL) {
341
- srcURL = '??';
342
- }
343
- throw new Issue("error", "too-costly", null, 'VALUESET_TOO_COSTLY', this.worker.i18n.translate('VALUESET_TOO_COSTLY', this.params.httpLanguages, [srcURL, '>' + this.limitCount]), null, 422).withDiagnostics(this.worker.opContext.diagnostics());
344
- }
345
- }
398
+ this.#checkLimit(srcURL);
346
399
 
347
400
  if (expansion && !this.noDetails) {
348
401
  const s = this.canonical(system, version);
@@ -357,7 +410,10 @@ class ValueSetExpander {
357
410
 
358
411
  const s = this.keyS(system, version, code);
359
412
 
360
- if (!this.map.has(s)) {
413
+ if (!this.map.has(s) && !this.walkedKeys.has(s)) {
414
+ // a code outside the page, on an expansion that doesn't walk to it: only counted and kept
415
+ // for deduplication, never returned (see #outsidePage)
416
+ const outsidePage = this.#outsidePage();
361
417
  const n = {};
362
418
  n.system = system;
363
419
  n.code = code;
@@ -370,162 +426,351 @@ class ValueSetExpander {
370
426
  if (isInactive) {
371
427
  n.inactive = true;
372
428
  }
373
- if (!this.noDetails) {
374
- if (status && status.toLowerCase() !== 'active') {
375
- this.defineProperty(expansion, n, 'http://hl7.org/fhir/concept-properties#status', 'status', "valueCode", status);
376
- } else if (deprecated) {
377
- this.defineProperty(expansion, n, 'http://hl7.org/fhir/concept-properties#status', 'status', "valueCode", 'deprecated');
429
+ if (deferredRead) {
430
+ DEFERRED_DETAILS.set(n, { expander: this, cs, expansion, read: deferredRead });
431
+ } else if (!this.noDetails && !outsidePage) {
432
+ this.#addDetails(n, cs, code, status, deprecated, displays, definition, expansion, csExtList, vsExtList, csProps);
433
+ }
434
+ this.fullList.push(n);
435
+ this.map.set(s, n);
436
+ if (parent != null && !this.noDetails) {
437
+ if (!parent.contains) {
438
+ parent.contains = [];
378
439
  }
440
+ parent.contains.push(n);
441
+ } else {
442
+ this.rootList.push(n);
443
+ }
444
+ result = n;
445
+ } else {
446
+ this.canBeHierarchy = false;
447
+ }
448
+ return result;
449
+ }
379
450
 
380
- if (Extensions.has(csExtList, 'http://hl7.org/fhir/StructureDefinition/codesystem-label')) {
381
- this.defineProperty(expansion, n, 'http://hl7.org/fhir/concept-properties#label', 'label', "valueString", Extensions.readString(csExtList, 'http://hl7.org/fhir/StructureDefinition/codesystem-label'));
382
- }
383
- if (Extensions.has(vsExtList, 'http://hl7.org/fhir/StructureDefinition/valueset-label')) {
384
- this.defineProperty(expansion, n, 'http://hl7.org/fhir/concept-properties#label', 'label', "valueString", Extensions.readString(vsExtList, 'http://hl7.org/fhir/StructureDefinition/valueset-label'));
385
- }
451
+ /**
452
+ * The details of a code in the expansion: status and other properties, label, order and weight,
453
+ * the extensions carried over, the display and designations (see includeCode)
454
+ */
455
+ #addDetails(n, cs, code, status, deprecated, displays, definition, expansion, csExtList, vsExtList, csProps) {
456
+ if (status && status.toLowerCase() !== 'active') {
457
+ this.defineProperty(expansion, n, 'http://hl7.org/fhir/concept-properties#status', 'status', "valueCode", status);
458
+ } else if (deprecated) {
459
+ this.defineProperty(expansion, n, 'http://hl7.org/fhir/concept-properties#status', 'status', "valueCode", 'deprecated');
460
+ }
386
461
 
387
- if (Extensions.has(csExtList, 'http://hl7.org/fhir/StructureDefinition/codesystem-conceptOrder')) {
388
- this.defineProperty(expansion, n, 'http://hl7.org/fhir/concept-properties#order', 'order', "valueDecimal", Extensions.readNumber(csExtList, 'http://hl7.org/fhir/StructureDefinition/codesystem-conceptOrder', undefined));
389
- }
390
- if (Extensions.has(vsExtList, 'http://hl7.org/fhir/StructureDefinition/valueset-conceptOrder')) {
391
- this.defineProperty(expansion, n, 'http://hl7.org/fhir/concept-properties#order', 'order', "valueDecimal", Extensions.readNumber(vsExtList, 'http://hl7.org/fhir/StructureDefinition/valueset-conceptOrder', undefined));
392
- }
462
+ if (Extensions.has(csExtList, 'http://hl7.org/fhir/StructureDefinition/codesystem-label')) {
463
+ this.defineProperty(expansion, n, 'http://hl7.org/fhir/concept-properties#label', 'label', "valueString", Extensions.readString(csExtList, 'http://hl7.org/fhir/StructureDefinition/codesystem-label'));
464
+ }
465
+ if (Extensions.has(vsExtList, 'http://hl7.org/fhir/StructureDefinition/valueset-label')) {
466
+ this.defineProperty(expansion, n, 'http://hl7.org/fhir/concept-properties#label', 'label', "valueString", Extensions.readString(vsExtList, 'http://hl7.org/fhir/StructureDefinition/valueset-label'));
467
+ }
393
468
 
394
- if (Extensions.has(csExtList, 'http://hl7.org/fhir/StructureDefinition/itemWeight')) {
395
- this.defineProperty(expansion, n, 'http://hl7.org/fhir/concept-properties#itemWeight', 'weight', "valueDecimal", Extensions.readNumber(csExtList, 'http://hl7.org/fhir/StructureDefinition/itemWeight', undefined));
396
- }
397
- if (Extensions.has(vsExtList, 'http://hl7.org/fhir/StructureDefinition/itemWeight')) {
398
- this.defineProperty(expansion, n, 'http://hl7.org/fhir/concept-properties#itemWeight', 'weight', "valueDecimal", Extensions.readNumber(vsExtList, 'http://hl7.org/fhir/StructureDefinition/itemWeight', undefined));
399
- }
469
+ if (Extensions.has(csExtList, 'http://hl7.org/fhir/StructureDefinition/codesystem-conceptOrder')) {
470
+ this.defineProperty(expansion, n, 'http://hl7.org/fhir/concept-properties#order', 'order', "valueDecimal", Extensions.readNumber(csExtList, 'http://hl7.org/fhir/StructureDefinition/codesystem-conceptOrder', undefined));
471
+ }
472
+ if (Extensions.has(vsExtList, 'http://hl7.org/fhir/StructureDefinition/valueset-conceptOrder')) {
473
+ this.defineProperty(expansion, n, 'http://hl7.org/fhir/concept-properties#order', 'order', "valueDecimal", Extensions.readNumber(vsExtList, 'http://hl7.org/fhir/StructureDefinition/valueset-conceptOrder', undefined));
474
+ }
400
475
 
401
- if (csExtList != null) {
402
- for (const ext of csExtList) {
403
- if (['http://hl7.org/fhir/StructureDefinition/coding-sctdescid', 'http://hl7.org/fhir/StructureDefinition/rendering-style',
404
- 'http://hl7.org/fhir/StructureDefinition/rendering-xhtml', 'http://hl7.org/fhir/StructureDefinition/codesystem-alternate'].includes(ext.url)) {
405
- if (!n.extension) {
406
- n.extension = []
407
- }
408
- n.extension.push(ext);
409
- }
410
- if (['http://hl7.org/fhir/StructureDefinition/structuredefinition-standards-status'].includes(ext.url)) {
411
- this.defineProperty(expansion, n, 'http://hl7.org/fhir/concept-properties#status', 'status', "valueCode", getValuePrimitive(ext));
412
- }
476
+ if (Extensions.has(csExtList, 'http://hl7.org/fhir/StructureDefinition/itemWeight')) {
477
+ this.defineProperty(expansion, n, 'http://hl7.org/fhir/concept-properties#itemWeight', 'weight', "valueDecimal", Extensions.readNumber(csExtList, 'http://hl7.org/fhir/StructureDefinition/itemWeight', undefined));
478
+ }
479
+ if (Extensions.has(vsExtList, 'http://hl7.org/fhir/StructureDefinition/itemWeight')) {
480
+ this.defineProperty(expansion, n, 'http://hl7.org/fhir/concept-properties#itemWeight', 'weight', "valueDecimal", Extensions.readNumber(vsExtList, 'http://hl7.org/fhir/StructureDefinition/itemWeight', undefined));
481
+ }
482
+
483
+ if (csExtList != null) {
484
+ for (const ext of csExtList) {
485
+ if (['http://hl7.org/fhir/StructureDefinition/coding-sctdescid', 'http://hl7.org/fhir/StructureDefinition/rendering-style',
486
+ 'http://hl7.org/fhir/StructureDefinition/rendering-xhtml', 'http://hl7.org/fhir/StructureDefinition/codesystem-alternate'].includes(ext.url)) {
487
+ if (!n.extension) {
488
+ n.extension = []
413
489
  }
490
+ n.extension.push(ext);
414
491
  }
492
+ if (['http://hl7.org/fhir/StructureDefinition/structuredefinition-standards-status'].includes(ext.url)) {
493
+ this.defineProperty(expansion, n, 'http://hl7.org/fhir/concept-properties#status', 'status', "valueCode", getValuePrimitive(ext));
494
+ }
495
+ }
496
+ }
415
497
 
416
- if (vsExtList != null) {
417
- for (const ext of vsExtList || []) {
418
- if (['http://hl7.org/fhir/StructureDefinition/valueset-supplement', 'http://hl7.org/fhir/StructureDefinition/valueset-deprecated',
419
- 'http://hl7.org/fhir/StructureDefinition/structuredefinition-standards-status',
420
- 'http://hl7.org/fhir/StructureDefinition/valueset-concept-definition', 'http://hl7.org/fhir/StructureDefinition/coding-sctdescid',
421
- 'http://hl7.org/fhir/StructureDefinition/rendering-style', 'http://hl7.org/fhir/StructureDefinition/rendering-xhtml'].includes(ext.url)) {
422
- if (!n.extension) {
423
- n.extension = []
424
- }
425
- n.extension.push(ext);
426
- }
498
+ if (vsExtList != null) {
499
+ for (const ext of vsExtList || []) {
500
+ if (['http://hl7.org/fhir/StructureDefinition/valueset-supplement', 'http://hl7.org/fhir/StructureDefinition/valueset-deprecated',
501
+ 'http://hl7.org/fhir/StructureDefinition/structuredefinition-standards-status',
502
+ 'http://hl7.org/fhir/StructureDefinition/valueset-concept-definition', 'http://hl7.org/fhir/StructureDefinition/coding-sctdescid',
503
+ 'http://hl7.org/fhir/StructureDefinition/rendering-style', 'http://hl7.org/fhir/StructureDefinition/rendering-xhtml'].includes(ext.url)) {
504
+ if (!n.extension) {
505
+ n.extension = []
427
506
  }
507
+ n.extension.push(ext);
428
508
  }
509
+ }
510
+ }
429
511
 
430
- // display and designations
431
- const pref = displays.preferredDesignation(this.params.workingLanguages(), this.reportedSupplements);
432
- if (pref && pref.value) {
433
- n.display = pref.value;
434
- }
512
+ // display and designations
513
+ const pref = displays.preferredDesignation(this.params.workingLanguages(), this.reportedSupplements);
514
+ if (pref && pref.value) {
515
+ n.display = pref.value;
516
+ }
435
517
 
436
- if (this.params.includeDesignations) {
437
- for (const t of displays.designations) {
438
- if (t !== pref && this.useDesignation(t) && t.value != null && !this.redundantDisplay(n, t.language, t.use, t.value)) {
439
- if (!n.designation) {
440
- n.designation = [];
441
- }
442
- if (t.supplement && t.supplement.vurl) {
443
- this.reportedSupplements.add(t.supplement.vurl);
444
- }
445
- n.designation.push(t.asObject());
446
- }
518
+ if (this.params.includeDesignations) {
519
+ for (const t of displays.designations) {
520
+ if (t !== pref && this.useDesignation(t) && t.value != null && !this.redundantDisplay(n, t.language, t.use, t.value)) {
521
+ if (!n.designation) {
522
+ n.designation = [];
523
+ }
524
+ if (t.supplement && t.supplement.vurl) {
525
+ this.reportedSupplements.add(t.supplement.vurl);
447
526
  }
527
+ n.designation.push(t.asObject());
448
528
  }
529
+ }
530
+ }
531
+
532
+ // '*' asks for every property the code system defines for the concept (R6's
533
+ // ValueSet.compose.property allows it, and a request can name it too). It is handled
534
+ // here rather than by expanding it into a list of names, because which properties
535
+ // exist is a property of the concept, not of the request
536
+ if (this.params.properties.includes('*') && csProps != null && cs != null) {
537
+ for (const cp of csProps) {
538
+ const vn = getValueName(cp);
539
+ this.defineProperty(expansion, n, this.getPropUrl(cs, cp.code, cp), cp.code, vn, cp[vn], true);
540
+ }
541
+ }
449
542
 
450
- // '*' asks for every property the code system defines for the concept (R6's
451
- // ValueSet.compose.property allows it, and a request can name it too). It is handled
452
- // here rather than by expanding it into a list of names, because which properties
453
- // exist is a property of the concept, not of the request
454
- if (this.params.properties.includes('*') && csProps != null && cs != null) {
455
- for (const cp of csProps) {
543
+ // properties the code system always reports (ICD-11's classificationCode), unless
544
+ // the request asked for them anyway and the loops here add them
545
+ if (csProps != null && cs != null && !this.params.properties.includes('*')) {
546
+ for (const pn of cs.mandatoryExpansionProperties()) {
547
+ if (!this.params.properties.includes(pn)) {
548
+ for (const cp of csProps.filter(p => p.code === pn)) {
456
549
  const vn = getValueName(cp);
457
- this.defineProperty(expansion, n, this.getPropUrl(cs, cp.code, cp), cp.code, vn, cp[vn], true);
550
+ this.defineProperty(expansion, n, this.getPropUrl(cs, pn, cp), pn, vn, cp[vn], true);
458
551
  }
459
552
  }
553
+ }
554
+ }
460
555
 
461
- // properties the code system always reports (ICD-11's classificationCode), unless
462
- // the request asked for them anyway and the loops here add them
463
- if (csProps != null && cs != null && !this.params.properties.includes('*')) {
464
- for (const pn of cs.mandatoryExpansionProperties()) {
465
- if (!this.params.properties.includes(pn)) {
466
- for (const cp of csProps.filter(p => p.code === pn)) {
467
- const vn = getValueName(cp);
468
- this.defineProperty(expansion, n, this.getPropUrl(cs, pn, cp), pn, vn, cp[vn], true);
469
- }
470
- }
471
- }
556
+ for (const pn of this.params.properties) {
557
+ if (pn === '*') {
558
+ continue; // done above
559
+ } else if (pn === 'definition') {
560
+ if (definition) {
561
+ this.defineProperty(expansion, n, 'http://hl7.org/fhir/concept-properties#definition', pn, "valueString", definition);
472
562
  }
473
-
474
- for (const pn of this.params.properties) {
475
- if (pn === '*') {
476
- continue; // done above
477
- } else if (pn === 'definition') {
478
- if (definition) {
479
- this.defineProperty(expansion, n, 'http://hl7.org/fhir/concept-properties#definition', pn, "valueString", definition);
480
- }
481
- } else if (pn === 'usage-count') {
482
- let counter = cs.usages().get(code);
483
- this.defineProperty(expansion, n, 'http://fhir.org/FHIRsmith/CodeSystem/concept-properties#usage-count', pn, "valueInteger", counter ? counter.count : 0);
484
- } else if (csProps != null && cs != null) {
485
- for (const cp of csProps) {
486
- if (cp.code === pn) {
487
- let vn = getValueName(cp);
488
- let v = cp[vn];
489
- // a concept may carry the same property more than once (the
490
- // "dup" property in the test code system has two values)
491
- this.defineProperty(expansion, n, this.getPropUrl(cs, pn, cp), pn, vn, v, true);
492
- }
493
- }
563
+ } else if (pn === 'usage-count') {
564
+ let counter = cs.usages().get(code);
565
+ this.defineProperty(expansion, n, 'http://fhir.org/FHIRsmith/CodeSystem/concept-properties#usage-count', pn, "valueInteger", counter ? counter.count : 0);
566
+ } else if (csProps != null && cs != null) {
567
+ for (const cp of csProps) {
568
+ if (cp.code === pn) {
569
+ let vn = getValueName(cp);
570
+ let v = cp[vn];
571
+ // a concept may carry the same property more than once (the
572
+ // "dup" property in the test code system has two values)
573
+ this.defineProperty(expansion, n, this.getPropUrl(cs, pn, cp), pn, vn, v, true);
494
574
  }
495
575
  }
496
576
  }
497
- this.fullList.push(n);
498
- this.map.set(s, n);
499
- if (parent != null && !this.noDetails) {
500
- if (!parent.contains) {
501
- parent.contains = [];
502
- }
503
- parent.contains.push(n);
504
- } else {
505
- this.rootList.push(n);
577
+ }
578
+ }
579
+
580
+ /**
581
+ * expandLimitation() caps how many codes a single code system may contribute (e.g. CPT limits
582
+ * enumeration to 1000 codes by agreement with the AMA): count this code against its system's
583
+ * cap, and say whether it is over. (NB: expandLimitation and system are methods - read as
584
+ * properties, cs.expandLimitation > 0 compares the function object and is always false, which
585
+ * silently disabled this cap.)
586
+ */
587
+ #overSystemCap(cs) {
588
+ const expandLimit = cs != null ? cs.expandLimitation() : 0;
589
+ if (expandLimit > 0) {
590
+ let cnt = this.csCounter.get(cs.system());
591
+ if (cnt == null) {
592
+ cnt = new ValueSetCounter();
593
+ this.csCounter.set(cs.system(), cnt);
594
+ }
595
+ cnt.increment();
596
+ if (cnt.count > expandLimit) {
597
+ this.#noteCapCut(cs);
598
+ return true;
506
599
  }
507
- result = n;
508
- } else {
509
- this.canBeHierarchy = false;
510
600
  }
511
- return result;
601
+ return false;
512
602
  }
513
603
 
514
- excludeCode(cs, system, version, code, expansion, imports, srcURL) {
515
- this.worker.deadCheck('excludeCode');
516
- if (!this.passesImports(imports, system, code, 0)) {
517
- return;
604
+ /**
605
+ * Whether the code system has already contributed all the codes its cap (expandLimitation)
606
+ * allows: then nothing more of it can be in the expansion, so there's no need to walk it further
607
+ */
608
+ #capReached(cs) {
609
+ const expandLimit = cs != null ? cs.expandLimitation() : 0;
610
+ if (!(expandLimit > 0)) {
611
+ return false;
518
612
  }
613
+ const cnt = this.csCounter.get(cs.system());
614
+ return cnt != null && cnt.count >= expandLimit;
615
+ }
519
616
 
520
- if (this.limitCount > 0 && this.fullList.length >= this.limitCount && !this.hasExclusions) {
521
- if (this.count > -1 && this.offset > -1 && this.count + this.offset > 0 && this.fullList.length >= this.count + this.offset) {
522
- throw new Issue('information', 'informational', null, null, null, null).setFinished();
617
+ /**
618
+ * A code system's cap kept a code out: if the page asked for wasn't yet full (or there is no
619
+ * page - all the codes were asked for), the response is missing codes it would otherwise have,
620
+ * and gets an expansion-issue saying why. A code kept out after the page is full only changes
621
+ * the total, which is reported anyway (see the whole-system include in includeCodes)
622
+ */
623
+ #noteCapCut(cs) {
624
+ if (this.capCut == null && !this.pageFilled()) {
625
+ this.capCut = { system: cs.system(), limit: cs.expandLimitation() };
626
+ }
627
+ }
628
+
629
+ /**
630
+ * Tell the user something about how the expansion was made that they should know (e.g. that
631
+ * codes are missing, and why): an expansion-issue extension on the expansion, referring to a
632
+ * contained OperationOutcome with the issue
633
+ * @param {Object} vs the ValueSet being returned
634
+ * @param {Object} exp its expansion
635
+ * @param {Issue} issue the issue
636
+ */
637
+ addExpansionIssue(vs, exp, issue) {
638
+ if (!vs.contained) {
639
+ vs.contained = [];
640
+ }
641
+ let n = 1;
642
+ while (vs.contained.some(r => r.id === 'oo' + n)) {
643
+ n++;
644
+ }
645
+ const id = 'oo' + n;
646
+ vs.contained.push({ resourceType: 'OperationOutcome', id, issue: [issue.asIssue()] });
647
+ if (!exp.extension) {
648
+ exp.extension = [];
649
+ }
650
+ exp.extension.push({ url: 'http://hl7.org/fhir/StructureDefinition/expansion-issue', valueReference: { reference: '#' + id } });
651
+ }
652
+
653
+ /**
654
+ * Before the page of a paged expansion (count and offset, see handleCompose), a code that would
655
+ * be included is only counted: its key is kept, for deduplication, and nothing is read or built.
656
+ * The tests are includeCode's, in the same order, so a code counts here exactly when it would be
657
+ * included there. Call it first, before reading the concept's details.
658
+ * @returns {Promise<boolean|null>} null: not before the page (or not paging) - include the code
659
+ * as usual; true: counted, go on to the next; false: it would not be included, go on to the next
660
+ */
661
+ async #beforePage(cs, system, version, code, isInactive, excludeInactive, imports) {
662
+ if (!this.paging || (this.walked >= this.offset && !this.pageFilled())) {
663
+ return null;
664
+ }
665
+ if (!this.passesImports(imports, system, code, 0) || (isInactive && excludeInactive)
666
+ || await this.isExcluded(system, version, code) || this.#overSystemCap(cs)) {
667
+ return false;
668
+ }
669
+ return this.#walk(this.keyS(system, version, code));
670
+ }
671
+
672
+ /**
673
+ * Count a code before the page, or after it (for the total): false if it was already counted
674
+ * or included
675
+ */
676
+ #walk(key) {
677
+ if (this.walkedKeys.has(key) || this.map.has(key)) {
678
+ this.canBeHierarchy = false;
679
+ return false;
680
+ }
681
+ this.walkedKeys.add(key);
682
+ if (this.pageFilled()) {
683
+ this.counted++;
684
+ this.#finishIfPageFilled();
685
+ return true;
686
+ }
687
+ this.walked++;
688
+ if (this.walked + this.count > this.walkLimit) {
689
+ const label = this.vsLabel();
690
+ throw new Issue('error', 'too-costly', null, 'VALUESET_TOO_COSTLY_PAGE', label
691
+ ? this.worker.i18n.translate('VALUESET_TOO_COSTLY_PAGE', this.params.httpLanguages, [label, this.offset, this.count, this.walkLimit])
692
+ : this.worker.i18n.translate('VALUESET_TOO_COSTLY_PAGE_REQUEST', this.params.httpLanguages, [this.offset, this.count, this.walkLimit]),
693
+ null, 422).withDiagnostics(this.worker.opContext.diagnostics());
694
+ }
695
+ return true;
696
+ }
697
+
698
+ /**
699
+ * Paging: once the page is full, the rest of the walk only counts codes, for the total - and the
700
+ * total is only reported for an expansion within the response limit, as on an unpaged one (where
701
+ * includeCode stops at the limit). So once the codes seen reach the limit, stop: no total. And when
702
+ * the provider has said how big the selection is (providerSize), there is nothing to count: stop
703
+ * as soon as the page is full
704
+ */
705
+ #finishIfPageFilled() {
706
+ if (this.paging && this.pageFilled() && this.providerSize != null) {
707
+ // the provider has said how many codes the walk would count: report that, as the walk would
708
+ // have (only within the limit), and stop
709
+ if (this.providerSize < this.limitCount) {
710
+ this.total = this.providerSize;
711
+ this.totalStatus = 'set';
523
712
  } else {
524
- if (!srcURL) {
525
- srcURL = '??';
526
- }
527
- throw new Issue("error", "too-costly", null, 'VALUESET_TOO_COSTLY', this.worker.i18n.translate('VALUESET_TOO_COSTLY', this.params.httpLanguages, [srcURL, '>' + this.limitCount]), null, 422).withDiagnostics(this.worker.opContext.diagnostics());
713
+ this.noTotal();
528
714
  }
715
+ throw new Issue('information', 'informational', null, null, null, null).setFinished();
716
+ }
717
+ if (this.paging && this.pageFilled() && this.walked + this.fullList.length + this.counted >= this.limitCount) {
718
+ this.noTotal();
719
+ throw new Issue('information', 'informational', null, null, null, null).setFinished();
720
+ }
721
+ }
722
+
723
+ /**
724
+ * Paging: whether the next n codes, if all are included, fall inside the page - so each will be
725
+ * built, and none only counted. (Fewer may be included: filters, excludes and repeats only make
726
+ * the list shorter, so it still fits)
727
+ * @param {number} n
728
+ * @returns {boolean}
729
+ */
730
+ #pageHolds(n) {
731
+ const start = this.walked + this.fullList.length;
732
+ return start >= this.offset && start + n <= this.offset + this.count;
733
+ }
734
+
735
+ /**
736
+ * Whether the size of the one selection an include makes (a whole code system, or one filter set)
737
+ * is the total of the expansion, as count=0 trusts it: the include is the whole value set (no
738
+ * other include, no exclude), and no code in the selection is left out (inactive ones, or over
739
+ * the code system's cap on how many codes it contributes)
740
+ */
741
+ #sizeIsTotal(compose, cs, excludeInactive) {
742
+ return !excludeInactive && !this.params.activeOnly && !this.hasExclusions
743
+ && (compose.include || []).length === 1 && (compose.exclude || []).length === 0
744
+ && !(cs.expandLimitation() > 0);
745
+ }
746
+
747
+ /**
748
+ * A paged expansion: the provider's size of the selection it walks is its total (see
749
+ * #finishIfPageFilled), when the provider gives one
750
+ */
751
+ #useProviderSize(size) {
752
+ if (typeof size === 'number' && size >= 0) {
753
+ this.worker.opContext.log('paged: the provider gives the total');
754
+ this.providerSize = size;
755
+ }
756
+ }
757
+
758
+ /**
759
+ * Whether the page asked for (count and offset) is one the limits allow, as far as can be told
760
+ * before the walk: a paged walk checks the walk limit as it goes (#walk), so a value set that
761
+ * ends before the page is an empty page, not a refusal
762
+ */
763
+ #pageFits() {
764
+ if (!(this.count > 0) || this.count > this.limitCount) {
765
+ return false;
766
+ }
767
+ return this.paging || this.count + Math.max(this.offset, 0) <= this.limitCount;
768
+ }
769
+
770
+ excludeCode(cs, system, version, code, expansion, imports) {
771
+ this.worker.deadCheck('excludeCode');
772
+ if (!this.passesImports(imports, system, code, 0)) {
773
+ return;
529
774
  }
530
775
 
531
776
  if (expansion) {
@@ -560,7 +805,12 @@ class ValueSetExpander {
560
805
  }
561
806
  }
562
807
 
563
- async expandValueSet(uri, version, vs, filter, notClosed) {
808
+ /**
809
+ * @param {boolean} [deferDetails] - its codes' details are left to be read when this expansion
810
+ * returns them (see deferDetails): for an import used only for which codes it has, or one this
811
+ * expansion returns only some codes of (#importDefers)
812
+ */
813
+ async expandValueSet(uri, version, vs, filter, notClosed, deferDetails = false) {
564
814
 
565
815
  if (!vs) {
566
816
  if (version) {
@@ -579,6 +829,7 @@ class ValueSetExpander {
579
829
  let expander = new ValueSetExpander(worker, paramsInner);
580
830
  expander.internalLimit = this.internalLimit;
581
831
  expander.externalLimit = this.internalLimit; // it's deliberate that this is the internal limit
832
+ expander.deferDetails = deferDetails;
582
833
  let result = await expander.expand(vs, filter, false);
583
834
  if (result == null) {
584
835
  throw new Issue('error', 'not-found', null, 'VS_EXP_IMPORT_UNK', this.worker.i18n.translate('VS_EXP_IMPORT_UNK', this.params.httpLanguages, [uri]), 'unknown');
@@ -600,19 +851,35 @@ class ValueSetExpander {
600
851
  this.checkResourceCanonicalStatus(expansion, vs, this.valueSet);
601
852
 
602
853
  const imported = [];
854
+ // the expansions whose property declarations the codes returned use: the import's, and, for
855
+ // codes whose details were read only now (see #buildDeferred), the ones that added them
856
+ const sources = new Set([vs.expansion]);
603
857
  for (const c of vs.expansion.contains || []) {
604
858
  await this.worker.checkAndYield('importValueSet');
605
- count += await this.importValueSetItem(null, c, imports, offset, imported);
859
+ count += await this.importValueSetItem(null, c, imports, offset, imported, sources);
860
+ }
861
+ for (const source of sources) {
862
+ this.importPropertyDefinitions(expansion, source.property, imported);
606
863
  }
607
- this.importPropertyDefinitions(expansion, vs.expansion.property, imported);
608
864
  return count;
609
865
  }
610
866
 
611
- async importValueSetItem(p, c, imports, offset, imported) {
867
+ async importValueSetItem(p, c, imports, offset, imported, sources) {
612
868
  let count = 0;
613
869
  await this.worker.checkAndYield('importValueSetItem');
614
870
  const s = this.keyC(c);
615
- if (this.passesImports(imports, c.system, c.code, offset) && !this.map.has(s) && !this.isExcluded(c.system, c.version, c.code)) {
871
+ if (this.paging && (this.walked < this.offset || this.pageFilled()) && this.passesImports(imports, c.system, c.code, offset)
872
+ && !(await this.isExcluded(c.system, c.version, c.code))) {
873
+ if (this.#walk(s)) {
874
+ count++;
875
+ }
876
+ } else if (this.passesImports(imports, c.system, c.code, offset) && !this.map.has(s) && !this.walkedKeys.has(s) && !(await this.isExcluded(c.system, c.version, c.code))) {
877
+ // an imported code counts against this expansion's limit like any other
878
+ this.#checkLimit(this.valueSet ? this.valueSet.vurl : null);
879
+ if (!this.deferDetails && !this.noDetails && !this.#outsidePage()) {
880
+ // returned: its details, if they were left to be read
881
+ await this.#buildDeferred(c, sources);
882
+ }
616
883
  count++;
617
884
  if (imported) {
618
885
  imported.push(c);
@@ -628,7 +895,7 @@ class ValueSetExpander {
628
895
  }
629
896
  for (const cc of c.contains || []) {
630
897
  await this.worker.checkAndYield('importValueSetItem');
631
- count += await this.importValueSetItem(c, cc, imports, offset, imported);
898
+ count += await this.importValueSetItem(c, cc, imports, offset, imported, sources);
632
899
  }
633
900
  return count;
634
901
  }
@@ -652,9 +919,9 @@ class ValueSetExpander {
652
919
  walk(vs.expansion.contains);
653
920
  }
654
921
 
655
- async checkSource(cset, exp, filter, srcURL, ts, vsInfo , source) {
922
+ async checkSource(cset, exp, filter, srcURL, ts, vsInfo , source, path = 'ValueSet.compose.include') {
656
923
  await this.worker.checkAndYield('checkSource');
657
- Extensions.checkNoModifiers(cset, 'ValueSetExpander.checkSource', 'set', srcURL);
924
+ Extensions.checkNoModifiers(cset, 'ValueSetExpander.checkSource', path, source && source.vurl ? source.vurl : srcURL);
658
925
  let imp = false;
659
926
  for (const u of cset.valueSet || []) {
660
927
  await this.worker.checkAndYield('checkSource');
@@ -682,13 +949,14 @@ class ValueSetExpander {
682
949
  if (vsInfo && vsInfo.isSimple) {
683
950
  vsInfo.handleByCS = cs.handlesSelecting();
684
951
  }
952
+ // A code system with no content to expand from is refused like an unknown one: 422
685
953
  if (!cs.contentMode()) {
686
- throw new Issue('error', 'business-rule', null, null, 'The code system definition for ' + cset.system + ' has no content property, so this expansion cannot be performed', 'invalid');
954
+ throw new Issue('error', 'business-rule', null, null, 'The code system definition for ' + cset.system + ' has no content property, so this expansion cannot be performed', 'invalid', 422);
687
955
  } else if (cs.contentMode() !== 'complete') {
688
956
  if (cs.contentMode() === 'not-present') {
689
- throw new Issue('error', 'business-rule', null, null, 'The code system definition for ' + cset.system + ' has no content, so this expansion cannot be performed', 'invalid');
957
+ throw new Issue('error', 'business-rule', null, null, 'The code system definition for ' + cset.system + ' has no content, so this expansion cannot be performed', 'invalid', 422);
690
958
  } else if (cs.contentMode() === 'supplement') {
691
- throw new Issue('error', 'business-rule', null, null, 'The code system definition for ' + cset.system + ' defines a supplement, so this expansion cannot be performed', 'invalid');
959
+ throw new Issue('error', 'business-rule', null, null, 'The code system definition for ' + cset.system + ' defines a supplement, so this expansion cannot be performed', 'invalid', 422);
692
960
  } else if (cs.contentMode() === 'fragment') {
693
961
  this.addParamUri(exp, 'used-fragment', cs.system()+ (cs.version() ? '|' + cs.version(): ""));
694
962
  Extensions.addBoolean(exp, "http://hl7.org/fhir/StructureDefinition/valueset-unclosed", true);
@@ -715,12 +983,14 @@ class ValueSetExpander {
715
983
  // NB: totalCount is a method - as a property read (cs.totalCount) the
716
984
  // comparison was against the function object, always false, so this
717
985
  // guard was silently dead.
718
- if (!imp && this.count !== 0 && this.limitCount > 0 && await cs.totalCount() > this.limitCount
719
- && !(this.count > 0 && this.count + Math.max(this.offset, 0) <= this.limitCount)) {
986
+ // (not when the provider will be handed the compose: it pages and counts the
987
+ // selection itself - see processCodes)
988
+ if (!imp && !(vsInfo && vsInfo.handleByCS) && this.count !== 0 && this.limitCount > 0 && await cs.totalCount() > this.limitCount
989
+ && !this.#pageFits()) {
720
990
  // Report the versioned url (vurl), matching the message the
721
991
  // enumeration path produces when it hits the same limit - this
722
992
  // guard is just that refusal moved earlier.
723
- throw new Issue("error", "too-costly", null, 'VALUESET_TOO_COSTLY', this.worker.i18n.translate('VALUESET_TOO_COSTLY', this.params.httpLanguages, [source && source.vurl ? source.vurl : srcURL, '>' + this.limitCount]), null, 422).withDiagnostics(this.worker.opContext.diagnostics());
993
+ throw new Issue("error", "too-costly", null, 'VALUESET_TOO_COSTLY', this.tooCostlyMessage(source && source.vurl ? source.vurl : srcURL, '>' + this.limitCount), null, 422).withDiagnostics(this.worker.opContext.diagnostics());
724
994
  }
725
995
  }
726
996
  }
@@ -728,22 +998,119 @@ class ValueSetExpander {
728
998
  }
729
999
  }
730
1000
 
1001
+ /**
1002
+ * Hand the whole compose to the code system provider (CodeSystemProvider.handlesSelecting /
1003
+ * processSelection). Only called for a compose that selects from one system|version and imports
1004
+ * no value sets (scanValueSet: isSimple), when the provider says it handles selecting.
1005
+ *
1006
+ * Returns true when the provider took the compose: its codes are in the expansion, already
1007
+ * paged by the provider, with the total it reported. Returns false, having added nothing, when
1008
+ * the compose is one this hand-off doesn't cover (a text filter, an enumerated concept list, a
1009
+ * filter with no value, content that isn't complete) or the provider declines by returning
1010
+ * null: the compose then goes through the usual include/exclude path.
1011
+ */
731
1012
  async processCodes(path, vsSrc, compose, filter, expansion, excludeInactive, notClosed, vsInfo) {
1013
+ const includes = compose.include || [];
1014
+ const excludes = compose.exclude || [];
1015
+ if (!filter.isNull) {
1016
+ // the provider isn't told about a text filter
1017
+ return false;
1018
+ }
1019
+ if ([...includes, ...excludes].some(cset => cset.concept)) {
1020
+ // an enumerated concept carries displays, designations and extensions from the value set
1021
+ return false;
1022
+ }
1023
+ if ([...includes, ...excludes].some(cset => (cset.filter || []).some(fc => !fc.value))) {
1024
+ // the usual path reports the error
1025
+ return false;
1026
+ }
732
1027
  const cs = await this.worker.findCodeSystem(vsInfo.system, vsInfo.version, this.params, ['complete', 'fragment'],
733
1028
  false, false, true, null, this.requiredSupplements);
734
- if (cs != null) {
1029
+ if (cs == null || cs.contentMode() !== 'complete') {
1030
+ return false;
1031
+ }
1032
+ for (const cset of [...includes, ...excludes]) {
1033
+ Extensions.checkNoModifiers(cset, 'ValueSetExpander.processCodes', 'set', vsSrc.vurl);
1034
+ for (const fc of cset.filter || []) {
1035
+ Extensions.checkNoModifiers(fc, 'ValueSetExpander.processCodes', 'filter', vsSrc.vurl);
1036
+ if (await supplementFilter(cs, fc)) {
1037
+ // a filter on a property that a supplement in use defines: the provider doesn't know
1038
+ // it, and the usual path evaluates it from the supplement
1039
+ return false;
1040
+ }
1041
+ }
1042
+ }
1043
+
1044
+ const selection = await cs.processSelection(this.params, includes, excludes, excludeInactive, this.offset, this.count,
1045
+ { valueSet: { url: vsSrc.url, version: vsSrc.version } });
1046
+ if (selection == null) {
1047
+ return false;
1048
+ }
1049
+ this.worker.opContext.log('compose handled by the code system provider');
1050
+ // checkSource's up-front refusal of a whole-system include is skipped for a compose handed to
1051
+ // the provider (it is told the paging). Apply it here, now that the selection's total is known,
1052
+ // with the same message: it names the versioned url, where includeCode's refusal would name the
1053
+ // plain url. Only the page counts against the limit: the provider pages itself, so a deep page
1054
+ // costs no more than the first one (on the usual path offset + count must fit, because the
1055
+ // codes before the page are built too)
1056
+ const wholeSystemInclude = includes.some(cset => !(cset.filter || []).length);
1057
+ if (wholeSystemInclude && this.count !== 0 && this.limitCount > 0 && selection.total > this.limitCount
1058
+ && !(this.count > 0 && this.count <= this.limitCount)) {
1059
+ throw new Issue("error", "too-costly", null, 'VALUESET_TOO_COSTLY', this.worker.i18n.translate('VALUESET_TOO_COSTLY', this.params.httpLanguages, [vsSrc.vurl || vsSrc.url, '>' + this.limitCount]), null, 422).withDiagnostics(this.worker.opContext.diagnostics());
1060
+ }
1061
+
1062
+ for (const cset of [...includes, ...excludes]) {
1063
+ this.worker.checkSupplements(cs, cset, this.requiredSupplements, this.usedSupplements);
1064
+ }
1065
+ this.checkProviderCanonicalStatus(expansion, cs, this.valueSet);
1066
+ this.addParamUri(expansion, 'used-codesystem', this.canonical(cs.system(), cs.version()));
1067
+ if (selection.notClosed) {
1068
+ notClosed.value = true;
1069
+ const wholeSystem = includes.some(cset => !(cset.filter || []).length);
1070
+ if (wholeSystem && cs.isNotClosed() && !Extensions.has(expansion, "http://hl7.org/fhir/StructureDefinition/valueset-unclosed-reason")) {
1071
+ // as the usual path says for a whole grammar-based code system
1072
+ Extensions.addString(expansion, "http://hl7.org/fhir/StructureDefinition/valueset-unclosed-reason", "The code System '" + cs.system() + "' has a grammar and so has infinite members");
1073
+ }
1074
+ }
735
1075
 
736
- // set up the call to the provider
737
- // call the provider
738
- // include the codes
1076
+ // the provider has applied offset and count: every code it returns is in the page
1077
+ vsInfo.csDoOffset = true;
1078
+ this.paging = false;
1079
+ this.providerPaged = true;
1080
+ this.canBeHierarchy = false;
1081
+ const cds = new Designations(this.worker.i18n.languageDefinitions);
1082
+ for (const set of selection.sets || []) {
1083
+ while (await cs.filterMore(null, set)) {
1084
+ await this.worker.checkAndYield('processCodes#selection');
1085
+ const c = await cs.filterConcept(null, set);
1086
+ cds.clear();
1087
+ if (this.noDetails) {
1088
+ await this.includeCode(cs, null, cs.system(), cs.version(), await cs.code(c), await cs.isAbstract(c), await cs.isInactive(c),
1089
+ null, null, cds, null, null, expansion, null, null, null, null, null, excludeInactive, vsSrc.url);
1090
+ } else if (this.#defers(cs, await cs.code(c))) {
1091
+ await this.includeCode(cs, null, cs.system(), cs.version(), await cs.code(c), await cs.isAbstract(c), await cs.isInactive(c),
1092
+ null, null, cds, null, null, expansion, null, null, null, null, null, excludeInactive, vsSrc.url, () => this.#readDetails(cs, c));
1093
+ } else {
1094
+ await this.listDisplaysFromProvider(cds, cs, c);
1095
+ await this.includeCode(cs, null, cs.system(), cs.version(), await cs.code(c), await cs.isAbstract(c), await cs.isInactive(c),
1096
+ await cs.isDeprecated(c), await cs.getStatus(c), cds, await cs.definition(c), await cs.itemWeight(c),
1097
+ expansion, null, await cs.extensions(c), null, await this.#conceptProperties(cs, c), null, excludeInactive, vsSrc.url);
1098
+ }
1099
+ }
739
1100
  }
1101
+ if (selection.total != null && selection.total >= 0) {
1102
+ this.addToTotal(selection.total);
1103
+ } else {
1104
+ this.noTotal();
1105
+ }
1106
+ return true;
740
1107
  }
741
1108
 
742
1109
  async includeCodes(cset, path, vsSrc, compose, filter, expansion, excludeInactive, notClosed) {
743
1110
  await this.worker.checkAndYield('processCodes#1');
744
1111
  const valueSets = [];
745
1112
 
746
- Extensions.checkNoModifiers(cset, 'ValueSetExpander.processCodes', 'set', vsSrc.vurl);
1113
+ Extensions.checkNoModifiers(cset, 'ValueSetExpander.processCodes', path, vsSrc.vurl);
747
1114
 
748
1115
  if (cset.valueSet || cset.concept || (cset.filter || []).length > 1) {
749
1116
  this.canBeHierarchy = false;
@@ -755,7 +1122,8 @@ class ValueSetExpander {
755
1122
  const s = this.worker.pinValueSet(u);
756
1123
  this.worker.opContext.log('import value set ' + s);
757
1124
  let vs = await this.worker.findValueSet(s, '', vsSrc);
758
- const ivs = new ImportedValueSet(await this.expandValueSet(s, '', vs, filter, notClosed));
1125
+ // the first value set's codes are this include's; the others' only test membership
1126
+ const ivs = new ImportedValueSet(await this.expandValueSet(s, '', vs, filter, notClosed, valueSets.length > 0 || this.#importDefers()));
759
1127
  this. checkResourceCanonicalStatus(expansion, ivs.valueSet, this.valueSet);
760
1128
  if (!vs.isContained) {
761
1129
  this.addParamUri(expansion, 'used-valueset', this.worker.makeVurl(ivs.valueSet));
@@ -782,7 +1150,8 @@ class ValueSetExpander {
782
1150
  const s = this.worker.pinValueSet(u);
783
1151
  this.worker.opContext.log('import value set ' + s);
784
1152
  let vs = await this.worker.findValueSet(s, '', vsSrc);
785
- const ivs = new ImportedValueSet(await this.expandValueSet(s, '', vs, filter, notClosed));
1153
+ // (only for which codes it has: the include's codes come from the code system)
1154
+ const ivs = new ImportedValueSet(await this.expandValueSet(s, '', vs, filter, notClosed, true));
786
1155
  this.checkResourceCanonicalStatus(expansion, ivs.valueSet, this.valueSet);
787
1156
  if (!vs.isContained) {
788
1157
  this.addParamUri(expansion, 'used-valueset', this.worker.makeVurl(ivs.valueSet));
@@ -794,7 +1163,7 @@ class ValueSetExpander {
794
1163
  if (cs.specialEnumeration() && filters.length === 0) {
795
1164
  this.worker.opContext.log('import special value set ' + cs.specialEnumeration());
796
1165
  let vs = await this.worker.findValueSet(cs.specialEnumeration(), '', null);
797
- const base = await this.expandValueSet(cs.specialEnumeration(), '', vs, filter, notClosed);
1166
+ const base = await this.expandValueSet(cs.specialEnumeration(), '', vs, filter, notClosed, this.#importDefers());
798
1167
  Extensions.addBoolean(expansion, "http://hl7.org/fhir/StructureDefinition/valueset-unclosed", true);
799
1168
  Extensions.addString(expansion, "http://hl7.org/fhir/StructureDefinition/valueset-unclosed-reason", "The code System '" + cs.system() + "' has a grammar and so has infinite members. This extension is based on " + cs.specialEnumeration());
800
1169
  await this.importValueSet(base, expansion, valueSets, 0);
@@ -815,9 +1184,17 @@ class ValueSetExpander {
815
1184
  Extensions.addString(expansion, "http://hl7.org/fhir/StructureDefinition/valueset-unclosed-reason", "The code System '" + cs.system() + "' has a grammar and so has infinite members");
816
1185
  }
817
1186
  notClosed.value = true;
1187
+ } else if (cs.openCodeSpaceReason()) {
1188
+ // all the codes the code system defines are listed, but not the ones users define in its
1189
+ // reserved code spaces: $validate-code accepts those as members, so the expansion isn't closed
1190
+ if (!Extensions.has(expansion, "http://hl7.org/fhir/StructureDefinition/valueset-unclosed")) {
1191
+ Extensions.addBoolean(expansion, "http://hl7.org/fhir/StructureDefinition/valueset-unclosed", true);
1192
+ Extensions.addString(expansion, "http://hl7.org/fhir/StructureDefinition/valueset-unclosed-reason", cs.openCodeSpaceReason());
1193
+ }
1194
+ notClosed.value = true;
818
1195
  }
819
1196
 
820
- if (this.count === 0 && filters.length === 0 && valueSets.length === 0) {
1197
+ if (this.count === 0 && filters.length === 0 && valueSets.length === 0 && !this.hasExclusions) {
821
1198
  // count=0 (count only) with a single whole-code-system include and no
822
1199
  // additional filters: report the code count directly rather than
823
1200
  // enumerating - which for a poly-hierarchy (e.g. SNOMED) is both
@@ -825,10 +1202,16 @@ class ValueSetExpander {
825
1202
  this.addToTotal(await cs.countAllCodes(excludeInactive));
826
1203
  } else {
827
1204
  const iter = await cs.iterator(null);
828
- if (valueSets.length === 0 && this.limitCount > 0 && (iter && iter.total > this.limitCount) && this.offset < 0) {
829
- throw new Issue("error", "too-costly", null, 'VALUESET_TOO_COSTLY', this.worker.i18n.translate('VALUESET_TOO_COSTLY', this.params.httpLanguages, [vsSrc.vurl, '>' + this.limitCount]), null, 422).withDiagnostics(this.worker.opContext.diagnostics());
1205
+ if (valueSets.length === 0 && this.count !== 0 && this.limitCount > 0 && (iter && iter.total > this.limitCount) && this.offset < 0) {
1206
+ throw new Issue("error", "too-costly", null, 'VALUESET_TOO_COSTLY', this.tooCostlyMessage(vsSrc.vurl, '>' + this.limitCount), null, 422).withDiagnostics(this.worker.opContext.diagnostics());
830
1207
 
831
1208
  }
1209
+ if (this.paging && valueSets.length === 0 && filters.length === 0 && this.#sizeIsTotal(compose, cs, excludeInactive) && !this.params.excludeNotForUI
1210
+ && !cs.isNotClosed()) {
1211
+ // the walk would count every code of the code system (excludeNotForUI would skip the
1212
+ // abstract ones; a grammar's walk lists only some): its size is the total
1213
+ this.#useProviderSize(await cs.countAllCodes(excludeInactive));
1214
+ }
832
1215
  let tcount = 0;
833
1216
  // One visited set across all roots: in a poly-hierarchy the walks
834
1217
  // from different roots can reach the same concepts.
@@ -836,19 +1219,31 @@ class ValueSetExpander {
836
1219
  let c = await cs.nextContext(iter);
837
1220
  while (c) {
838
1221
  await this.worker.checkAndYield('processCodes#3a');
1222
+ if (this.#capReached(cs)) {
1223
+ // the code system's cap is reached: nothing more of it can be added
1224
+ this.#noteCapCut(cs);
1225
+ break;
1226
+ }
839
1227
  if (await this.passesFilters(cs, c, prep, filters, 0)) {
840
1228
  tcount += await this.includeCodeAndDescendants(cs, c, expansion, valueSets, null, excludeInactive, vsSrc.vurl, visited);
841
1229
  }
842
1230
  c = await cs.nextContext(iter);
843
1231
  }
844
- this.addToTotal(tcount);
1232
+ if (cs.expandLimitation() > 0 && filters.length === 0 && valueSets.length === 0 && !this.hasExclusions) {
1233
+ // a code system that caps how many of its codes an expansion may list (e.g. CPT): the
1234
+ // cap limits which codes can be on a page, not how many the value set has, so the
1235
+ // total is all of them, as for count=0 (and the walk stopped at the cap)
1236
+ this.addToTotal(await cs.countAllCodes(excludeInactive));
1237
+ } else {
1238
+ this.addToTotal(tcount);
1239
+ }
845
1240
  }
846
1241
  } else {
847
1242
  this.worker.opContext.log('prepare filters');
848
1243
  if (cs.isNotClosed(filter)) {
849
1244
  notClosed.value = true;
850
1245
  }
851
- const prep = await cs.getPrepContext(true);
1246
+ const prep = await cs.getPrepContext(true, this.params.requestedLanguages()); // the text filter's language
852
1247
  const ctxt = await cs.searchFilter(prep, filter, true);
853
1248
  let set = await cs.executeFilters(prep);
854
1249
  if (this.count === 0 && valueSets.length === 0) {
@@ -862,17 +1257,31 @@ class ValueSetExpander {
862
1257
  await this.worker.checkAndYield('processCodes#4');
863
1258
  const c = await cs.filterConcept(ctxt, set[0]);
864
1259
  if (await this.passesFilters(cs, c, prep, set, 1)) {
1260
+ const walked = await this.#beforePage(cs, await cs.system(), await cs.version(), await cs.code(c), await cs.isInactive(c), excludeInactive, valueSets);
1261
+ if (walked !== null) {
1262
+ // (no total on this path: once the page is full there's nothing to count)
1263
+ if (this.pageFilled()) {
1264
+ break;
1265
+ }
1266
+ continue;
1267
+ }
865
1268
  const cds = new Designations(this.worker.i18n.languageDefinitions);
866
1269
  let added;
867
- if (this.noDetails) {
1270
+ if (!this.noDetails && this.#outsidePage()) {
1271
+ added = await this.includeCode(cs, null, await cs.system(), await cs.version(), await cs.code(c), await cs.isAbstract(c), await cs.isInactive(c), false, null,
1272
+ cds, null, null, expansion, valueSets, null, null, null, null, excludeInactive, vsSrc.url);
1273
+ } else if (this.noDetails) {
868
1274
  await this.listDisplaysFromProvider(cds, cs, c);
869
1275
  added = await this.includeCode(cs, null, await cs.system(), await cs.version(), await cs.code(c), await cs.isAbstract(c), await cs.isInactive(c), false, null,
870
1276
  cds, null, null, expansion, valueSets, null, null, null, null, excludeInactive, vsSrc.url);
871
1277
 
1278
+ } else if (this.#defers(cs, await cs.code(c))) {
1279
+ added = await this.includeCode(cs, null, await cs.system(), await cs.version(), await cs.code(c), await cs.isAbstract(c), await cs.isInactive(c), false, null,
1280
+ cds, null, null, expansion, valueSets, null, null, null, null, excludeInactive, vsSrc.url, () => this.#readDetails(cs, c));
872
1281
  } else {
873
1282
  await this.listDisplaysFromProvider(cds, cs, c);
874
1283
  added = await this.includeCode(cs, null, await cs.system(), await cs.version(), await cs.code(c), await cs.isAbstract(c), await cs.isInactive(c), await cs.isDeprecated(c), await cs.getStatus(c),
875
- cds, await cs.definition(c), await cs.itemWeight(c), expansion, valueSets, await cs.extensions(c), null, await cs.properties(c), null, excludeInactive, vsSrc.url);
1284
+ cds, await cs.definition(c), await cs.itemWeight(c), expansion, valueSets, await cs.extensions(c), null, await this.#conceptProperties(cs, c), null, excludeInactive, vsSrc.url);
876
1285
  }
877
1286
  if (added) {
878
1287
  this.addToTotal();
@@ -892,35 +1301,86 @@ class ValueSetExpander {
892
1301
  }
893
1302
  }
894
1303
 
895
- if (cset.concept) {
1304
+ // A value set that lists its codes: count them first - locating each code is cheap,
1305
+ // building it (displays, properties) is not. count=0 asks only for the total, so when
1306
+ // every include lists its codes, the count is the answer; otherwise, if the codes
1307
+ // would take the expansion over the limit, refuse now rather than after building them
1308
+ // (only worth a pass when it could change anything: the list alone could be over the limit)
1309
+ const countFirst = cset.concept && filter.isNull
1310
+ && ((this.count === 0 && this.composeOnlyListsCodes) || this.#listedCodesOverLimit(cset.concept.length));
1311
+ const counted = countFirst ? await this.#countListedCodes(cs, cset, valueSets, filters, prep, excludeInactive) : null;
1312
+ if (counted && this.count === 0 && this.composeOnlyListsCodes) {
1313
+ this.worker.opContext.log('count only: counted the listed codes');
1314
+ for (const key of counted) {
1315
+ this.countedKeys.add(key);
1316
+ }
1317
+ if (counted.size > 0 && !this.noDetails) {
1318
+ this.addParamUri(expansion, 'used-codesystem', this.canonical(cs.system(), cs.version()));
1319
+ for (const vs of cs.listSupplements(false)) {
1320
+ this.reportedSupplements.add(vs);
1321
+ }
1322
+ }
1323
+ this.addToTotal(counted.size);
1324
+ } else if (cset.concept) {
1325
+ if (counted && this.#listedCodesOverLimit(counted.size)) {
1326
+ throw new Issue("error", "too-costly", null, 'VALUESET_TOO_COSTLY', this.tooCostlyMessage(vsSrc.url, '>' + this.limitCount), null, 422).withDiagnostics(this.worker.opContext.diagnostics());
1327
+ }
896
1328
  this.worker.opContext.log('iterate concepts');
897
1329
  const cds = new Designations(this.worker.i18n.languageDefinitions);
898
-
899
- for (const cc of cset.concept) {
1330
+ // a text filter matches designations in one language (see CodeSystemProvider.searchLanguage)
1331
+ const searchLang = filter.isNull ? null : cs.searchLanguage(this.params.requestedLanguages());
1332
+
1333
+ // each listed code is located next: a provider may read them together, and hand back what it read
1334
+ // (paging: most of them may only be counted, and those on the page are read at locate -
1335
+ // unless the page holds the whole list, when every one is built. And not when their
1336
+ // details are left to be read later: see deferDetails)
1337
+ const prefetched = await cs.prefetchCodes(cset.concept.map(cc => cc.code),
1338
+ !this.noDetails && !(filter.isNull && this.#defers(cs)) && (!this.paging || this.#pageHolds(cset.concept.length)));
1339
+ for (const [ci, cc] of cset.concept.entries()) {
900
1340
  await this.worker.checkAndYield('processCodes#3');
901
1341
  cds.clear();
902
- Extensions.checkNoModifiers(cc, 'ValueSetExpander.processCodes', 'set concept reference', vsSrc.vurl);
903
- const cctxt = await cs.locate(cc.code, this.allAltCodes);
1342
+ Extensions.checkNoModifiers(cc, 'ValueSetExpander.processCodes', path + '.concept[' + ci + ']', vsSrc.vurl);
1343
+ const cctxt = await cs.locate(cc.code, prefetched);
904
1344
  if (cctxt && cctxt.context && (!this.params.activeOnly || !await cs.isInactive(cctxt.context)) && await this.passesFilters(cs, cctxt.context, prep, filters, 0)) {
905
1345
  let added;
906
1346
  const ecode = await cs.expansionCode(cctxt.context, cc.code);
1347
+ const walked = await this.#beforePage(cs, cs.system(), cs.version(), ecode, await cs.isInactive(cctxt.context), excludeInactive, valueSets);
1348
+ if (walked !== null) {
1349
+ if (walked) {
1350
+ this.addToTotal();
1351
+ }
1352
+ continue;
1353
+ }
907
1354
  if (this.noDetails) {
908
1355
  added = await this.includeCode(cs, null, cs.system(), cs.version(), ecode, await cs.isAbstract(cctxt.context), await cs.isInactive(cctxt.context), null, null, cds,
909
1356
  null, null, expansion, valueSets, null, null, null, null, excludeInactive, vsSrc.url);
1357
+ } else if (filter.isNull && this.#outsidePage()) {
1358
+ added = await this.includeCode(cs, null, cs.system(), cs.version(), ecode, await cs.isAbstract(cctxt.context), await cs.isInactive(cctxt.context), null, null, cds,
1359
+ null, null, expansion, valueSets, null, null, null, null, excludeInactive, vsSrc.url);
1360
+ if (added) {
1361
+ this.addToTotal();
1362
+ }
1363
+ } else if (filter.isNull && this.#defers(cs, ecode)) {
1364
+ added = await this.includeCode(cs, null, cs.system(), cs.version(), ecode, await cs.isAbstract(cctxt.context), await cs.isInactive(cctxt.context), null, null, cds,
1365
+ null, null, expansion, valueSets, null, null, null, null, excludeInactive, vsSrc.url, () => this.#readDetails(cs, cctxt.context, cc, vsSrc));
1366
+ if (added) {
1367
+ this.addToTotal();
1368
+ }
910
1369
  } else {
911
1370
  await this.listDisplaysFromProvider(cds, cs, cctxt.context);
912
1371
  this.listDisplaysFromIncludeConcept(cds, cc, vsSrc);
913
- if (filter.passesDesignations(cds) || filter.passes(cc.code)) {
1372
+ if (filter.passesDesignations(cds, searchLang, cs.defLang()) || filter.passes(cc.code)) {
914
1373
  let ov = Extensions.readString(cc, 'http://hl7.org/fhir/StructureDefinition/itemWeight');
915
1374
  if (!ov) {
916
1375
  ov = await cs.itemWeight(cctxt.context);
917
1376
  }
918
1377
  added = await this.includeCode(cs, null, cs.system(), cs.version(), ecode, await cs.isAbstract(cctxt.context), await cs.isInactive(cctxt.context), await cs.isDeprecated(cctxt.context), await cs.getStatus(cctxt.context), cds,
919
- await cs.definition(cctxt.context), ov, expansion, valueSets, await cs.extensions(cctxt.context), cc.extension, await cs.properties(cctxt.context), null, excludeInactive, vsSrc.url);
1378
+ await cs.definition(cctxt.context), ov, expansion, valueSets, await cs.extensions(cctxt.context), cc.extension, await this.#conceptProperties(cs, cctxt.context), null, excludeInactive, vsSrc.url);
920
1379
  }
921
1380
  if (added) {
922
1381
  this.addToTotal();
923
1382
  }
1383
+ this.#finishIfPageFilled();
924
1384
  }
925
1385
  }
926
1386
  }
@@ -930,7 +1390,7 @@ class ValueSetExpander {
930
1390
  if (cset.filter && cset.filter.length > 0) {
931
1391
  this.worker.opContext.log('prepare filters');
932
1392
  const fcl = cset.filter;
933
- const prep = await cs.getPrepContext(true);
1393
+ const prep = await cs.getPrepContext(true, this.params.requestedLanguages());
934
1394
 
935
1395
  if (!filter.isNull) {
936
1396
  await cs.searchFilter(prep, filter, true);
@@ -943,17 +1403,23 @@ class ValueSetExpander {
943
1403
  }
944
1404
 
945
1405
  for (let i = 0; i < fcl.length; i++) {
946
- await this.worker.checkAndYield('processCodes#4a');
947
1406
  const fc = fcl[i];
948
1407
  if (!fc.value) {
949
1408
  throw new Issue('error', 'invalid', path + ".filter[" + i + "]", 'UNABLE_TO_HANDLE_SYSTEM_FILTER_WITH_NO_VALUE', this.worker.i18n.translate('UNABLE_TO_HANDLE_SYSTEM_FILTER_WITH_NO_VALUE', this.params.httpLanguages, [cs.system(), fc.property, fc.op]), 'vs-invalid', 400);
950
1409
  }
951
- Extensions.checkNoModifiers(fc, 'ValueSetExpander.processCodes', 'filter', vsSrc.vurl);
1410
+ Extensions.checkNoModifiers(fc, 'ValueSetExpander.processCodes', path + '.filter[' + i + ']', vsSrc.vurl);
1411
+ }
1412
+ // filters on properties that supplements in use define are evaluated from the
1413
+ // supplements; the code system does the rest
1414
+ const { base: baseFilters, supplement: suppFilters } = await splitFilters(cs, fcl);
1415
+ for (let i = 0; i < baseFilters.length; i++) {
1416
+ await this.worker.checkAndYield('processCodes#4a');
1417
+ const fc = baseFilters[i];
952
1418
  await cs.filter(prep, i == 0, fc.property, fc.op, fc.value, vsSrc.isCached ? fc : null);
953
1419
  }
954
1420
 
955
- const fset = await cs.executeFilters(prep);
956
- const filtersOpen = await cs.filtersNotClosed(prep);
1421
+ const fset = baseFilters.length > 0 ? await cs.executeFilters(prep) : [];
1422
+ const filtersOpen = baseFilters.length > 0 && await cs.filtersNotClosed(prep);
957
1423
  if (filtersOpen) {
958
1424
  notClosed.value = true;
959
1425
  }
@@ -963,32 +1429,49 @@ class ValueSetExpander {
963
1429
  // total is the set's size, and walking it would only end in VALUESET_TOO_COSTLY once
964
1430
  // the set is bigger than the limit. Anything less certain is counted by walking, as
965
1431
  // before: an earlier version added filterSize *and* walked, and double counted (#91, #93).
966
- const countOnly = this.count === 0 && !filtersOpen && fset.length === 1 && filter.isNull
967
- && !excludeInactive && !this.params.activeOnly && !this.hasExclusions
968
- && !(cset.concept && cset.concept.length) && !(cset.valueSet && cset.valueSet.length)
969
- && (compose.include || []).length === 1 && (compose.exclude || []).length === 0;
1432
+ // The same holds for a page: once it is full, the walk would only count the rest of the set
1433
+ const sizeIsTotal = !filtersOpen && fset.length === 1 && suppFilters.length === 0 && filter.isNull
1434
+ && !(cset.concept && cset.concept.length) && !(cset.valueSet && cset.valueSet.length) && this.#sizeIsTotal(compose, cs, excludeInactive);
1435
+ const countOnly = this.count === 0 && sizeIsTotal;
970
1436
  if (countOnly) {
971
1437
  this.worker.opContext.log('count only: the filter size is the total');
972
1438
  this.addToTotal(await cs.filterSize(prep, fset[0]));
1439
+ } else if (this.paging && sizeIsTotal) {
1440
+ this.#useProviderSize(await cs.filterSize(prep, fset[0]));
973
1441
  }
974
1442
 
975
1443
  this.worker.opContext.log('iterate filters');
976
1444
  this.addToTotal(0);
977
1445
  const cds = new Designations(this.worker.i18n.languageDefinitions);
978
- while (!countOnly && await cs.filterMore(prep, fset[0])) {
979
- await this.worker.checkAndYield('processCodes#5');
980
- const c = await cs.filterConcept(prep, fset[0]);
981
- const ok = (!this.params.activeOnly || !await cs.isInactive(c)) && (await this.passesFilters(cs, c, prep, fset, 1));
982
- if (ok) {
1446
+ const addFound = async (c) => {
1447
+ {
983
1448
  cds.clear();
984
1449
  // count++;
985
1450
  if (this.passesImports(valueSets, cs.system(), await cs.code(c), 0)) {
1451
+ const walked = await this.#beforePage(cs, await cs.system(), await cs.version(), await cs.code(c), await cs.isInactive(c), excludeInactive, null);
1452
+ if (walked !== null) {
1453
+ if (walked) {
1454
+ this.addToTotal();
1455
+ }
1456
+ return;
1457
+ }
986
1458
  let added;
987
- if (this.noDetails) {
1459
+ if (this.noDetails || this.#outsidePage()) {
988
1460
  added = await this.includeCode(cs, null, await cs.system(), await cs.version(), await cs.code(c), await cs.isAbstract(c), await cs.isInactive(c),
989
1461
  null, null, cds, null, null,
990
1462
  expansion, null, null, null, null, null, excludeInactive, vsSrc.url);
991
1463
 
1464
+ } else if (this.#defers(cs, await cs.code(c))) {
1465
+ // (the parent as below: the hierarchy decides the order an importing expansion takes the codes in)
1466
+ let parent = null;
1467
+ if (cs.hasParents()) {
1468
+ parent = this.map.get(this.keyS(cs.system(), cs.version(), await cs.parent(c)));
1469
+ } else {
1470
+ this.canBeHierarchy = false;
1471
+ }
1472
+ added = await this.includeCode(cs, parent, await cs.system(), await cs.version(), await cs.code(c), await cs.isAbstract(c), await cs.isInactive(c),
1473
+ null, null, cds, null, null,
1474
+ expansion, null, null, null, null, null, excludeInactive, vsSrc.url, () => this.#readDetails(cs, c));
992
1475
  } else {
993
1476
  await this.listDisplaysFromProvider(cds, cs, c);
994
1477
  let parent = null;
@@ -999,11 +1482,31 @@ class ValueSetExpander {
999
1482
  }
1000
1483
  added = await this.includeCode(cs, parent, await cs.system(), await cs.version(), await cs.code(c), await cs.isAbstract(c), await cs.isInactive(c),
1001
1484
  await cs.isDeprecated(c), await cs.getStatus(c), cds, await cs.definition(c), await cs.itemWeight(c),
1002
- expansion, null, await cs.extensions(c), null, await cs.properties(c), null, excludeInactive, vsSrc.url);
1485
+ expansion, null, await cs.extensions(c), null, await this.#conceptProperties(cs, c), null, excludeInactive, vsSrc.url);
1003
1486
  }
1004
1487
  if (added) {
1005
1488
  this.addToTotal();
1006
1489
  }
1490
+ this.#finishIfPageFilled();
1491
+ }
1492
+ }
1493
+ };
1494
+ if (baseFilters.length > 0) {
1495
+ while (!countOnly && await cs.filterMore(prep, fset[0])) {
1496
+ await this.worker.checkAndYield('processCodes#5');
1497
+ const c = await cs.filterConcept(prep, fset[0]);
1498
+ const ok = (!this.params.activeOnly || !await cs.isInactive(c)) && (await this.passesFilters(cs, c, prep, fset, 1))
1499
+ && (await this.#passesSupplementFilters(cs, c, suppFilters));
1500
+ if (ok) {
1501
+ await addFound(c);
1502
+ }
1503
+ }
1504
+ } else {
1505
+ // only filters on supplement properties: the codes are those the supplements list
1506
+ for (const c of await this.#supplementFilterConcepts(cs, suppFilters, path)) {
1507
+ await this.worker.checkAndYield('processCodes#5');
1508
+ if ((!this.params.activeOnly || !await cs.isInactive(c)) && (filter.isNull || filter.passes(await cs.code(c)) || filter.passes(await cs.display(c)))) {
1509
+ await addFound(c);
1007
1510
  }
1008
1511
  }
1009
1512
  }
@@ -1013,6 +1516,41 @@ class ValueSetExpander {
1013
1516
  }
1014
1517
  }
1015
1518
 
1519
+ async #passesSupplementFilters(cs, c, suppFilters) {
1520
+ if (suppFilters.length === 0) {
1521
+ return true;
1522
+ }
1523
+ const code = await cs.code(c);
1524
+ return suppFilters.every(sf => sf.matches(code));
1525
+ }
1526
+
1527
+ /**
1528
+ * The concepts selected by filters that are all on supplement properties: the codes a
1529
+ * supplement lists that pass them all, and that the code system has (a supplement can list
1530
+ * a code the code system doesn't)
1531
+ */
1532
+ async #supplementFilterConcepts(cs, suppFilters, path) {
1533
+ const driver = suppFilters.find(sf => sf.enumerable);
1534
+ if (!driver) {
1535
+ // not-in / exists = false select codes by what the supplement doesn't say, which needs the
1536
+ // rest of the code system to select from
1537
+ const sf = suppFilters[0];
1538
+ throw new Issue('error', 'not-supported', path + '.filter', 'FILTER_NOT_UNDERSTOOD',
1539
+ this.worker.i18n.translate('FILTER_NOT_UNDERSTOOD', this.params.HTTPLanguages, [sf.property, sf.op, sf.value]), 'vs-invalid', 422);
1540
+ }
1541
+ const result = [];
1542
+ for (const code of driver.codes()) {
1543
+ await this.worker.checkAndYield('supplementFilterConcepts');
1544
+ if (suppFilters.every(sf => sf === driver || sf.matches(code))) {
1545
+ const loc = await cs.locate(code);
1546
+ if (loc && loc.context) {
1547
+ result.push(loc.context);
1548
+ }
1549
+ }
1550
+ }
1551
+ return result;
1552
+ }
1553
+
1016
1554
  async passesFilters(cs, c, prep, filters, offset) {
1017
1555
  for (let j = offset; j < filters.length; j++) {
1018
1556
  const f = filters[j];
@@ -1028,7 +1566,7 @@ class ValueSetExpander {
1028
1566
  await this.worker.checkAndYield('processCodes#1');
1029
1567
  const valueSets = [];
1030
1568
 
1031
- Extensions.checkNoModifiers(cset, 'ValueSetExpander.processCodes', 'set', vsSrc.vurl);
1569
+ Extensions.checkNoModifiers(cset, 'ValueSetExpander.processCodes', path, vsSrc.vurl);
1032
1570
 
1033
1571
  if (cset.valueSet || cset.concept || (cset.filter || []).length > 1) {
1034
1572
  this.canBeHierarchy = false;
@@ -1041,7 +1579,8 @@ class ValueSetExpander {
1041
1579
  const s = this.worker.pinValueSet(u);
1042
1580
  await this.worker.checkAndYield('processCodes#2');
1043
1581
  let vs = await this.worker.findValueSet(s, '', vsSrc);
1044
- const ivs = new ImportedValueSet(await this.expandValueSet(s, '', vs, filter, notClosed));
1582
+ // (only for which codes it has, as for the excludes below)
1583
+ const ivs = new ImportedValueSet(await this.expandValueSet(s, '', vs, filter, notClosed, true));
1045
1584
  this.checkResourceCanonicalStatus(expansion, ivs.valueSet, this.valueSet);
1046
1585
  // ivs.valueSet is the expansion (plain JSON), so it has no vurl - as for the
1047
1586
  // includes, build it
@@ -1071,7 +1610,7 @@ class ValueSetExpander {
1071
1610
  const s = this.worker.pinValueSet(u);
1072
1611
  this.worker.opContext.log('import value set ' + s);
1073
1612
  let vs = await this.worker.findValueSet(s, '', vsSrc);
1074
- const ivs = new ImportedValueSet(await this.expandValueSet(s, '', vs, filter, notClosed));
1613
+ const ivs = new ImportedValueSet(await this.expandValueSet(s, '', vs, filter, notClosed, true));
1075
1614
  this.checkResourceCanonicalStatus(expansion, ivs.valueSet, this.valueSet);
1076
1615
  if (!vs.isContained) {
1077
1616
  this.addParamUri(expansion, 'used-valueset', this.worker.makeVurl(ivs.valueSet));
@@ -1086,15 +1625,11 @@ class ValueSetExpander {
1086
1625
  // excluding a whole system - we don't list the codes in this case
1087
1626
  this.excludedSystems.add(cset.system + (this.doingVersion && cset.version ? '|' + cset.version : ''));
1088
1627
  } else {
1089
- const iter = await cs.iteratorAll();
1090
- if (iter) {
1091
- let c = await cs.nextContext(iter);
1092
- while (c) {
1093
- await this.worker.checkAndYield('processCodes#3aa');
1094
- this.excludeCode(cs, cs.system(), cs.version(), await cs.code(c), expansion, valueSets, vsSrc.url);
1095
- c = await cs.nextContext(iter);
1096
- }
1097
- }
1628
+ // include one version, exclude another whole version: a code is
1629
+ // excluded if it exists in the excluded version. We don't list
1630
+ // the excluded version's codes (most providers can't - e.g.
1631
+ // SNOMED CT, LOINC); we ask it about each included code instead
1632
+ this.excludedProviders.push({ system: cs.system(), cs });
1098
1633
  }
1099
1634
  } else {
1100
1635
  if (cs.isNotClosed(filter)) {
@@ -1110,15 +1645,15 @@ class ValueSetExpander {
1110
1645
  }
1111
1646
  }
1112
1647
 
1113
- const iter = await cs.iteratorAll();
1114
- if (iter) {
1115
- let c = await cs.nextContext(iter);
1116
- while (c) {
1117
- await this.worker.checkAndYield('processCodes#3a');
1118
- if (await this.passesFilters(cs, c, prep, filters, 0)) {
1119
- this.excludeCode(cs, cs.system(), cs.version(), await cs.code(c), expansion, valueSets, vsSrc.url);
1120
- }
1121
- c = await cs.nextContext(iter);
1648
+ // the codes of this system that are in all the value sets. Walk the
1649
+ // first value set's codes, not the code system (most providers
1650
+ // can't list theirs - e.g. SNOMED CT, LOINC); excludeCode checks
1651
+ // the other value sets
1652
+ const system = cs.system();
1653
+ for (const entry of valueSets[0].codeMap.values()) {
1654
+ await this.worker.checkAndYield('processCodes#3a');
1655
+ if (entry.system === system && await this.#knownTo(cs, entry.code)) {
1656
+ this.excludeCode(cs, system, cs.version(), entry.code, expansion, valueSets);
1122
1657
  }
1123
1658
  }
1124
1659
  }
@@ -1127,18 +1662,22 @@ class ValueSetExpander {
1127
1662
  if (cset.concept) {
1128
1663
  this.worker.opContext.log('iterate concepts');
1129
1664
  const cds = new Designations(this.worker.i18n.languageDefinitions);
1130
- for (const cc of cset.concept) {
1665
+ // a text filter matches designations in one language (see CodeSystemProvider.searchLanguage)
1666
+ const searchLang = filter.isNull ? null : cs.searchLanguage(this.params.requestedLanguages());
1667
+ // each listed code is located next: a provider may read them together, and hand back what it read
1668
+ const prefetched = await cs.prefetchCodes(cset.concept.map(cc => cc.code), false);
1669
+ for (const [ci, cc] of cset.concept.entries()) {
1131
1670
  await this.worker.checkAndYield('processCodes#3');
1132
1671
  cds.clear();
1133
- Extensions.checkNoModifiers(cc, 'ValueSetExpander.processCodes', 'set concept reference', vsSrc.vurl);
1134
- const cctxt = await cs.locate(cc.code, this.allAltCodes);
1672
+ Extensions.checkNoModifiers(cc, 'ValueSetExpander.processCodes', path + '.concept[' + ci + ']', vsSrc.vurl);
1673
+ const cctxt = await cs.locate(cc.code, prefetched);
1135
1674
  if (cctxt && cctxt.context && (!this.params.activeOnly || !await cs.isInactive(cctxt.context)) && await this.passesFilters(cs, cctxt.context, prep, filters, 0)) {
1136
- if (filter.passesDesignations(cds) || filter.passes(cc.code)) {
1675
+ if (filter.passesDesignations(cds, searchLang, cs.defLang()) || filter.passes(cc.code)) {
1137
1676
  let ov = Extensions.readString(cc, 'http://hl7.org/fhir/StructureDefinition/itemWeight');
1138
1677
  if (!ov) {
1139
1678
  ov = await cs.itemWeight(cctxt.context);
1140
1679
  }
1141
- this.excludeCode(cs, await cs.system(), await cs.version(), cc.code, expansion, valueSets, vsSrc.url);
1680
+ this.excludeCode(cs, await cs.system(), await cs.version(), cc.code, expansion, valueSets);
1142
1681
  }
1143
1682
  }
1144
1683
  }
@@ -1146,7 +1685,7 @@ class ValueSetExpander {
1146
1685
 
1147
1686
  if (cset.filter) {
1148
1687
  this.worker.opContext.log('prep filters');
1149
- const prep = await cs.getPrepContext(true);
1688
+ const prep = await cs.getPrepContext(true, this.params.requestedLanguages());
1150
1689
  if (!filter.isNull) {
1151
1690
  await cs.searchFilter(prep, filter, true);
1152
1691
  }
@@ -1157,30 +1696,42 @@ class ValueSetExpander {
1157
1696
  Extensions.addString(expansion, "http://hl7.org/fhir/StructureDefinition/valueset-unclosed-reason", "The code System '" + cs.system() + "' has a grammar and so has infinite members. This extension is based on " + cs.specialEnumeration());
1158
1697
  }
1159
1698
 
1699
+ for (const [i, fc] of cset.filter.entries()) {
1700
+ Extensions.checkNoModifiers(fc, 'ValueSetExpander.processCodes', path + '.filter[' + i + ']', vsSrc.vurl);
1701
+ }
1702
+ // as for an include: filters on properties that supplements define are evaluated from them
1703
+ const { base: baseFilters, supplement: suppFilters } = await splitFilters(cs, cset.filter);
1160
1704
  let first = true;
1161
- for (let fc of cset.filter) {
1705
+ for (let fc of baseFilters) {
1162
1706
  await this.worker.checkAndYield('processCodes#4a');
1163
- Extensions.checkNoModifiers(fc, 'ValueSetExpander.processCodes', 'filter', vsSrc.vurl);
1164
1707
  await cs.filter(prep, first, fc.property, fc.op, fc.value, vsSrc.isCached ? fc : null);
1165
1708
  first = false;
1166
1709
  }
1167
1710
 
1168
1711
  this.worker.opContext.log('iterate filters');
1169
- const fset = await cs.executeFilters(prep);
1170
- if (await cs.filtersNotClosed(prep)) {
1171
- notClosed.value = true;
1172
- }
1173
- //let count = 0;
1174
- while (await cs.filterMore(prep, fset[0])) {
1175
- await this.worker.checkAndYield('processCodes#5');
1176
- const c = await cs.filterConcept(prep, fset[0]);
1177
- const ok = (!this.params.activeOnly || !await cs.isInactive(c)) && (await this.passesFilters(cs, c, prep, fset, 1));
1178
- if (ok) {
1179
- //count++;
1180
- if (this.passesImports(valueSets, await cs.system(), await cs.code(c), 0)) {
1181
- this.excludeCode(cs, await cs.system(), await cs.version(), await cs.code(c), expansion, null, vsSrc.url);
1712
+ const excludeFound = async (c) => {
1713
+ if (this.passesImports(valueSets, await cs.system(), await cs.code(c), 0)) {
1714
+ this.excludeCode(cs, await cs.system(), await cs.version(), await cs.code(c), expansion, null);
1715
+ }
1716
+ };
1717
+ if (baseFilters.length > 0) {
1718
+ const fset = await cs.executeFilters(prep);
1719
+ if (await cs.filtersNotClosed(prep)) {
1720
+ notClosed.value = true;
1721
+ }
1722
+ while (await cs.filterMore(prep, fset[0])) {
1723
+ await this.worker.checkAndYield('processCodes#5');
1724
+ const c = await cs.filterConcept(prep, fset[0]);
1725
+ const ok = (!this.params.activeOnly || !await cs.isInactive(c)) && (await this.passesFilters(cs, c, prep, fset, 1))
1726
+ && (await this.#passesSupplementFilters(cs, c, suppFilters));
1727
+ if (ok) {
1728
+ await excludeFound(c);
1182
1729
  }
1183
1730
  }
1731
+ } else {
1732
+ for (const c of await this.#supplementFilterConcepts(cs, suppFilters, 'ValueSet.compose.exclude')) {
1733
+ await excludeFound(c);
1734
+ }
1184
1735
  }
1185
1736
  this.worker.opContext.log('iterate filters finished');
1186
1737
  }
@@ -1212,6 +1763,11 @@ class ValueSetExpander {
1212
1763
  return 0;
1213
1764
  }
1214
1765
  visited.add(code);
1766
+ if (this.#capReached(cs)) {
1767
+ // the code system's cap is reached: nothing more of it can be added, so don't walk it further
1768
+ this.#noteCapCut(cs);
1769
+ return 0;
1770
+ }
1215
1771
 
1216
1772
  if (expansion) {
1217
1773
  const vs = this.canonical(await cs.system(), await cs.version());
@@ -1223,17 +1779,28 @@ class ValueSetExpander {
1223
1779
  }
1224
1780
 
1225
1781
  let n = null;
1226
- if ((!this.params.excludeNotForUI || !await cs.isAbstract(context)) && (!this.params.activeOnly || !await cs.isInactive(context))) {
1782
+ const walked = (!this.params.excludeNotForUI || !await cs.isAbstract(context)) && (!this.params.activeOnly || !await cs.isInactive(context))
1783
+ ? await this.#beforePage(cs, await cs.system(), await cs.version(), code, await cs.isInactive(context), excludeInactive, imports) : null;
1784
+ if (walked !== null) {
1785
+ // before the page: counted (or not), and its descendants are walked below
1786
+ if (walked) {
1787
+ result++;
1788
+ }
1789
+ n = parent;
1790
+ } else if ((!this.params.excludeNotForUI || !await cs.isAbstract(context)) && (!this.params.activeOnly || !await cs.isInactive(context))) {
1227
1791
  const cds = new Designations(this.worker.i18n.languageDefinitions);
1228
1792
  let t;
1229
- if (this.noDetails) {
1793
+ if (this.noDetails || this.#outsidePage()) {
1230
1794
  t = await this.includeCode(cs, null, await cs.system(), await cs.version(), code, await cs.isAbstract(context), await cs.isInactive(context), null, null, null,
1231
1795
  null, null, expansion, imports, null, null, null, null, excludeInactive, srcUrl);
1232
1796
 
1797
+ } else if (this.#defers(cs, code)) {
1798
+ t = await this.includeCode(cs, parent, await cs.system(), await cs.version(), code, await cs.isAbstract(context), await cs.isInactive(context), null, null, null,
1799
+ null, null, expansion, imports, null, null, null, null, excludeInactive, srcUrl, () => this.#readDetails(cs, context));
1233
1800
  } else {
1234
1801
  await this.listDisplaysFromProvider(cds, cs, context);
1235
1802
  t = await this.includeCode(cs, parent, await cs.system(), await cs.version(), code, await cs.isAbstract(context), await cs.isInactive(context), await cs.isDeprecated(context), await cs.getStatus(context), cds, await cs.definition(context),
1236
- await cs.itemWeight(context), expansion, imports, await cs.extensions(context), null, await cs.properties(context), null, excludeInactive, srcUrl);
1803
+ await cs.itemWeight(context), expansion, imports, await cs.extensions(context), null, await this.#conceptProperties(cs, context), null, excludeInactive, srcUrl);
1237
1804
  }
1238
1805
  if (t != null) {
1239
1806
  result++;
@@ -1241,6 +1808,7 @@ class ValueSetExpander {
1241
1808
  if (n == null) {
1242
1809
  n = t;
1243
1810
  }
1811
+ this.#finishIfPageFilled();
1244
1812
  } else {
1245
1813
  n = parent;
1246
1814
  }
@@ -1286,7 +1854,7 @@ class ValueSetExpander {
1286
1854
  const cds = new Designations(this.worker.i18n.languageDefinitions);
1287
1855
  await this.listDisplaysFromProvider(cds, cs, context);
1288
1856
  await this.worker.checkAndYield('processCodeAndDescendants#2');
1289
- this.excludeCode(cs, await cs.system(), await cs.version(), code, expansion, imports, srcUrl);
1857
+ this.excludeCode(cs, await cs.system(), await cs.version(), code, expansion, imports);
1290
1858
  }
1291
1859
 
1292
1860
  const iter = await cs.iterator(context);
@@ -1302,21 +1870,32 @@ class ValueSetExpander {
1302
1870
  this.worker.opContext.log('compose #1');
1303
1871
 
1304
1872
  this.doingVersion = false;
1873
+ // count=0 can be answered by counting listed codes when that's all the value set is
1874
+ this.composeOnlyListsCodes = (source.jsonObj.compose.include || []).every(inc => inc.system && inc.concept && inc.concept.length > 0
1875
+ && !(inc.filter && inc.filter.length) && !(inc.valueSet && inc.valueSet.length));
1876
+ // A page (count, and offset - 0 when not given) is the only part of the expansion built: the
1877
+ // walk passes the codes before it keeping only their keys (#beforePage), so a page can be
1878
+ // further in than the limit on what is built, and after it only counts codes, for the total
1879
+ // (#finishIfPageFilled). With exclusions too: they are all processed before any include, so a
1880
+ // code's place in the list is final once it is added. Not with a text filter, which is matched
1881
+ // against the codes' designations, so each must be read. (A hierarchy is
1882
+ // only shown when the whole expansion fits in count: then every code is on the page)
1883
+ this.paging = this.count > 0 && this.offset >= 0 && filter.isNull;
1305
1884
  const ts = new Map();
1306
- for (const c of source.jsonObj.compose.include || []) {
1885
+ for (const [i, c] of (source.jsonObj.compose.include || []).entries()) {
1307
1886
  await this.worker.checkAndYield('handleCompose#2');
1308
- await this.checkSource(c, expansion, filter, source.url, ts, vsInfo, source);
1887
+ await this.checkSource(c, expansion, filter, source.url, ts, vsInfo, source, 'ValueSet.compose.include[' + i + ']');
1309
1888
  }
1310
- for (const c of source.jsonObj.compose.exclude || []) {
1889
+ for (const [i, c] of (source.jsonObj.compose.exclude || []).entries()) {
1311
1890
  await this.worker.checkAndYield('handleCompose#3');
1312
1891
  this.hasExclusions = true;
1313
- await this.checkSource(c, expansion, filter, source.url, ts, null, source);
1892
+ await this.checkSource(c, expansion, filter, source.url, ts, null, source, 'ValueSet.compose.exclude[' + i + ']');
1314
1893
  }
1315
1894
 
1316
1895
  this.worker.opContext.log('compose #2');
1317
1896
 
1318
- if (vsInfo.handleByCS) {
1319
- await this.processCodes("ValueSet.compose", source, source.jsonObj.compose, filter, expansion, this.excludeInactives(source), notClosed, vsInfo);
1897
+ if (vsInfo.handleByCS && await this.processCodes("ValueSet.compose", source, source.jsonObj.compose, filter, expansion, this.excludeInactives(source), notClosed, vsInfo)) {
1898
+ // the code system provider selected the codes (processSelection)
1320
1899
  } else {
1321
1900
  this.checkForExclusionVersionSpecialCase(source, expansion);
1322
1901
 
@@ -1342,6 +1921,71 @@ class ValueSetExpander {
1342
1921
  }
1343
1922
  }
1344
1923
 
1924
+ /**
1925
+ * Whether a value set's stored expansion is the answer. When it has a compose, the compose is
1926
+ * what's expanded - a stored expansion may have been made from other code system versions, or
1927
+ * just be wrong, and there's no telling without expanding the compose - unless a code system the
1928
+ * compose includes isn't available here (not loaded, or not the version the include names): then
1929
+ * the compose can't be expanded, and the stored expansion is the best answer there is
1930
+ * @param {Object} vs - the value set (JSON)
1931
+ * @returns {Promise<boolean>}
1932
+ */
1933
+ async useStoredExpansion(vs) {
1934
+ if (!vs.compose) {
1935
+ return true;
1936
+ }
1937
+ // finding the code systems records the version parameters it applied; this is only a look
1938
+ const found = this.worker.foundParameters.length;
1939
+ try {
1940
+ for (const inc of [...(vs.compose.include || []), ...(vs.compose.exclude || [])]) {
1941
+ if (inc.system) {
1942
+ const cs = await this.worker.findCodeSystem(inc.system, inc.version || '', this.params, ['complete', 'fragment'], null, true);
1943
+ if (!cs) {
1944
+ this.worker.opContext.log(`${inc.system}${inc.version ? '|' + inc.version : ''} isn't available: using the stored expansion of ${vs.url}`);
1945
+ return true;
1946
+ }
1947
+ }
1948
+ }
1949
+ } catch (error) {
1950
+ return false; // expanding the compose will report the problem properly
1951
+ } finally {
1952
+ this.worker.foundParameters.length = found;
1953
+ }
1954
+ return false;
1955
+ }
1956
+
1957
+ /**
1958
+ * How a value set is named in a message: by the url (|version) given, else as the value set this
1959
+ * expander is expanding - its url|version, else its id - else null, for one with neither (a value
1960
+ * set passed inline in the request)
1961
+ * @param {string} [name] - the url (|version) the caller has, if any
1962
+ * @returns {string}
1963
+ */
1964
+ vsLabel(name) {
1965
+ if (name && !String(name).startsWith('undefined')) { // (a vurl with no url is 'undefined|version')
1966
+ return name;
1967
+ }
1968
+ const json = this.valueSet ? (this.valueSet.jsonObj || this.valueSet) : null;
1969
+ if (json && json.url) {
1970
+ return json.version ? json.url + '|' + json.version : json.url;
1971
+ }
1972
+ if (json && json.id) {
1973
+ return json.id;
1974
+ }
1975
+ return null;
1976
+ }
1977
+
1978
+ /**
1979
+ * The VALUESET_TOO_COSTLY message for a value set (see vsLabel), or for one passed in the
1980
+ * request with no url or id, which it can't name, VALUESET_TOO_COSTLY_REQUEST - the message
1981
+ * id reported is VALUESET_TOO_COSTLY either way
1982
+ */
1983
+ tooCostlyMessage(name, limit) {
1984
+ const label = this.vsLabel(name);
1985
+ return label ? this.worker.i18n.translate('VALUESET_TOO_COSTLY', this.params.httpLanguages, [label, limit])
1986
+ : this.worker.i18n.translate('VALUESET_TOO_COSTLY_REQUEST', this.params.httpLanguages, [limit]);
1987
+ }
1988
+
1345
1989
  excludeInactives(source) {
1346
1990
  return source.jsonObj.compose && source.jsonObj.compose.inactive != undefined && !source.jsonObj.compose.inactive;
1347
1991
  }
@@ -1388,7 +2032,11 @@ class ValueSetExpander {
1388
2032
  }
1389
2033
 
1390
2034
  if (result.expansion) {
1391
- return result; // just return the expansion
2035
+ if (await this.useStoredExpansion(source.jsonObj)) {
2036
+ return result; // just return the expansion
2037
+ }
2038
+ // otherwise it's expanded from the compose, as if it had no expansion
2039
+ result.expansion = undefined;
1392
2040
  }
1393
2041
  if (!source.jsonObj.compose) {
1394
2042
  throw new Issue('error', 'invalid', null, 'VALUESET_NO_COMPOSE',
@@ -1482,7 +2130,7 @@ class ValueSetExpander {
1482
2130
 
1483
2131
  let vsInfo = this.scanValueSet(source.jsonObj.compose);
1484
2132
  try {
1485
- if (source.jsonObj.compose && Extensions.checkNoModifiers(source.jsonObj.compose, 'ValueSetExpander.Expand', 'compose', source.vurl)
2133
+ if (source.jsonObj.compose && Extensions.checkNoModifiers(source.jsonObj.compose, 'ValueSetExpander.Expand', 'ValueSet.compose', source.vurl)
1486
2134
  && this.worker.checkNoLockedDate(source.url, source.jsonObj.compose)) {
1487
2135
  await this.handleCompose(source, filter, exp, notClosed, vsInfo);
1488
2136
  }
@@ -1517,6 +2165,10 @@ class ValueSetExpander {
1517
2165
  for (const v of ts) {
1518
2166
  this.addParamUri(exp, 'used-supplement', v);
1519
2167
  }
2168
+ if (this.capCut) {
2169
+ this.addExpansionIssue(result, exp, new Issue('warning', 'business-rule', null, 'VALUESET_EXPANSION_CODESYSTEM_LIMITED',
2170
+ this.worker.i18n.translate('VALUESET_EXPANSION_CODESYSTEM_LIMITED', this.params.httpLanguages, [this.capCut.system, this.capCut.limit]), 'business-rule'));
2171
+ }
1520
2172
 
1521
2173
  this.worker.opContext.log('finish up');
1522
2174
 
@@ -1556,9 +2208,10 @@ class ValueSetExpander {
1556
2208
  }
1557
2209
  }
1558
2210
 
1559
- if (this.offset + this.count < 0 && this.fullList.length > this.limit) {
1560
- this.log.log('Operation took too long @ expand (' + this.constructor.name + ')');
1561
- throw new Issue("error", "too-costly", null, 'VALUESET_TOO_COSTLY', this.worker.i18n.translate('VALUESET_TOO_COSTLY', this.params.httpLanguages, [source.vurl, '>' + this.limit]), null, 422).withDiagnostics(this.worker.opContext.diagnostics());
2211
+ if (this.count < 0 && this.limitCount > 0 && this.fullList.length > this.limitCount) {
2212
+ // a backstop for an unpaged expansion: every path that adds a code checks the limit first
2213
+ // (#checkLimit). Not for count=0, which only counts, as far as the walk limit allows
2214
+ throw new Issue("error", "too-costly", null, 'VALUESET_TOO_COSTLY', this.tooCostlyMessage(source.vurl, '>' + this.limitCount), null, 422).withDiagnostics(this.worker.opContext.diagnostics());
1562
2215
  } else {
1563
2216
  let t = 0;
1564
2217
  let o = 0;
@@ -1567,7 +2220,8 @@ class ValueSetExpander {
1567
2220
  const c = list[i];
1568
2221
  if (this.map.has(this.keyC(c))) {
1569
2222
  o++;
1570
- if ((vsInfo.csDoOffset) || (o > this.offset && (this.count < 0 || t < this.count))) {
2223
+ // (the provider paged it, or the walk passed the codes before the page: all that's here is the page)
2224
+ if ((vsInfo.csDoOffset) || (o > (this.paging ? 0 : this.offset) && (this.count < 0 || t < this.count))) {
1571
2225
  t++;
1572
2226
  if (!exp.contains) {
1573
2227
  exp.contains = [];
@@ -1591,6 +2245,10 @@ class ValueSetExpander {
1591
2245
  }
1592
2246
  }
1593
2247
 
2248
+ // declare only the properties the returned codes use: codes outside the page (or a count=0
2249
+ // expansion) declared theirs as they were added, and a declaration nothing uses is noise
2250
+ this.#pruneUnusedPropertyDeclarations(exp);
2251
+
1594
2252
  if (result.expansion.contains && result.expansion.contains.length > 0) {
1595
2253
  let hasChildren = false;
1596
2254
  for (let c of result.expansion.contains) {
@@ -1729,11 +2387,31 @@ class ValueSetExpander {
1729
2387
  }
1730
2388
 
1731
2389
 
2390
+ #pruneUnusedPropertyDeclarations(exp) {
2391
+ if (!exp.property || exp.property.length === 0) {
2392
+ return;
2393
+ }
2394
+ const used = new Set();
2395
+ const walk = (list) => {
2396
+ for (const c of list || []) {
2397
+ for (const p of c.property || []) {
2398
+ used.add(p.code);
2399
+ }
2400
+ walk(c.contains);
2401
+ }
2402
+ };
2403
+ walk(exp.contains);
2404
+ exp.property = exp.property.filter(pd => used.has(pd.code));
2405
+ if (exp.property.length === 0) {
2406
+ delete exp.property;
2407
+ }
2408
+ }
2409
+
1732
2410
  hasParam(exp, name, value) {
1733
2411
  return (exp.parameter || []).find((ex => ex.name == name && getValuePrimitive(ex) == value));
1734
2412
  }
1735
2413
 
1736
- isExcluded(system, version, code) {
2414
+ async isExcluded(system, version, code) {
1737
2415
  if (this.excludedSystems.has(system)) {
1738
2416
  return true;
1739
2417
  }
@@ -1742,7 +2420,27 @@ class ValueSetExpander {
1742
2420
  return true;
1743
2421
  }
1744
2422
  key = (this.doingVersion && !this.params.versionsMatch ? system+'|'+version : system)+'#'+code;
1745
- return this.excluded.has(key);
2423
+ if (this.excluded.has(key)) {
2424
+ return true;
2425
+ }
2426
+ for (const ep of this.excludedProviders) {
2427
+ if (ep.system === system && await this.#knownTo(ep.cs, code)) {
2428
+ return true;
2429
+ }
2430
+ }
2431
+ return false;
2432
+ }
2433
+
2434
+ /**
2435
+ * @returns {Promise<boolean>} whether the code exists in that provider's code system (version)
2436
+ */
2437
+ async #knownTo(cs, code) {
2438
+ try {
2439
+ const loc = await cs.locate(code);
2440
+ return !!(loc && loc.context);
2441
+ } catch (e) {
2442
+ return false;
2443
+ }
1746
2444
  }
1747
2445
 
1748
2446
  keyS(system, version, code) {
@@ -1848,14 +2546,106 @@ class ValueSetExpander {
1848
2546
 
1849
2547
  /**
1850
2548
  * True when the requested page (offset + count) has been filled, so an
1851
- * iteration that is not counting a total can stop. Exclusions are handled
1852
- * after the fact and can remove codes already collected, so a value set
1853
- * with any exclusion never stops early.
2549
+ * iteration that is not counting a total can stop. (Exclusions are processed
2550
+ * before any include, so they never remove a code already collected.)
1854
2551
  * @returns {boolean}
1855
2552
  */
1856
2553
  pageFilled() {
1857
- return this.count > 0 && this.offset > -1 && !this.hasExclusions
1858
- && this.fullList.length >= this.count + this.offset;
2554
+ return this.count > 0 && this.offset > -1
2555
+ && this.walked + this.fullList.length >= this.count + this.offset;
2556
+ }
2557
+
2558
+ /**
2559
+ * Whether the next code added falls outside the page asked for (offset, count) on an expansion
2560
+ * that builds every code it includes - one with an exclude, or a text filter, which don't walk
2561
+ * past the codes before the page (see handleCompose): it is counted and kept for deduplication,
2562
+ * but never returned, so there's no point reading its displays, definition and properties.
2563
+ * Excludes don't move a code once it is added: every exclude is processed before the includes,
2564
+ * and includeCode refuses an excluded code before adding it. Not when paging (the walk counts
2565
+ * the codes outside the page without adding them), nor when the provider paged the compose
2566
+ * (every code it returns is on the page)
2567
+ * @returns {boolean}
2568
+ */
2569
+ #outsidePage() {
2570
+ if (this.paging || this.providerPaged || !(this.offset > 0 || this.count > -1)) {
2571
+ return false;
2572
+ }
2573
+ const start = Math.max(this.offset, 0);
2574
+ const i = this.fullList.length;
2575
+ return i < start || (this.count > -1 && i >= start + this.count);
2576
+ }
2577
+
2578
+ /**
2579
+ * Whether this code's details are left to be read later (deferDetails). Not when a supplement
2580
+ * that applies to every value set (a global one, such as a language pack) has the code: that
2581
+ * supplement is reported as used (used-supplement) only when it gives the code's display or a
2582
+ * designation, and this expansion's parameters are made before the importing expansion reads the
2583
+ * details. (A supplement the value set names is reported for every code, read or not)
2584
+ * @param {string} [code] - the code; without one, whether no global supplement could have it
2585
+ */
2586
+ #defers(cs, code = null) {
2587
+ if (!this.deferDetails || this.noDetails || cs == null) {
2588
+ return false;
2589
+ }
2590
+ if (cs.listSupplements(true).length === cs.listSupplements(false).length) {
2591
+ return true;
2592
+ }
2593
+ return code != null && Array.isArray(cs.supplements)
2594
+ && !cs.supplements.some(sp => sp.isGlobalSupplement() && sp.getConceptByCode(code));
2595
+ }
2596
+
2597
+ /**
2598
+ * A concept's details, read as includeCode's callers read them: displays and designations from
2599
+ * the provider, then those the value set lists for the code (cc, an include's concept), and the
2600
+ * concept's status, definition, extensions and properties
2601
+ */
2602
+ async #readDetails(cs, context, cc = null, vsSrc = null) {
2603
+ const displays = new Designations(this.worker.i18n.languageDefinitions);
2604
+ await this.listDisplaysFromProvider(displays, cs, context);
2605
+ if (cc) {
2606
+ this.listDisplaysFromIncludeConcept(displays, cc, vsSrc);
2607
+ }
2608
+ return {
2609
+ displays, deprecated: await cs.isDeprecated(context), status: await cs.getStatus(context), definition: await cs.definition(context),
2610
+ csExtList: await cs.extensions(context), vsExtList: cc ? cc.extension : null, csProps: await this.#conceptProperties(cs, context)
2611
+ };
2612
+ }
2613
+
2614
+ /**
2615
+ * An imported code that this expansion returns: if its details were left to be read
2616
+ * (deferDetails), read and add them now - by the expander that added the code, into its own
2617
+ * expansion, as it would have when it added it. Its property declarations are then on that
2618
+ * expansion, which is added to sources (see importValueSet)
2619
+ */
2620
+ async #buildDeferred(n, sources) {
2621
+ const d = DEFERRED_DETAILS.get(n);
2622
+ if (!d) {
2623
+ return;
2624
+ }
2625
+ DEFERRED_DETAILS.delete(n);
2626
+ const r = await d.read();
2627
+ d.expander.#addDetails(n, d.cs, n.code, r.status, r.deprecated, r.displays, r.definition, d.expansion, r.csExtList, r.vsExtList, r.csProps);
2628
+ sources.add(d.expansion);
2629
+ }
2630
+
2631
+ /**
2632
+ * Whether a value set this expansion imports, for its codes, can be expanded leaving their
2633
+ * details to be read later (deferDetails): when this expansion returns only some of them - a
2634
+ * page, a count, or none at all (count=0) - or is itself expanded so. Not when the expansion is
2635
+ * sorted by display or property, which needs every code's details; nor for a $compare
2636
+ * (noDetails), which takes the imported codes as they come
2637
+ */
2638
+ #importDefers() {
2639
+ return !this.noDetails && !this.#sortsByDetails()
2640
+ && (this.deferDetails || this.paging || this.count > -1 || this.offset > 0);
2641
+ }
2642
+
2643
+ /**
2644
+ * Whether the expansion is sorted by something read with a code's details (its display, a property)
2645
+ */
2646
+ #sortsByDetails() {
2647
+ const sort = (this.params.sort || '').replace(/^-/, '');
2648
+ return sort === 'display' || sort.startsWith('prop:');
1859
2649
  }
1860
2650
 
1861
2651
  addToTotal(t = 1) {
@@ -1877,11 +2667,81 @@ class ValueSetExpander {
1877
2667
  this.totalStatus = 'off';
1878
2668
  }
1879
2669
 
2670
+ /**
2671
+ * The codes an include that lists them would add to the expansion, without building them:
2672
+ * the same checks, in the same order, as the concept loop and includeCode make (located,
2673
+ * active if asked, imports, exclusions, the code system's own cap on how many codes it
2674
+ * contributes, not already in the expansion). Nothing is changed - the cap's counter is
2675
+ * simulated - so the include can still be walked afterwards
2676
+ * @returns {Promise<Set<string>>} the keys (see keyS) of the codes that would be added
2677
+ */
2678
+ async #countListedCodes(cs, cset, valueSets, filters, prep, excludeInactive) {
2679
+ const keys = new Set();
2680
+ const expandLimit = cs.expandLimitation();
2681
+ let capCount = expandLimit > 0 && this.csCounter.has(cs.system()) ? this.csCounter.get(cs.system()).count : 0;
2682
+ // each listed code is located next: a provider may read them together, and hand back what it
2683
+ // read; counting builds nothing, so designations aren't wanted
2684
+ const prefetched = await cs.prefetchCodes(cset.concept.map(cc => cc.code), false);
2685
+ for (const cc of cset.concept) {
2686
+ await this.worker.checkAndYield('countListedCodes');
2687
+ const cctxt = await cs.locate(cc.code, prefetched);
2688
+ if (!cctxt || !cctxt.context) {
2689
+ continue;
2690
+ }
2691
+ const inactive = await cs.isInactive(cctxt.context);
2692
+ if ((this.params.activeOnly && inactive) || !(await this.passesFilters(cs, cctxt.context, prep, filters, 0))) {
2693
+ continue;
2694
+ }
2695
+ const code = await cs.expansionCode(cctxt.context, cc.code);
2696
+ if (!this.passesImports(valueSets, cs.system(), code, 0) || (inactive && excludeInactive)
2697
+ || await this.isExcluded(cs.system(), cs.version(), code)) {
2698
+ continue;
2699
+ }
2700
+ if (expandLimit > 0 && ++capCount > expandLimit) {
2701
+ continue;
2702
+ }
2703
+ const key = this.keyS(cs.system(), cs.version(), code);
2704
+ if (!this.map.has(key) && !this.countedKeys.has(key) && !this.walkedKeys.has(key)) {
2705
+ keys.add(key);
2706
+ }
2707
+ }
2708
+ return keys;
2709
+ }
2710
+
2711
+ /**
2712
+ * Whether adding this many codes would end in VALUESET_TOO_COSTLY in includeCode: over the
2713
+ * limit, and not a page that the limit leaves room for
2714
+ */
2715
+ #listedCodesOverLimit(n) {
2716
+ // (limitCount is only set by expand(): an expander used for something else - $related - has no limit)
2717
+ if (!(this.limitCount > 0) || this.fullList.length + n <= this.limitCount) {
2718
+ return false;
2719
+ }
2720
+ return !this.#pageFits();
2721
+ }
2722
+
2723
+ /**
2724
+ * The properties of a concept that this expansion might report: providers that compute their
2725
+ * properties (from a database, from a lookup) are told which ones are wanted - the requested
2726
+ * names, and those the code system always reports - so that an expansion that asks for none
2727
+ * does not compute any
2728
+ */
2729
+ async #conceptProperties(cs, context) {
2730
+ const names = [...this.params.properties];
2731
+ for (const pn of cs.mandatoryExpansionProperties()) {
2732
+ if (!names.includes(pn)) {
2733
+ names.push(pn);
2734
+ }
2735
+ }
2736
+ return await cs.properties(context, names);
2737
+ }
2738
+
1880
2739
  getPropUrl(cs, pn, cp) {
1881
2740
  if (cp.definition?.uri) {
1882
2741
  return cp.definition.uri;
1883
2742
  }
1884
- for (let p of cs.propertyDefinitions()) {
2743
+ // a provider need not declare its properties (propertyDefinitions() is null by default)
2744
+ for (let p of cs.propertyDefinitions() || []) {
1885
2745
  if (pn == p.code) {
1886
2746
  return p.uri;
1887
2747
  }
@@ -1913,13 +2773,20 @@ class ValueSetExpander {
1913
2773
  }
1914
2774
  if (simple && result.csset.size == 1) {
1915
2775
  result.isSimple = true;
2776
+ const first = (compose.include || [])[0] || (compose.exclude || [])[0];
2777
+ if (first) {
2778
+ result.system = first.system;
2779
+ result.version = first.version;
2780
+ }
1916
2781
  }
1917
2782
  return result;
1918
2783
  }
1919
2784
 
1920
2785
  isSimpleSelect(inc, set) {
1921
2786
  set.add(inc.system+"|"+inc.version);
1922
- return !inc.valueset || inc.valueset.length == 0;
2787
+ // (the element is valueSet: the lowercase spelling never matched, which was harmless only
2788
+ // while processCodes did nothing)
2789
+ return !!inc.system && (!inc.valueSet || inc.valueSet.length == 0);
1923
2790
  }
1924
2791
 
1925
2792
  excludeFilterList(exc) {
@@ -1992,6 +2859,8 @@ class ValueSetExpander {
1992
2859
  class ExpandWorker extends TerminologyWorker {
1993
2860
  internalLimit = INTERNAL_DEFAULT_LIMIT;
1994
2861
  externalLimit = EXTERNAL_DEFAULT_LIMIT;
2862
+ // how far a paged expansion may walk to reach its page (null: WALK_LIMIT_FACTOR x internalLimit)
2863
+ walkLimit = null;
1995
2864
 
1996
2865
 
1997
2866
  /**
@@ -2000,12 +2869,17 @@ class ExpandWorker extends TerminologyWorker {
2000
2869
  * @param {Provider} provider - Provider for code systems and resources
2001
2870
  * @param {LanguageDefinitions} languages - Language definitions
2002
2871
  * @param {I18nSupport} i18n - Internationalization support
2872
+ * @param {number} [internalLimit] - the most codes an expansion builds
2873
+ * @param {number} [externalLimit] - the most codes a response holds
2874
+ * @param {number} [walkLimit] - the furthest a paged expansion walks to reach its page (offset +
2875
+ * count); by default WALK_LIMIT_FACTOR x internalLimit
2003
2876
  */
2004
2877
  constructor(opContext, log, provider, languages,
2005
- i18n, internalLimit = INTERNAL_DEFAULT_LIMIT, externalLimit = EXTERNAL_DEFAULT_LIMIT) {
2878
+ i18n, internalLimit = INTERNAL_DEFAULT_LIMIT, externalLimit = EXTERNAL_DEFAULT_LIMIT, walkLimit = null) {
2006
2879
  super(opContext, log, provider, languages, i18n);
2007
2880
  this.externalLimit = externalLimit;
2008
2881
  this.internalLimit = internalLimit;
2882
+ this.walkLimit = walkLimit;
2009
2883
  }
2010
2884
 
2011
2885
  /**
@@ -2110,6 +2984,9 @@ class ExpandWorker extends TerminologyWorker {
2110
2984
 
2111
2985
  let txp = new TxParameters(this.opContext.i18n.languageDefinitions, this.opContext.i18n, false);
2112
2986
  txp.readParams(params);
2987
+ // set now, not only in performExpansion: finding the value set by url can depend on them
2988
+ // (an implicit value set is drawn from the code system version the request asks for)
2989
+ this.params = txp;
2113
2990
 
2114
2991
  // If no valueSet yet, try to find by url
2115
2992
  if (!valueSet) {
@@ -2174,6 +3051,9 @@ class ExpandWorker extends TerminologyWorker {
2174
3051
 
2175
3052
  let txp = new TxParameters(this.opContext.i18n.languageDefinitions, this.opContext.i18n, false);
2176
3053
  txp.readParams(params);
3054
+ // set now, not only in performExpansion: finding the value set by url can depend on them
3055
+ // (an implicit value set is drawn from the code system version the request asks for)
3056
+ this.params = txp;
2177
3057
 
2178
3058
  // Perform the expansion
2179
3059
  return await this.doExpand(valueSet, txp, logExtraOutput);
@@ -2222,6 +3102,9 @@ class ExpandWorker extends TerminologyWorker {
2222
3102
 
2223
3103
  let txp = new TxParameters(this.opContext.i18n.languageDefinitions, this.opContext.i18n, false);
2224
3104
  txp.readParams(params);
3105
+ // set now, not only in performExpansion: finding the value set by url can depend on them
3106
+ // (an implicit value set is drawn from the code system version the request asks for)
3107
+ this.params = txp;
2225
3108
 
2226
3109
  // Perform the expansion
2227
3110
  const result = await this.doExpand(valueSet, txp, logExtraOutput);
@@ -2247,7 +3130,9 @@ class ExpandWorker extends TerminologyWorker {
2247
3130
  // Compute cache key (only if caching is available and not debugging)
2248
3131
  let cacheKey = null;
2249
3132
  if (expansionCache && (CACHE_WHEN_DEBUGGING || !this.opContext.debugging)) {
2250
- cacheKey = expansionCache.computeKey(valueSet, params, this.additionalResources);
3133
+ // (the response limit is in the key: a request with a lower x-too-costly-threshold mustn't
3134
+ // be answered with an expansion made under a higher one)
3135
+ cacheKey = expansionCache.computeKey(valueSet, params, this.additionalResources, this.externalLimit);
2251
3136
 
2252
3137
  // Check for cached expansion
2253
3138
  const cached = expansionCache.get(cacheKey);
@@ -2330,9 +3215,8 @@ module.exports = {
2330
3215
  ImportedValueSet,
2331
3216
  ValueSetFilterContext,
2332
3217
  EmptyFilterContext,
2333
- EXTERNAL_DEFAULT_LIMIT,
2334
3218
  INTERNAL_DEFAULT_LIMIT,
2335
- EXTERNAL_TEST_DEFAULT_LIMIT,
2336
- TotalStatus,
2337
- EXPANSION_DEAD_TIME_SECS
3219
+ WALK_LIMIT_FACTOR,
3220
+ EXTERNAL_DEFAULT_LIMIT,
3221
+ TotalStatus
2338
3222
  };