fhirsmith 0.18.1 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (214) hide show
  1. package/ACCESSIBILITY.md +119 -0
  2. package/CHANGELOG.md +190 -9
  3. package/CONFIGURATION.md +305 -0
  4. package/LICENSE +29 -0
  5. package/README.md +29 -122
  6. package/cluster.js +1220 -0
  7. package/cluster.md +707 -0
  8. package/config-template.json +77 -8
  9. package/extension-tracker/extension-tracker-template.html +5 -8
  10. package/extension-tracker/extension-tracker.js +0 -2
  11. package/folder/folder.js +7 -6
  12. package/ips/analysis.js +560 -0
  13. package/ips/ips-template.html +5 -8
  14. package/ips/ips.js +331 -33
  15. package/ips/jurisdictions.js +65 -0
  16. package/ips/nationality.js +120 -0
  17. package/ips/pipeline.js +131 -17
  18. package/ips/presenter.js +1 -1
  19. package/ips/profile-editor.js +283 -4
  20. package/ips/readme.md +170 -20
  21. package/ips/render.js +769 -109
  22. package/ips/scripts.js +46 -0
  23. package/ips/store.js +123 -11
  24. package/ips/structure.js +247 -0
  25. package/ips/terminology.js +1 -1
  26. package/library/body-limit.js +22 -0
  27. package/library/cache-generation.js +15 -0
  28. package/library/cgroup-memory.js +133 -1
  29. package/library/cluster-counts.js +121 -0
  30. package/library/cluster-events.js +153 -0
  31. package/library/cluster-metrics.js +193 -0
  32. package/library/cluster-status.js +468 -0
  33. package/library/config-reader.js +96 -0
  34. package/library/daily-report.js +100 -2
  35. package/library/html-server.js +27 -1
  36. package/library/i18nsupport.js +23 -1
  37. package/library/keep-alive.js +18 -0
  38. package/library/languages.js +17 -3
  39. package/library/logger.js +31 -0
  40. package/library/memory-guard.js +428 -0
  41. package/library/native-memory.js +52 -0
  42. package/library/npm-audit.js +51 -25
  43. package/library/openapi-doc.js +7 -5
  44. package/library/package-manager.js +8 -0
  45. package/library/role.js +197 -0
  46. package/library/server-credentials.js +169 -0
  47. package/library/sqlite-utilities.js +59 -0
  48. package/library/ssrf-guard.js +100 -0
  49. package/library/static-response-stats.js +16 -0
  50. package/library/version-utilities.js +50 -0
  51. package/library/worker-count.js +192 -0
  52. package/nginx.md +30 -11
  53. package/npmprojector/indexer.js +18 -10
  54. package/package.json +11 -8
  55. package/packages/feed-check-page.js +117 -0
  56. package/packages/feed-check.js +279 -0
  57. package/packages/npm-check.js +57 -3
  58. package/packages/npm-signing.js +219 -0
  59. package/packages/openapi.yaml +207 -1
  60. package/packages/package-crawler.js +322 -41
  61. package/packages/packages-template.html +6 -8
  62. package/packages/packages.js +604 -57
  63. package/packages/readme.md +113 -1
  64. package/publisher/publisher-template.html +2 -5
  65. package/publisher/publisher.js +121 -110
  66. package/publisher/readme.md +75 -0
  67. package/registry/api.js +179 -1
  68. package/registry/crawler.js +86 -108
  69. package/registry/documentation/api.md +68 -2
  70. package/registry/documentation/configuration.md +34 -0
  71. package/registry/documentation/internals.md +4 -1
  72. package/registry/documentation/readme.md +3 -2
  73. package/registry/model.js +8 -0
  74. package/registry/openapi.yaml +84 -0
  75. package/registry/registry-template.html +5 -8
  76. package/registry/registry.js +58 -6
  77. package/root-bare-template.html +7 -4
  78. package/root-template.html +5 -8
  79. package/server.js +398 -198
  80. package/shl/shl.js +38 -37
  81. package/static/a11y/a11y-check.js +832 -0
  82. package/static/a11y/axe.min.js +12 -0
  83. package/static/assets/css/bootstrap-fhir.css +15 -15
  84. package/static/assets/images/flags/au.svg +18 -0
  85. package/static/assets/images/flags/unknown.svg +11 -0
  86. package/static/assets/images/flags/us.svg +63 -0
  87. package/static/assets/images/icon_audit.png +0 -0
  88. package/static/assets/images/icon_clinical.png +0 -0
  89. package/static/assets/images/icon_document.png +0 -0
  90. package/static/assets/images/icon_external.png +0 -0
  91. package/static/assets/images/icon_medication.png +0 -0
  92. package/static/assets/images/icon_organization.png +0 -0
  93. package/static/assets/images/icon_other.png +0 -0
  94. package/static/assets/images/icon_person.png +0 -0
  95. package/static/assets/images/icon_section.png +0 -0
  96. package/stats-db.js +45 -0
  97. package/stats.js +197 -10
  98. package/testing/testing-template.html +5 -8
  99. package/token/nginx_passport_setup.md +1 -1
  100. package/token/security_guide.md +1 -1
  101. package/token/token-template.html +2 -5
  102. package/token/token.js +40 -36
  103. package/translations/Messages.properties +19 -0
  104. package/tx/README.md +158 -4
  105. package/tx/closure/closure-store.js +126 -3
  106. package/tx/cm/cm-database.js +69 -57
  107. package/tx/cm/cm-package.js +50 -16
  108. package/tx/cs/cs-api.js +284 -19
  109. package/tx/cs/cs-api.md +41 -4
  110. package/tx/cs/cs-areacode.js +1 -1
  111. package/tx/cs/cs-country.js +17 -5
  112. package/tx/cs/cs-cpt.js +33 -16
  113. package/tx/cs/cs-cs.js +199 -24
  114. package/tx/cs/cs-currency.js +1 -1
  115. package/tx/cs/cs-icd11.js +21 -13
  116. package/tx/cs/cs-icf.js +22 -3
  117. package/tx/cs/cs-lang.js +2 -2
  118. package/tx/cs/cs-loinc.js +193 -67
  119. package/tx/cs/cs-mimetypes.js +12 -2
  120. package/tx/cs/cs-ndc.js +40 -25
  121. package/tx/cs/cs-omop.js +54 -29
  122. package/tx/cs/cs-rxnorm.js +300 -196
  123. package/tx/cs/cs-snomed.js +861 -95
  124. package/tx/cs/cs-ucum.js +14 -4
  125. package/tx/cs/cs-unii.js +18 -14
  126. package/tx/cs/cs-virtual.js +210 -0
  127. package/tx/ecosystem-resolver.js +413 -0
  128. package/tx/html/cluster-dash-metrics.liquid +26 -0
  129. package/tx/html/cluster-home-metrics.liquid +31 -0
  130. package/tx/html/cluster-metrics-script.liquid +251 -0
  131. package/tx/html/codesystem-operations.liquid +3 -3
  132. package/tx/html/ecl-panel.liquid +1 -1
  133. package/tx/html/external-metrics.liquid +17 -0
  134. package/tx/html/operations-form.liquid +17 -17
  135. package/tx/html/search-form.liquid +11 -11
  136. package/tx/html/status-line.liquid +58 -0
  137. package/tx/html/status-panel.liquid +140 -0
  138. package/tx/html/status-tasks.liquid +25 -0
  139. package/tx/html/tx-template.html +2 -5
  140. package/tx/importers/atc-to-fhir.js +17 -16
  141. package/tx/importers/export-icd11-codesystem.js +3 -2
  142. package/tx/importers/import-icd11.js +4 -3
  143. package/tx/importers/import-loinc.module.js +118 -56
  144. package/tx/importers/import-ndc.module.js +56 -44
  145. package/tx/importers/import-rxnorm.module.js +118 -79
  146. package/tx/importers/import-sct.module.js +125 -61
  147. package/tx/importers/import-unii.module.js +37 -36
  148. package/tx/importers/subset-loinc.module.js +49 -48
  149. package/tx/importers/subset-rxnorm.module.js +29 -28
  150. package/tx/importers/terminal.js +18 -0
  151. package/tx/importers/tx-import-base.js +13 -12
  152. package/tx/importers/tx-import-settings.js +5 -4
  153. package/tx/importers/tx-import.js +53 -52
  154. package/tx/library/canonical-resource.js +49 -0
  155. package/tx/library/codesystem.js +94 -5
  156. package/tx/library/designations.js +45 -2
  157. package/tx/library/extensions.js +13 -7
  158. package/tx/library/implicit-valuesets.js +66 -0
  159. package/tx/library/renderer.js +77 -21
  160. package/tx/library/sqlite-utilities.js +2 -0
  161. package/tx/library/tho-precedence.js +40 -0
  162. package/tx/library/ucum-parsers.js +11 -1
  163. package/tx/library/wait-for-cache.js +43 -0
  164. package/tx/library.js +321 -22
  165. package/tx/openapi-operations.js +6 -4
  166. package/tx/operation-context.js +460 -99
  167. package/tx/params.js +25 -0
  168. package/tx/problems.js +3 -5
  169. package/tx/provider.js +314 -67
  170. package/tx/resource-cache-sqlite.js +1050 -0
  171. package/tx/sct/documentation.md +45 -21
  172. package/tx/sct/ecl.js +81 -17
  173. package/tx/sct/editions.js +39 -0
  174. package/tx/sct/expressions.js +10 -1
  175. package/tx/sct/load-config.js +107 -0
  176. package/tx/sct/mmap.js +109 -0
  177. package/tx/sct/structures.js +175 -15
  178. package/tx/sct/text-index.js +5 -1
  179. package/tx/syndication/syndication-feed.js +2 -2
  180. package/tx/tx-html.js +44 -8
  181. package/tx/tx.fhir.org.yml +8 -8
  182. package/tx/tx.js +282 -80
  183. package/tx/usage-tracker.js +39 -11
  184. package/tx/vs/vs-api.js +19 -0
  185. package/tx/vs/vs-database.js +386 -62
  186. package/tx/vs/vs-package.js +44 -17
  187. package/tx/vs/vs-vsac.js +649 -101
  188. package/tx/workers/batch-validate.js +37 -10
  189. package/tx/workers/batch.js +6 -1
  190. package/tx/workers/cache-control.js +15 -4
  191. package/tx/workers/closure.js +156 -13
  192. package/tx/workers/compare.js +8 -5
  193. package/tx/workers/expand.js +1173 -289
  194. package/tx/workers/metadata.js +35 -0
  195. package/tx/workers/search.js +53 -1
  196. package/tx/workers/supplement-filters.js +145 -0
  197. package/tx/workers/translate.js +169 -4
  198. package/tx/workers/validate.js +223 -119
  199. package/tx/workers/worker.js +123 -31
  200. package/tx/xversion/xv-valueset.js +7 -4
  201. package/utilities/submit-test-report.js +3 -1
  202. package/utilities/tx-stress-test.js +482 -0
  203. package/vcl/readme.md +37 -0
  204. package/vcl/vcl-parser.js +15 -0
  205. package/xig/readme.md +52 -0
  206. package/xig/xig-template.html +5 -8
  207. package/xig/xig.js +238 -422
  208. package/changelog-template.md +0 -22
  209. package/registry/test-runner.js +0 -237
  210. package/release.sh +0 -80
  211. package/test-scripts/repro-re2-wasm-leak.js +0 -34
  212. package/tsconfig.eslint.json +0 -9
  213. package/tx/html/dash-metrics.liquid +0 -147
  214. package/tx/html/home-metrics.liquid +0 -225
package/tx/library.js CHANGED
@@ -22,6 +22,8 @@ const {RxNormServicesFactory} = require("./cs/cs-rxnorm");
22
22
  const {NdcServicesFactory} = require("./cs/cs-ndc");
23
23
  const {UniiServicesFactory} = require("./cs/cs-unii");
24
24
  const {SnomedServicesFactory} = require("./cs/cs-snomed");
25
+ const {SnomedFileReader} = require("./sct/structures");
26
+ const {SnomedLoadConfig} = require("./sct/load-config");
25
27
  const {CPTServicesFactory} = require("./cs/cs-cpt");
26
28
  const {OMOPServicesFactory} = require("./cs/cs-omop");
27
29
  const {ICD11ServicesFactory} = require("./cs/cs-icd11");
@@ -31,9 +33,10 @@ const {IETFLanguageCodeFactory} = require("./cs/cs-lang");
31
33
  const {LanguageDefinitions} = require("../library/languages");
32
34
  const {VersionUtilities} = require("../library/version-utilities");
33
35
  const {ListCodeSystemProvider} = require("./cs/cs-provider-list");
34
- const { Provider } = require("./provider");
36
+ const { Provider, CodeSystemMap } = require("./provider");
35
37
  const {I18nSupport} = require("../library/i18nsupport");
36
38
  const folders = require('../library/folder-setup');
39
+ const memoryGuard = require('../library/memory-guard');
37
40
  const {VSACValueSetProvider} = require("./vs/vs-vsac");
38
41
  const { OCLCodeSystemProvider, OCLSourceCodeSystemFactory } = require('./ocl/cs-ocl');
39
42
  const { OCLValueSetProvider } = require('./ocl/vs-ocl');
@@ -42,6 +45,32 @@ const {UriServicesFactory} = require("./cs/cs-uri");
42
45
  const {debugLog} = require("./operation-context");
43
46
  const { checkIdPrefix } = require("./library/resource-ids");
44
47
 
48
+ // folder:/file: sources keep their SQLite caches in <terminology-cache>/folder-source-<hash>
49
+ const LOCAL_SOURCE_CACHE_PREFIX = 'folder-source-';
50
+ // how long an unused folder-source-* cache is kept, in case another Library is using it
51
+ const LOCAL_SOURCE_CACHE_MAX_AGE = 24 * 60 * 60 * 1000;
52
+ // an unused .db or .cache file or package folder changed more recently than this isn't pruned
53
+ // (see pruneUnusedCaches)
54
+ const PRUNE_MIN_AGE = 24 * 60 * 60 * 1000;
55
+
56
+ /**
57
+ * The bytes in the files under a folder
58
+ * @param {string} dir
59
+ * @returns {Promise<number>}
60
+ */
61
+ async function folderSize(dir) {
62
+ let total = 0;
63
+ for (const entry of await fs.readdir(dir, { withFileTypes: true })) {
64
+ const p = path.join(dir, entry.name);
65
+ if (entry.isDirectory()) {
66
+ total += await folderSize(p);
67
+ } else if (entry.isFile()) {
68
+ total += (await fs.stat(p)).size;
69
+ }
70
+ }
71
+ return total;
72
+ }
73
+
45
74
  /**
46
75
  * This class holds all the loaded content ready for processing
47
76
  *
@@ -85,6 +114,28 @@ class Library {
85
114
  */
86
115
  #givenPrefixes = new Set();
87
116
 
117
+ /**
118
+ * {Set<string>} the folder-source-* cache folders this load used - the rest are pruned
119
+ * once they're stale (see #pruneLocalSourceCaches)
120
+ */
121
+ #localSourceCaches = new Set();
122
+
123
+ /**
124
+ * {Set<string>} the names of the files in the cache folder that this load used (see
125
+ * getOrDownloadFile) - with pruneTerminologyCache, the other .db and .cache files there are
126
+ * deleted once the library has loaded (see pruneTerminologyCache)
127
+ */
128
+ #usedCacheFiles = new Set();
129
+
130
+ /**
131
+ * {boolean} once the library and the endpoints' core packages are loaded, delete the .db and
132
+ * .cache files and the package folders in the cache folder that weren't used (the tx module's
133
+ * pruneTerminologyCache setting; see pruneUnusedCaches). Off by default: the cache folder is
134
+ * shared by everything that runs from the same data directory - a development server and the
135
+ * tests, say - and what one doesn't use, another may
136
+ */
137
+ pruneTerminologyCacheFiles = false;
138
+
88
139
  baseUrl = null;
89
140
  cacheFolder = null;
90
141
  startTime = Date.now();
@@ -92,6 +143,35 @@ class Library {
92
143
  lastTime = null;
93
144
  totalDownloaded = 0;
94
145
  vsacCfg = undefined;
146
+ /**
147
+ * {string|null} with several processes on one data directory (cluster.md), a name every
148
+ * process in the cluster shares (the master's role): the folder-source-* caches are the
149
+ * master's, named folder-source-<tag>-<hash>. The master builds each in a temporary sibling
150
+ * and renames it into place, so a worker never sees one half built; a worker
151
+ * (packageCachesReadOnly) only reads them, waiting for the master as for package caches,
152
+ * and never prunes
153
+ */
154
+ localSourceTag = null;
155
+
156
+ /**
157
+ * {boolean} a tx-worker (cluster.md): the package value set and concept map caches are the
158
+ * master's. This process opens them read-only, waits for the master to have finished one,
159
+ * and never builds, rebuilds or migrates one
160
+ */
161
+ packageCachesReadOnly = false;
162
+
163
+ /**
164
+ * {boolean} a tx-worker (cluster.md): the cached IANA media types registry is the master's.
165
+ * This process reads it and never downloads or writes it
166
+ */
167
+ mimeTypesReadOnly = false;
168
+
169
+ /**
170
+ * {boolean} a tx-worker (cluster.md): the LOINC, RxNorm, NDC, UNII, CPT and OMOP databases
171
+ * are the master's, which sets them up (LOINC's journal mode, the LOINC and RxNorm indexes)
172
+ * before the workers start. This process opens them read-only
173
+ */
174
+ terminologyDatabasesReadOnly = false;
95
175
 
96
176
  /**
97
177
  * {SyndicationStore|null} where the content this library loads is recorded for the
@@ -112,11 +192,20 @@ class Library {
112
192
  }
113
193
  }
114
194
 
115
- constructor(configFile, vsacCfg, log, stats) {
195
+ /**
196
+ * @param {string} configFile - the library yml
197
+ * @param {object} vsacCfg
198
+ * @param {object} log
199
+ * @param {object} stats
200
+ * @param {object} [snomedCfg] - the tx module's "snomed" block: how SNOMED CT cache files are
201
+ * loaded (see tx/sct/load-config.js)
202
+ */
203
+ constructor(configFile, vsacCfg, log, stats, snomedCfg = undefined) {
116
204
  this.configFile = configFile;
117
205
  this.vsacCfg = vsacCfg;
118
206
  this.log = log;
119
207
  this.stats = stats;
208
+ this.snomedLoad = new SnomedLoadConfig(snomedCfg, (msg) => (this.log ? this.log.warn(msg) : console.warn(msg)));
120
209
 
121
210
  // Only synchronous initialization here
122
211
  this.codeSystemFactories = new Map();
@@ -215,6 +304,7 @@ class Library {
215
304
  throw error;
216
305
  }
217
306
  }
307
+ this.snomedLoad.checkUnused();
218
308
  this.log.info('Loading Packages');
219
309
  this.#logPackagesHeader();
220
310
 
@@ -238,6 +328,11 @@ class Library {
238
328
  external: endMemory.external - this.startMemory.external
239
329
  };
240
330
 
331
+ if (!this.packageCachesReadOnly) {
332
+ // the master, or no role: a worker leaves the master's caches alone
333
+ await this.#pruneLocalSourceCaches();
334
+ }
335
+
241
336
  this.log.info(`Loading Time: ${(totalTime / 1000).toLocaleString()}s`);
242
337
  this.log.info(`Memory Used: ${(memoryIncrease.rss / 1024 / 1024).toFixed(2)} MB`);
243
338
 
@@ -541,7 +636,7 @@ class Library {
541
636
  break;
542
637
  }
543
638
  case "mimetypes" : {
544
- const mime = new MimeTypeServicesFactory(this.i18n, this.cacheFolder, this.log);
639
+ const mime = new MimeTypeServicesFactory(this.i18n, this.cacheFolder, this.log, { readOnly: this.mimeTypesReadOnly });
545
640
  await mime.load();
546
641
  this.registerProvider('internal', mime);
547
642
  break;
@@ -610,7 +705,7 @@ class Library {
610
705
  return;
611
706
  }
612
707
 
613
- const loinc = new LoincServicesFactory(this.i18n, loincFN);
708
+ const loinc = new LoincServicesFactory(this.i18n, loincFN, { readOnly: this.terminologyDatabasesReadOnly });
614
709
  await loinc.load();
615
710
  this.registerProvider(loincFN, loinc, isDefault);
616
711
  this.#syndicateBinary('loinc', loincFN, loinc);
@@ -621,7 +716,7 @@ class Library {
621
716
  if (mode === "fetch" || mode === "npm") {
622
717
  return;
623
718
  }
624
- const rxn = new RxNormServicesFactory(this.i18n, rxNormFN);
719
+ const rxn = new RxNormServicesFactory(this.i18n, rxNormFN, { readOnly: this.terminologyDatabasesReadOnly });
625
720
  await rxn.load();
626
721
  this.registerProvider(rxNormFN, rxn, isDefault);
627
722
  this.#syndicateBinary('rxnorm', rxNormFN, rxn);
@@ -632,7 +727,7 @@ class Library {
632
727
  if (mode === "fetch" || mode === "npm") {
633
728
  return;
634
729
  }
635
- const ndc = new NdcServicesFactory(this.i18n, ndcFN);
730
+ const ndc = new NdcServicesFactory(this.i18n, ndcFN, { readOnly: this.terminologyDatabasesReadOnly });
636
731
  await ndc.load();
637
732
  this.registerProvider(ndcFN, ndc, isDefault);
638
733
  this.#syndicateBinary('ndc', ndcFN, ndc);
@@ -643,7 +738,7 @@ class Library {
643
738
  if (mode === "fetch" || mode === "npm") {
644
739
  return;
645
740
  }
646
- const unii = new UniiServicesFactory(this.i18n, uniFN);
741
+ const unii = new UniiServicesFactory(this.i18n, uniFN, { readOnly: this.terminologyDatabasesReadOnly });
647
742
  await unii.load();
648
743
  this.registerProvider(uniFN, unii, isDefault);
649
744
  this.#syndicateBinary('unii', uniFN, unii);
@@ -654,8 +749,18 @@ class Library {
654
749
  if (mode === "fetch" || mode === "npm") {
655
750
  return;
656
751
  }
657
- const sct = new SnomedServicesFactory(this.i18n, sctFN);
752
+ const loadMode = this.snomedLoad.modeFor(sctFN, SnomedFileReader.versionUri(sctFN));
753
+ const sct = new SnomedServicesFactory(this.i18n, sctFN, loadMode, this.log);
658
754
  await sct.load();
755
+ if (loadMode !== 'read') {
756
+ // if it could not be mapped, mapFile has already said why, through this.log
757
+ this.log.info(`SNOMED CT ${path.basename(sctFN)}: load ${loadMode}, ` +
758
+ (sct.mapped ? 'memory-mapped' : 'not mapped, read into memory'));
759
+ }
760
+ if (sct.mapped && !memoryGuard._state.excludeFileBacked) {
761
+ memoryGuard.measureExcludingFileBacked();
762
+ this.log.info(`SNOMED CT cache files are memory-mapped. ${memoryGuard.describe()}`);
763
+ }
659
764
  this.registerProvider(sctFN, sct, isDefault);
660
765
  this.#syndicateBinary('snomed', sctFN, sct);
661
766
  }
@@ -665,7 +770,7 @@ class Library {
665
770
  if (mode === "fetch" || mode === "npm") {
666
771
  return;
667
772
  }
668
- const cpt = new CPTServicesFactory(this.i18n, cptFN);
773
+ const cpt = new CPTServicesFactory(this.i18n, cptFN, { readOnly: this.terminologyDatabasesReadOnly });
669
774
  await cpt.load();
670
775
  this.registerProvider(cptFN, cpt, isDefault);
671
776
  // the store syndicates the public fragment only, never the full (licensed) database
@@ -677,7 +782,7 @@ class Library {
677
782
  if (mode === "fetch" || mode === "npm") {
678
783
  return;
679
784
  }
680
- const omop = new OMOPServicesFactory(this.i18n, omopFN);
785
+ const omop = new OMOPServicesFactory(this.i18n, omopFN, { readOnly: this.terminologyDatabasesReadOnly });
681
786
  await omop.load();
682
787
  this.registerProvider(omopFN, omop, isDefault);
683
788
  this.#syndicateBinary('omop', omopFN, omop);
@@ -831,10 +936,10 @@ class Library {
831
936
  this.codeSystemProviders.push(cp);
832
937
  let vs = null;
833
938
  if (!csOnly) {
834
- vs = new PackageValueSetProvider(contentLoader);
939
+ vs = new PackageValueSetProvider(contentLoader, { readOnly: this.packageCachesReadOnly });
835
940
  await vs.initialize();
836
941
  this.valueSetProviders.push(vs);
837
- const cm = new PackageConceptMapProvider(contentLoader);
942
+ const cm = new PackageConceptMapProvider(contentLoader, { readOnly: this.packageCachesReadOnly });
838
943
  await cm.initialize();
839
944
  this.conceptMapProviders.push(cm);
840
945
  }
@@ -861,7 +966,7 @@ class Library {
861
966
  }
862
967
 
863
968
  const folderPath = this.#resolveLocalSourcePath(details);
864
- const cacheSubdir = await this.#prepareLocalSourceCache(folderPath);
969
+ const cacheSubdir = await this.#prepareLocalSourceCache(folderPath, dir => new FolderContentLoader(folderPath, dir));
865
970
  await this.#loadLocalSource(new FolderContentLoader(folderPath, cacheSubdir));
866
971
  }
867
972
 
@@ -886,7 +991,7 @@ class Library {
886
991
  }
887
992
 
888
993
  const filePath = this.#resolveLocalSourcePath(details);
889
- const cacheSubdir = await this.#prepareLocalSourceCache(filePath);
994
+ const cacheSubdir = await this.#prepareLocalSourceCache(filePath, dir => new FileContentLoader(filePath, dir));
890
995
  await this.#loadLocalSource(new FileContentLoader(filePath, cacheSubdir));
891
996
  }
892
997
 
@@ -906,16 +1011,207 @@ class Library {
906
1011
  * Parks the Package*Provider SQLite caches under the terminology cache rather
907
1012
  * than polluting the user's source folder, and wipes between runs so edits to
908
1013
  * the source are reliably picked up.
1014
+ *
1015
+ * In a cluster (localSourceTag set) every process uses the master's folder:
1016
+ * - a worker (packageCachesReadOnly) touches nothing; its providers open the caches
1017
+ * read-only, waiting for the master to have built them
1018
+ * - the master builds the caches in a temporary sibling, then renames it into place, so a
1019
+ * running worker never finds the folder half built
1020
+ *
1021
+ * @param {string} sourcePath
1022
+ * @param {function(string): FolderContentLoader} makeLoader - a loader with its caches in a folder
1023
+ * @returns {Promise<string>} the cache folder
909
1024
  * @private
910
1025
  */
911
- async #prepareLocalSourceCache(sourcePath) {
1026
+ async #prepareLocalSourceCache(sourcePath, makeLoader) {
912
1027
  const hash = crypto.createHash('sha1').update(sourcePath).digest('hex').substring(0, 16);
913
- const cacheSubdir = path.join(this.cacheFolder, 'folder-source-' + hash);
914
- await fs.rm(cacheSubdir, { recursive: true, force: true });
915
- await fs.mkdir(cacheSubdir, { recursive: true });
1028
+ const name = LOCAL_SOURCE_CACHE_PREFIX + (this.localSourceTag ? this.localSourceTag + '-' : '') + hash;
1029
+ const cacheSubdir = path.join(this.cacheFolder, name);
1030
+ this.#localSourceCaches.add(name);
1031
+ if (this.packageCachesReadOnly) {
1032
+ return cacheSubdir;
1033
+ }
1034
+ if (!this.localSourceTag) {
1035
+ await fs.rm(cacheSubdir, { recursive: true, force: true });
1036
+ await fs.mkdir(cacheSubdir, { recursive: true });
1037
+ return cacheSubdir;
1038
+ }
1039
+ const building = `${cacheSubdir}.building-${process.pid}`;
1040
+ const old = `${cacheSubdir}.old-${process.pid}`;
1041
+ await fs.rm(building, { recursive: true, force: true });
1042
+ await fs.rm(old, { recursive: true, force: true });
1043
+ await fs.mkdir(building, { recursive: true });
1044
+ const loader = makeLoader(building);
1045
+ await loader.initialize();
1046
+ const vs = new PackageValueSetProvider(loader);
1047
+ await vs.initialize();
1048
+ await vs.close();
1049
+ const cm = new PackageConceptMapProvider(loader);
1050
+ await cm.initialize();
1051
+ // a directory can't be renamed over a non-empty one: move the old one aside first. A
1052
+ // worker that looks in between finds no folder, and waits for it
1053
+ try {
1054
+ await fs.rename(cacheSubdir, old);
1055
+ } catch (e) {
1056
+ if (e.code !== 'ENOENT') {
1057
+ throw e;
1058
+ }
1059
+ }
1060
+ await fs.rename(building, cacheSubdir);
1061
+ await fs.rm(old, { recursive: true, force: true });
916
1062
  return cacheSubdir;
917
1063
  }
918
1064
 
1065
+ /**
1066
+ * Removes the folder-source-* caches left by sources that are no longer loaded (the
1067
+ * name is a hash of the source path, so a moved or temporary folder leaves one behind).
1068
+ *
1069
+ * The cache folder can be shared by more than one Library at a time (a server and the
1070
+ * tests, run from the same checkout), so a cache this load didn't use may be another
1071
+ * one's. Every load rebuilds its caches, so a live one is always recent - only those
1072
+ * untouched for LOCAL_SOURCE_CACHE_MAX_AGE are removed.
1073
+ * @private
1074
+ */
1075
+ async #pruneLocalSourceCaches() {
1076
+ let entries;
1077
+ try {
1078
+ entries = await fs.readdir(this.cacheFolder, { withFileTypes: true });
1079
+ } catch (e) {
1080
+ return;
1081
+ }
1082
+ const cutoff = Date.now() - LOCAL_SOURCE_CACHE_MAX_AGE;
1083
+ let count = 0;
1084
+ for (const entry of entries) {
1085
+ if (!entry.isDirectory() || !entry.name.startsWith(LOCAL_SOURCE_CACHE_PREFIX) || this.#localSourceCaches.has(entry.name)) {
1086
+ continue;
1087
+ }
1088
+ const dir = path.join(this.cacheFolder, entry.name);
1089
+ try {
1090
+ const stat = await fs.stat(dir);
1091
+ if (stat.mtimeMs < cutoff) {
1092
+ await fs.rm(dir, { recursive: true, force: true });
1093
+ count++;
1094
+ }
1095
+ } catch (e) {
1096
+ this.log.warn(`Unable to remove stale local source cache ${dir}: ${e.message}`);
1097
+ }
1098
+ }
1099
+ if (count > 0) {
1100
+ this.log.info(`Removed ${count} stale local source cache folder${count === 1 ? '' : 's'}`);
1101
+ }
1102
+ }
1103
+
1104
+ /**
1105
+ * With pruneTerminologyCacheFiles, and in a process that owns the caches (not a tx-worker):
1106
+ * delete what's in the cache folder that wasn't used - the .db and .cache files
1107
+ * (pruneTerminologyCache) and the package folders (prunePackages). Call it once everything is
1108
+ * loaded: the library (load) and every endpoint's core packages (cloneWithFhirVersion)
1109
+ * @returns {Promise<{files: string[], packages: string[]}>} what was deleted
1110
+ */
1111
+ async pruneUnusedCaches() {
1112
+ if (!this.pruneTerminologyCacheFiles || this.packageCachesReadOnly) {
1113
+ return { files: [], packages: [] };
1114
+ }
1115
+ return { files: await this.pruneTerminologyCache(), packages: await this.prunePackages() };
1116
+ }
1117
+
1118
+ /**
1119
+ * Deletes the package folders (id#version) in the cache folder that the package manager
1120
+ * hasn't returned this run (see PackageManager.usedPackages): packages no longer in the
1121
+ * library, older versions of ones that are, the core packages of FHIR versions no endpoint
1122
+ * has any more, and what failed or interrupted downloads left behind. Their value set and
1123
+ * concept map caches are in them, so go with them. A folder changed in the last
1124
+ * PRUNE_MIN_AGE is kept.
1125
+ * @returns {Promise<string[]>} the names of the folders deleted
1126
+ */
1127
+ async prunePackages() {
1128
+ let entries;
1129
+ try {
1130
+ entries = await fs.readdir(this.cacheFolder, { withFileTypes: true });
1131
+ } catch (e) {
1132
+ return [];
1133
+ }
1134
+ const used = this.packageManager.usedPackages;
1135
+ const cutoff = Date.now() - PRUNE_MIN_AGE;
1136
+ const deleted = [];
1137
+ let bytes = 0;
1138
+ for (const entry of entries) {
1139
+ if (!entry.isDirectory() || !entry.name.includes('#') || used.has(entry.name)) {
1140
+ continue;
1141
+ }
1142
+ const dir = path.join(this.cacheFolder, entry.name);
1143
+ try {
1144
+ const stat = await fs.stat(dir);
1145
+ if (stat.mtimeMs >= cutoff) {
1146
+ this.log.info(`Not deleting unused package ${entry.name} yet: it changed in the last day`);
1147
+ continue;
1148
+ }
1149
+ const size = await folderSize(dir);
1150
+ await fs.rm(dir, { recursive: true, force: true });
1151
+ deleted.push(entry.name);
1152
+ bytes += size;
1153
+ this.log.info(`Deleted unused package ${entry.name} (${(size / 1024 / 1024).toFixed(1)} MB) from the terminology cache`);
1154
+ } catch (e) {
1155
+ this.log.warn(`Unable to delete unused package ${dir}: ${e.message}`);
1156
+ }
1157
+ }
1158
+ if (deleted.length > 0) {
1159
+ this.log.info(`Deleted ${deleted.length} unused package${deleted.length === 1 ? '' : 's'} from the terminology cache, ${(bytes / 1024 / 1024).toFixed(1)} MB`);
1160
+ }
1161
+ return deleted;
1162
+ }
1163
+
1164
+ /**
1165
+ * Deletes the .db and .cache files directly in the cache folder (terminology-cache) that this
1166
+ * load didn't use: the downloaded databases and SNOMED CT caches of sources that are no longer
1167
+ * in the library, and the fallback of an "a|b" source whose first file is there. Their SQLite
1168
+ * side files (-wal, -shm, -journal) go with them. Package folders, and the other folders and
1169
+ * files there, are left alone.
1170
+ *
1171
+ * A file changed in the last PRUNE_MIN_AGE is kept: it may be being copied in, or built,
1172
+ * for a library that doesn't name it yet.
1173
+ *
1174
+ * See pruneUnusedCaches, which calls it
1175
+ * @returns {Promise<string[]>} the names of the files deleted
1176
+ */
1177
+ async pruneTerminologyCache() {
1178
+ let entries;
1179
+ try {
1180
+ entries = await fs.readdir(this.cacheFolder, { withFileTypes: true });
1181
+ } catch (e) {
1182
+ return [];
1183
+ }
1184
+ const cutoff = Date.now() - PRUNE_MIN_AGE;
1185
+ const deleted = [];
1186
+ let bytes = 0;
1187
+ for (const entry of entries) {
1188
+ if (!entry.isFile() || !/\.(db|cache)$/i.test(entry.name) || this.#usedCacheFiles.has(entry.name)) {
1189
+ continue;
1190
+ }
1191
+ const file = path.join(this.cacheFolder, entry.name);
1192
+ try {
1193
+ const stat = await fs.stat(file);
1194
+ if (stat.mtimeMs >= cutoff) {
1195
+ this.log.info(`Not deleting unused ${entry.name} yet: it changed in the last day`);
1196
+ continue;
1197
+ }
1198
+ await fs.rm(file, { force: true });
1199
+ for (const suffix of ['-wal', '-shm', '-journal']) {
1200
+ await fs.rm(file + suffix, { force: true });
1201
+ }
1202
+ deleted.push(entry.name);
1203
+ bytes += stat.size;
1204
+ this.log.info(`Deleted unused ${entry.name} (${(stat.size / 1024 / 1024).toFixed(1)} MB) from the terminology cache`);
1205
+ } catch (e) {
1206
+ this.log.warn(`Unable to delete unused ${file}: ${e.message}`);
1207
+ }
1208
+ }
1209
+ if (deleted.length > 0) {
1210
+ this.log.info(`Deleted ${deleted.length} unused file${deleted.length === 1 ? '' : 's'} from the terminology cache, ${(bytes / 1024 / 1024).toFixed(1)} MB`);
1211
+ }
1212
+ return deleted;
1213
+ }
1214
+
919
1215
  /**
920
1216
  * Shared loader for folder- and file-sourced content: registers the loader's
921
1217
  * CodeSystems, ValueSets and ConceptMaps with this library.
@@ -972,10 +1268,10 @@ class Library {
972
1268
  }
973
1269
  this.codeSystemProviders.push(cp);
974
1270
 
975
- const vs = new PackageValueSetProvider(contentLoader);
1271
+ const vs = new PackageValueSetProvider(contentLoader, { readOnly: this.packageCachesReadOnly });
976
1272
  await vs.initialize();
977
1273
  this.valueSetProviders.push(vs);
978
- const cm = new PackageConceptMapProvider(contentLoader);
1274
+ const cm = new PackageConceptMapProvider(contentLoader, { readOnly: this.packageCachesReadOnly });
979
1275
  await cm.initialize();
980
1276
  this.conceptMapProviders.push(cm);
981
1277
 
@@ -1000,10 +1296,12 @@ class Library {
1000
1296
 
1001
1297
  const firstPath = path.join(this.cacheFolder, firstName);
1002
1298
  if (await this.fileExists(firstPath)) {
1299
+ this.#usedCacheFiles.add(path.basename(firstPath));
1003
1300
  return firstPath;
1004
1301
  }
1005
1302
  }
1006
1303
  const filePath = path.join(this.cacheFolder, fileName);
1304
+ this.#usedCacheFiles.add(path.basename(filePath));
1007
1305
 
1008
1306
  // Check if file already exists
1009
1307
  if (await this.fileExists(filePath)) {
@@ -1118,7 +1416,7 @@ class Library {
1118
1416
  const provider = new Provider();
1119
1417
  provider.i18n = this.i18n;
1120
1418
  provider.codeSystemFactories = new Map(this.codeSystemFactories); // all of them
1121
- provider.codeSystems = new Map();
1419
+ provider.codeSystems = new CodeSystemMap();
1122
1420
  provider.valueSetProviders = [];
1123
1421
  provider.conceptMapProviders = [];
1124
1422
  provider.path = path;
@@ -1142,7 +1440,8 @@ class Library {
1142
1440
 
1143
1441
  // Load FHIR packages - these will be added to valueSetProviders first
1144
1442
  for (const packageId of fhirPackages) {
1145
- const contentLoader = await provider.loadNpm(this.packageManager, this.cacheFolder, packageId, false, "npm", false);
1443
+ const contentLoader = await provider.loadNpm(this.packageManager, this.cacheFolder, packageId, false, "npm",
1444
+ { readOnly: this.packageCachesReadOnly });
1146
1445
  // the core packages are the only version specific content in the syndication store
1147
1446
  this.syndicatePackage(contentLoader, { fhirVersion: VersionUtilities.getMajMin(String(fhirVersion)) });
1148
1447
  }
@@ -183,13 +183,15 @@ const OPERATIONS = [
183
183
  methods: ['post'],
184
184
  summary: 'Validate many codes against code systems',
185
185
  description: 'A batch of CodeSystem $validate-code requests: each `validation` is a ' +
186
- 'Parameters resource with the parameters of one. system, codeSystem, tx-resource and ' +
187
- 'lenient-display-validation at the top level apply to every validation that doesn\'t ' +
188
- 'have its own.',
186
+ 'Parameters resource with the parameters of one. url (the code system), version, system, ' +
187
+ 'codeSystem, tx-resource and lenient-display-validation at the top level apply to every ' +
188
+ 'validation that doesn\'t have its own. Every validation is against a code system.',
189
189
  postBody: ['Parameters'],
190
190
  in: [
191
191
  p('validation', 'Parameters', '*', 'The parameters of one $validate-code', { min: 1 }),
192
- p('system', 'uri', '1', 'The code system, for every validation'),
192
+ p('url', 'uri', '1', 'The code system, for every validation'),
193
+ p('version', 'string', '1', 'The code system version, for every validation'),
194
+ p('system', 'uri', '1', 'The code system, if url isn\'t given, for every validation'),
193
195
  p('codeSystem', 'CodeSystem', '1', 'The code system, for every validation'),
194
196
  p('lenient-display-validation', 'boolean', '1', 'For every validation'),
195
197
  p('tx-resource', 'Resource', '*', 'Resources for every validation')