fhirsmith 0.18.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ACCESSIBILITY.md +119 -0
- package/CHANGELOG.md +149 -0
- package/CONFIGURATION.md +305 -0
- package/LICENSE +29 -0
- package/README.md +29 -122
- package/cluster.js +1220 -0
- package/cluster.md +707 -0
- package/config-template.json +77 -8
- package/extension-tracker/extension-tracker-template.html +5 -8
- package/extension-tracker/extension-tracker.js +0 -2
- package/folder/folder.js +7 -6
- package/ips/analysis.js +560 -0
- package/ips/ips-template.html +5 -8
- package/ips/ips.js +132 -59
- package/ips/nationality.js +120 -0
- package/ips/pipeline.js +74 -21
- package/ips/presenter.js +1 -1
- package/ips/readme.md +132 -25
- package/ips/render.js +466 -77
- package/ips/store.js +79 -22
- package/ips/structure.js +247 -0
- package/ips/terminology.js +1 -1
- package/library/body-limit.js +22 -0
- package/library/cache-generation.js +15 -0
- package/library/cgroup-memory.js +133 -1
- package/library/cluster-counts.js +121 -0
- package/library/cluster-events.js +153 -0
- package/library/cluster-metrics.js +193 -0
- package/library/cluster-status.js +468 -0
- package/library/config-reader.js +96 -0
- package/library/daily-report.js +100 -2
- package/library/html-server.js +27 -1
- package/library/i18nsupport.js +23 -1
- package/library/keep-alive.js +18 -0
- package/library/languages.js +17 -3
- package/library/logger.js +31 -0
- package/library/memory-guard.js +428 -0
- package/library/native-memory.js +52 -0
- package/library/npm-audit.js +51 -25
- package/library/openapi-doc.js +7 -5
- package/library/package-manager.js +8 -0
- package/library/role.js +197 -0
- package/library/server-credentials.js +169 -0
- package/library/sqlite-utilities.js +59 -0
- package/library/ssrf-guard.js +100 -0
- package/library/static-response-stats.js +16 -0
- package/library/version-utilities.js +50 -0
- package/library/worker-count.js +192 -0
- package/nginx.md +30 -11
- package/npmprojector/indexer.js +18 -10
- package/package.json +11 -8
- package/packages/feed-check-page.js +117 -0
- package/packages/feed-check.js +279 -0
- package/packages/npm-check.js +57 -3
- package/packages/npm-signing.js +219 -0
- package/packages/openapi.yaml +207 -1
- package/packages/package-crawler.js +322 -41
- package/packages/packages-template.html +6 -8
- package/packages/packages.js +604 -57
- package/packages/readme.md +113 -1
- package/publisher/publisher-template.html +2 -5
- package/publisher/publisher.js +121 -110
- package/publisher/readme.md +75 -0
- package/registry/crawler.js +84 -108
- package/registry/documentation/configuration.md +34 -0
- package/registry/registry-template.html +5 -8
- package/registry/registry.js +5 -4
- package/root-bare-template.html +7 -4
- package/root-template.html +5 -8
- package/server.js +398 -198
- package/shl/shl.js +38 -37
- package/static/a11y/a11y-check.js +832 -0
- package/static/a11y/axe.min.js +12 -0
- package/static/assets/css/bootstrap-fhir.css +15 -15
- package/static/assets/images/flags/au.svg +18 -0
- package/static/assets/images/flags/unknown.svg +11 -0
- package/static/assets/images/flags/us.svg +63 -0
- package/static/assets/images/icon_audit.png +0 -0
- package/static/assets/images/icon_clinical.png +0 -0
- package/static/assets/images/icon_document.png +0 -0
- package/static/assets/images/icon_external.png +0 -0
- package/static/assets/images/icon_medication.png +0 -0
- package/static/assets/images/icon_organization.png +0 -0
- package/static/assets/images/icon_other.png +0 -0
- package/static/assets/images/icon_person.png +0 -0
- package/static/assets/images/icon_section.png +0 -0
- package/stats-db.js +45 -0
- package/stats.js +197 -10
- package/testing/testing-template.html +5 -8
- package/token/nginx_passport_setup.md +1 -1
- package/token/security_guide.md +1 -1
- package/token/token-template.html +2 -5
- package/token/token.js +40 -36
- package/translations/Messages.properties +19 -0
- package/tx/README.md +154 -4
- package/tx/closure/closure-store.js +126 -3
- package/tx/cm/cm-database.js +69 -57
- package/tx/cm/cm-package.js +29 -1
- package/tx/cs/cs-api.js +284 -19
- package/tx/cs/cs-api.md +41 -4
- package/tx/cs/cs-areacode.js +1 -1
- package/tx/cs/cs-country.js +17 -5
- package/tx/cs/cs-cpt.js +33 -16
- package/tx/cs/cs-cs.js +148 -14
- package/tx/cs/cs-currency.js +1 -1
- package/tx/cs/cs-icd11.js +21 -13
- package/tx/cs/cs-icf.js +22 -3
- package/tx/cs/cs-lang.js +2 -2
- package/tx/cs/cs-loinc.js +185 -66
- package/tx/cs/cs-mimetypes.js +12 -2
- package/tx/cs/cs-ndc.js +40 -25
- package/tx/cs/cs-omop.js +54 -29
- package/tx/cs/cs-rxnorm.js +300 -196
- package/tx/cs/cs-snomed.js +608 -64
- package/tx/cs/cs-ucum.js +14 -4
- package/tx/cs/cs-unii.js +18 -14
- package/tx/cs/cs-virtual.js +210 -0
- package/tx/ecosystem-resolver.js +413 -0
- package/tx/html/cluster-dash-metrics.liquid +26 -0
- package/tx/html/cluster-home-metrics.liquid +31 -0
- package/tx/html/cluster-metrics-script.liquid +251 -0
- package/tx/html/codesystem-operations.liquid +3 -3
- package/tx/html/ecl-panel.liquid +1 -1
- package/tx/html/external-metrics.liquid +17 -0
- package/tx/html/operations-form.liquid +17 -17
- package/tx/html/search-form.liquid +11 -11
- package/tx/html/status-line.liquid +58 -0
- package/tx/html/status-panel.liquid +140 -0
- package/tx/html/status-tasks.liquid +25 -0
- package/tx/html/tx-template.html +2 -5
- package/tx/importers/atc-to-fhir.js +17 -16
- package/tx/importers/export-icd11-codesystem.js +3 -2
- package/tx/importers/import-icd11.js +4 -3
- package/tx/importers/import-loinc.module.js +118 -56
- package/tx/importers/import-ndc.module.js +56 -44
- package/tx/importers/import-rxnorm.module.js +118 -79
- package/tx/importers/import-sct.module.js +57 -58
- package/tx/importers/import-unii.module.js +37 -36
- package/tx/importers/subset-loinc.module.js +49 -48
- package/tx/importers/subset-rxnorm.module.js +29 -28
- package/tx/importers/terminal.js +18 -0
- package/tx/importers/tx-import-base.js +13 -12
- package/tx/importers/tx-import-settings.js +5 -4
- package/tx/importers/tx-import.js +53 -52
- package/tx/library/canonical-resource.js +49 -0
- package/tx/library/codesystem.js +94 -5
- package/tx/library/designations.js +45 -2
- package/tx/library/extensions.js +13 -7
- package/tx/library/implicit-valuesets.js +66 -0
- package/tx/library/renderer.js +77 -21
- package/tx/library/sqlite-utilities.js +2 -0
- package/tx/library/ucum-parsers.js +11 -1
- package/tx/library/wait-for-cache.js +43 -0
- package/tx/library.js +321 -22
- package/tx/openapi-operations.js +6 -4
- package/tx/operation-context.js +460 -99
- package/tx/params.js +25 -0
- package/tx/problems.js +3 -5
- package/tx/provider.js +278 -42
- package/tx/resource-cache-sqlite.js +1050 -0
- package/tx/sct/ecl.js +47 -6
- package/tx/sct/editions.js +39 -0
- package/tx/sct/expressions.js +10 -1
- package/tx/sct/load-config.js +107 -0
- package/tx/sct/mmap.js +109 -0
- package/tx/sct/structures.js +166 -13
- package/tx/sct/text-index.js +5 -1
- package/tx/syndication/syndication-feed.js +2 -2
- package/tx/tx-html.js +44 -8
- package/tx/tx.fhir.org.yml +8 -8
- package/tx/tx.js +282 -80
- package/tx/usage-tracker.js +39 -11
- package/tx/vs/vs-api.js +19 -0
- package/tx/vs/vs-database.js +386 -62
- package/tx/vs/vs-package.js +27 -4
- package/tx/vs/vs-vsac.js +648 -100
- package/tx/workers/batch-validate.js +37 -10
- package/tx/workers/batch.js +6 -1
- package/tx/workers/cache-control.js +15 -4
- package/tx/workers/closure.js +156 -13
- package/tx/workers/compare.js +8 -5
- package/tx/workers/expand.js +1173 -289
- package/tx/workers/metadata.js +35 -0
- package/tx/workers/search.js +53 -1
- package/tx/workers/supplement-filters.js +145 -0
- package/tx/workers/translate.js +169 -4
- package/tx/workers/validate.js +223 -119
- package/tx/workers/worker.js +123 -31
- package/tx/xversion/xv-valueset.js +7 -4
- package/utilities/submit-test-report.js +3 -1
- package/utilities/tx-stress-test.js +482 -0
- package/vcl/readme.md +37 -0
- package/vcl/vcl-parser.js +15 -0
- package/xig/readme.md +52 -0
- package/xig/xig-template.html +5 -8
- package/xig/xig.js +238 -422
- package/changelog-template.md +0 -22
- package/registry/test-runner.js +0 -237
- package/release.sh +0 -80
- package/test-scripts/repro-re2-wasm-leak.js +0 -34
- package/tsconfig.eslint.json +0 -9
- package/tx/html/dash-metrics.liquid +0 -147
- package/tx/html/home-metrics.liquid +0 -225
package/tx/library.js
CHANGED
|
@@ -22,6 +22,8 @@ const {RxNormServicesFactory} = require("./cs/cs-rxnorm");
|
|
|
22
22
|
const {NdcServicesFactory} = require("./cs/cs-ndc");
|
|
23
23
|
const {UniiServicesFactory} = require("./cs/cs-unii");
|
|
24
24
|
const {SnomedServicesFactory} = require("./cs/cs-snomed");
|
|
25
|
+
const {SnomedFileReader} = require("./sct/structures");
|
|
26
|
+
const {SnomedLoadConfig} = require("./sct/load-config");
|
|
25
27
|
const {CPTServicesFactory} = require("./cs/cs-cpt");
|
|
26
28
|
const {OMOPServicesFactory} = require("./cs/cs-omop");
|
|
27
29
|
const {ICD11ServicesFactory} = require("./cs/cs-icd11");
|
|
@@ -31,9 +33,10 @@ const {IETFLanguageCodeFactory} = require("./cs/cs-lang");
|
|
|
31
33
|
const {LanguageDefinitions} = require("../library/languages");
|
|
32
34
|
const {VersionUtilities} = require("../library/version-utilities");
|
|
33
35
|
const {ListCodeSystemProvider} = require("./cs/cs-provider-list");
|
|
34
|
-
const { Provider } = require("./provider");
|
|
36
|
+
const { Provider, CodeSystemMap } = require("./provider");
|
|
35
37
|
const {I18nSupport} = require("../library/i18nsupport");
|
|
36
38
|
const folders = require('../library/folder-setup');
|
|
39
|
+
const memoryGuard = require('../library/memory-guard');
|
|
37
40
|
const {VSACValueSetProvider} = require("./vs/vs-vsac");
|
|
38
41
|
const { OCLCodeSystemProvider, OCLSourceCodeSystemFactory } = require('./ocl/cs-ocl');
|
|
39
42
|
const { OCLValueSetProvider } = require('./ocl/vs-ocl');
|
|
@@ -42,6 +45,32 @@ const {UriServicesFactory} = require("./cs/cs-uri");
|
|
|
42
45
|
const {debugLog} = require("./operation-context");
|
|
43
46
|
const { checkIdPrefix } = require("./library/resource-ids");
|
|
44
47
|
|
|
48
|
+
// folder:/file: sources keep their SQLite caches in <terminology-cache>/folder-source-<hash>
|
|
49
|
+
const LOCAL_SOURCE_CACHE_PREFIX = 'folder-source-';
|
|
50
|
+
// how long an unused folder-source-* cache is kept, in case another Library is using it
|
|
51
|
+
const LOCAL_SOURCE_CACHE_MAX_AGE = 24 * 60 * 60 * 1000;
|
|
52
|
+
// an unused .db or .cache file or package folder changed more recently than this isn't pruned
|
|
53
|
+
// (see pruneUnusedCaches)
|
|
54
|
+
const PRUNE_MIN_AGE = 24 * 60 * 60 * 1000;
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* The bytes in the files under a folder
|
|
58
|
+
* @param {string} dir
|
|
59
|
+
* @returns {Promise<number>}
|
|
60
|
+
*/
|
|
61
|
+
async function folderSize(dir) {
|
|
62
|
+
let total = 0;
|
|
63
|
+
for (const entry of await fs.readdir(dir, { withFileTypes: true })) {
|
|
64
|
+
const p = path.join(dir, entry.name);
|
|
65
|
+
if (entry.isDirectory()) {
|
|
66
|
+
total += await folderSize(p);
|
|
67
|
+
} else if (entry.isFile()) {
|
|
68
|
+
total += (await fs.stat(p)).size;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return total;
|
|
72
|
+
}
|
|
73
|
+
|
|
45
74
|
/**
|
|
46
75
|
* This class holds all the loaded content ready for processing
|
|
47
76
|
*
|
|
@@ -85,6 +114,28 @@ class Library {
|
|
|
85
114
|
*/
|
|
86
115
|
#givenPrefixes = new Set();
|
|
87
116
|
|
|
117
|
+
/**
|
|
118
|
+
* {Set<string>} the folder-source-* cache folders this load used - the rest are pruned
|
|
119
|
+
* once they're stale (see #pruneLocalSourceCaches)
|
|
120
|
+
*/
|
|
121
|
+
#localSourceCaches = new Set();
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* {Set<string>} the names of the files in the cache folder that this load used (see
|
|
125
|
+
* getOrDownloadFile) - with pruneTerminologyCache, the other .db and .cache files there are
|
|
126
|
+
* deleted once the library has loaded (see pruneTerminologyCache)
|
|
127
|
+
*/
|
|
128
|
+
#usedCacheFiles = new Set();
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* {boolean} once the library and the endpoints' core packages are loaded, delete the .db and
|
|
132
|
+
* .cache files and the package folders in the cache folder that weren't used (the tx module's
|
|
133
|
+
* pruneTerminologyCache setting; see pruneUnusedCaches). Off by default: the cache folder is
|
|
134
|
+
* shared by everything that runs from the same data directory - a development server and the
|
|
135
|
+
* tests, say - and what one doesn't use, another may
|
|
136
|
+
*/
|
|
137
|
+
pruneTerminologyCacheFiles = false;
|
|
138
|
+
|
|
88
139
|
baseUrl = null;
|
|
89
140
|
cacheFolder = null;
|
|
90
141
|
startTime = Date.now();
|
|
@@ -92,6 +143,35 @@ class Library {
|
|
|
92
143
|
lastTime = null;
|
|
93
144
|
totalDownloaded = 0;
|
|
94
145
|
vsacCfg = undefined;
|
|
146
|
+
/**
|
|
147
|
+
* {string|null} with several processes on one data directory (cluster.md), a name every
|
|
148
|
+
* process in the cluster shares (the master's role): the folder-source-* caches are the
|
|
149
|
+
* master's, named folder-source-<tag>-<hash>. The master builds each in a temporary sibling
|
|
150
|
+
* and renames it into place, so a worker never sees one half built; a worker
|
|
151
|
+
* (packageCachesReadOnly) only reads them, waiting for the master as for package caches,
|
|
152
|
+
* and never prunes
|
|
153
|
+
*/
|
|
154
|
+
localSourceTag = null;
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* {boolean} a tx-worker (cluster.md): the package value set and concept map caches are the
|
|
158
|
+
* master's. This process opens them read-only, waits for the master to have finished one,
|
|
159
|
+
* and never builds, rebuilds or migrates one
|
|
160
|
+
*/
|
|
161
|
+
packageCachesReadOnly = false;
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* {boolean} a tx-worker (cluster.md): the cached IANA media types registry is the master's.
|
|
165
|
+
* This process reads it and never downloads or writes it
|
|
166
|
+
*/
|
|
167
|
+
mimeTypesReadOnly = false;
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* {boolean} a tx-worker (cluster.md): the LOINC, RxNorm, NDC, UNII, CPT and OMOP databases
|
|
171
|
+
* are the master's, which sets them up (LOINC's journal mode, the LOINC and RxNorm indexes)
|
|
172
|
+
* before the workers start. This process opens them read-only
|
|
173
|
+
*/
|
|
174
|
+
terminologyDatabasesReadOnly = false;
|
|
95
175
|
|
|
96
176
|
/**
|
|
97
177
|
* {SyndicationStore|null} where the content this library loads is recorded for the
|
|
@@ -112,11 +192,20 @@ class Library {
|
|
|
112
192
|
}
|
|
113
193
|
}
|
|
114
194
|
|
|
115
|
-
|
|
195
|
+
/**
|
|
196
|
+
* @param {string} configFile - the library yml
|
|
197
|
+
* @param {object} vsacCfg
|
|
198
|
+
* @param {object} log
|
|
199
|
+
* @param {object} stats
|
|
200
|
+
* @param {object} [snomedCfg] - the tx module's "snomed" block: how SNOMED CT cache files are
|
|
201
|
+
* loaded (see tx/sct/load-config.js)
|
|
202
|
+
*/
|
|
203
|
+
constructor(configFile, vsacCfg, log, stats, snomedCfg = undefined) {
|
|
116
204
|
this.configFile = configFile;
|
|
117
205
|
this.vsacCfg = vsacCfg;
|
|
118
206
|
this.log = log;
|
|
119
207
|
this.stats = stats;
|
|
208
|
+
this.snomedLoad = new SnomedLoadConfig(snomedCfg, (msg) => (this.log ? this.log.warn(msg) : console.warn(msg)));
|
|
120
209
|
|
|
121
210
|
// Only synchronous initialization here
|
|
122
211
|
this.codeSystemFactories = new Map();
|
|
@@ -215,6 +304,7 @@ class Library {
|
|
|
215
304
|
throw error;
|
|
216
305
|
}
|
|
217
306
|
}
|
|
307
|
+
this.snomedLoad.checkUnused();
|
|
218
308
|
this.log.info('Loading Packages');
|
|
219
309
|
this.#logPackagesHeader();
|
|
220
310
|
|
|
@@ -238,6 +328,11 @@ class Library {
|
|
|
238
328
|
external: endMemory.external - this.startMemory.external
|
|
239
329
|
};
|
|
240
330
|
|
|
331
|
+
if (!this.packageCachesReadOnly) {
|
|
332
|
+
// the master, or no role: a worker leaves the master's caches alone
|
|
333
|
+
await this.#pruneLocalSourceCaches();
|
|
334
|
+
}
|
|
335
|
+
|
|
241
336
|
this.log.info(`Loading Time: ${(totalTime / 1000).toLocaleString()}s`);
|
|
242
337
|
this.log.info(`Memory Used: ${(memoryIncrease.rss / 1024 / 1024).toFixed(2)} MB`);
|
|
243
338
|
|
|
@@ -541,7 +636,7 @@ class Library {
|
|
|
541
636
|
break;
|
|
542
637
|
}
|
|
543
638
|
case "mimetypes" : {
|
|
544
|
-
const mime = new MimeTypeServicesFactory(this.i18n, this.cacheFolder, this.log);
|
|
639
|
+
const mime = new MimeTypeServicesFactory(this.i18n, this.cacheFolder, this.log, { readOnly: this.mimeTypesReadOnly });
|
|
545
640
|
await mime.load();
|
|
546
641
|
this.registerProvider('internal', mime);
|
|
547
642
|
break;
|
|
@@ -610,7 +705,7 @@ class Library {
|
|
|
610
705
|
return;
|
|
611
706
|
}
|
|
612
707
|
|
|
613
|
-
const loinc = new LoincServicesFactory(this.i18n, loincFN);
|
|
708
|
+
const loinc = new LoincServicesFactory(this.i18n, loincFN, { readOnly: this.terminologyDatabasesReadOnly });
|
|
614
709
|
await loinc.load();
|
|
615
710
|
this.registerProvider(loincFN, loinc, isDefault);
|
|
616
711
|
this.#syndicateBinary('loinc', loincFN, loinc);
|
|
@@ -621,7 +716,7 @@ class Library {
|
|
|
621
716
|
if (mode === "fetch" || mode === "npm") {
|
|
622
717
|
return;
|
|
623
718
|
}
|
|
624
|
-
const rxn = new RxNormServicesFactory(this.i18n, rxNormFN);
|
|
719
|
+
const rxn = new RxNormServicesFactory(this.i18n, rxNormFN, { readOnly: this.terminologyDatabasesReadOnly });
|
|
625
720
|
await rxn.load();
|
|
626
721
|
this.registerProvider(rxNormFN, rxn, isDefault);
|
|
627
722
|
this.#syndicateBinary('rxnorm', rxNormFN, rxn);
|
|
@@ -632,7 +727,7 @@ class Library {
|
|
|
632
727
|
if (mode === "fetch" || mode === "npm") {
|
|
633
728
|
return;
|
|
634
729
|
}
|
|
635
|
-
const ndc = new NdcServicesFactory(this.i18n, ndcFN);
|
|
730
|
+
const ndc = new NdcServicesFactory(this.i18n, ndcFN, { readOnly: this.terminologyDatabasesReadOnly });
|
|
636
731
|
await ndc.load();
|
|
637
732
|
this.registerProvider(ndcFN, ndc, isDefault);
|
|
638
733
|
this.#syndicateBinary('ndc', ndcFN, ndc);
|
|
@@ -643,7 +738,7 @@ class Library {
|
|
|
643
738
|
if (mode === "fetch" || mode === "npm") {
|
|
644
739
|
return;
|
|
645
740
|
}
|
|
646
|
-
const unii = new UniiServicesFactory(this.i18n, uniFN);
|
|
741
|
+
const unii = new UniiServicesFactory(this.i18n, uniFN, { readOnly: this.terminologyDatabasesReadOnly });
|
|
647
742
|
await unii.load();
|
|
648
743
|
this.registerProvider(uniFN, unii, isDefault);
|
|
649
744
|
this.#syndicateBinary('unii', uniFN, unii);
|
|
@@ -654,8 +749,18 @@ class Library {
|
|
|
654
749
|
if (mode === "fetch" || mode === "npm") {
|
|
655
750
|
return;
|
|
656
751
|
}
|
|
657
|
-
const
|
|
752
|
+
const loadMode = this.snomedLoad.modeFor(sctFN, SnomedFileReader.versionUri(sctFN));
|
|
753
|
+
const sct = new SnomedServicesFactory(this.i18n, sctFN, loadMode, this.log);
|
|
658
754
|
await sct.load();
|
|
755
|
+
if (loadMode !== 'read') {
|
|
756
|
+
// if it could not be mapped, mapFile has already said why, through this.log
|
|
757
|
+
this.log.info(`SNOMED CT ${path.basename(sctFN)}: load ${loadMode}, ` +
|
|
758
|
+
(sct.mapped ? 'memory-mapped' : 'not mapped, read into memory'));
|
|
759
|
+
}
|
|
760
|
+
if (sct.mapped && !memoryGuard._state.excludeFileBacked) {
|
|
761
|
+
memoryGuard.measureExcludingFileBacked();
|
|
762
|
+
this.log.info(`SNOMED CT cache files are memory-mapped. ${memoryGuard.describe()}`);
|
|
763
|
+
}
|
|
659
764
|
this.registerProvider(sctFN, sct, isDefault);
|
|
660
765
|
this.#syndicateBinary('snomed', sctFN, sct);
|
|
661
766
|
}
|
|
@@ -665,7 +770,7 @@ class Library {
|
|
|
665
770
|
if (mode === "fetch" || mode === "npm") {
|
|
666
771
|
return;
|
|
667
772
|
}
|
|
668
|
-
const cpt = new CPTServicesFactory(this.i18n, cptFN);
|
|
773
|
+
const cpt = new CPTServicesFactory(this.i18n, cptFN, { readOnly: this.terminologyDatabasesReadOnly });
|
|
669
774
|
await cpt.load();
|
|
670
775
|
this.registerProvider(cptFN, cpt, isDefault);
|
|
671
776
|
// the store syndicates the public fragment only, never the full (licensed) database
|
|
@@ -677,7 +782,7 @@ class Library {
|
|
|
677
782
|
if (mode === "fetch" || mode === "npm") {
|
|
678
783
|
return;
|
|
679
784
|
}
|
|
680
|
-
const omop = new OMOPServicesFactory(this.i18n, omopFN);
|
|
785
|
+
const omop = new OMOPServicesFactory(this.i18n, omopFN, { readOnly: this.terminologyDatabasesReadOnly });
|
|
681
786
|
await omop.load();
|
|
682
787
|
this.registerProvider(omopFN, omop, isDefault);
|
|
683
788
|
this.#syndicateBinary('omop', omopFN, omop);
|
|
@@ -831,10 +936,10 @@ class Library {
|
|
|
831
936
|
this.codeSystemProviders.push(cp);
|
|
832
937
|
let vs = null;
|
|
833
938
|
if (!csOnly) {
|
|
834
|
-
vs = new PackageValueSetProvider(contentLoader);
|
|
939
|
+
vs = new PackageValueSetProvider(contentLoader, { readOnly: this.packageCachesReadOnly });
|
|
835
940
|
await vs.initialize();
|
|
836
941
|
this.valueSetProviders.push(vs);
|
|
837
|
-
const cm = new PackageConceptMapProvider(contentLoader);
|
|
942
|
+
const cm = new PackageConceptMapProvider(contentLoader, { readOnly: this.packageCachesReadOnly });
|
|
838
943
|
await cm.initialize();
|
|
839
944
|
this.conceptMapProviders.push(cm);
|
|
840
945
|
}
|
|
@@ -861,7 +966,7 @@ class Library {
|
|
|
861
966
|
}
|
|
862
967
|
|
|
863
968
|
const folderPath = this.#resolveLocalSourcePath(details);
|
|
864
|
-
const cacheSubdir = await this.#prepareLocalSourceCache(folderPath);
|
|
969
|
+
const cacheSubdir = await this.#prepareLocalSourceCache(folderPath, dir => new FolderContentLoader(folderPath, dir));
|
|
865
970
|
await this.#loadLocalSource(new FolderContentLoader(folderPath, cacheSubdir));
|
|
866
971
|
}
|
|
867
972
|
|
|
@@ -886,7 +991,7 @@ class Library {
|
|
|
886
991
|
}
|
|
887
992
|
|
|
888
993
|
const filePath = this.#resolveLocalSourcePath(details);
|
|
889
|
-
const cacheSubdir = await this.#prepareLocalSourceCache(filePath);
|
|
994
|
+
const cacheSubdir = await this.#prepareLocalSourceCache(filePath, dir => new FileContentLoader(filePath, dir));
|
|
890
995
|
await this.#loadLocalSource(new FileContentLoader(filePath, cacheSubdir));
|
|
891
996
|
}
|
|
892
997
|
|
|
@@ -906,16 +1011,207 @@ class Library {
|
|
|
906
1011
|
* Parks the Package*Provider SQLite caches under the terminology cache rather
|
|
907
1012
|
* than polluting the user's source folder, and wipes between runs so edits to
|
|
908
1013
|
* the source are reliably picked up.
|
|
1014
|
+
*
|
|
1015
|
+
* In a cluster (localSourceTag set) every process uses the master's folder:
|
|
1016
|
+
* - a worker (packageCachesReadOnly) touches nothing; its providers open the caches
|
|
1017
|
+
* read-only, waiting for the master to have built them
|
|
1018
|
+
* - the master builds the caches in a temporary sibling, then renames it into place, so a
|
|
1019
|
+
* running worker never finds the folder half built
|
|
1020
|
+
*
|
|
1021
|
+
* @param {string} sourcePath
|
|
1022
|
+
* @param {function(string): FolderContentLoader} makeLoader - a loader with its caches in a folder
|
|
1023
|
+
* @returns {Promise<string>} the cache folder
|
|
909
1024
|
* @private
|
|
910
1025
|
*/
|
|
911
|
-
async #prepareLocalSourceCache(sourcePath) {
|
|
1026
|
+
async #prepareLocalSourceCache(sourcePath, makeLoader) {
|
|
912
1027
|
const hash = crypto.createHash('sha1').update(sourcePath).digest('hex').substring(0, 16);
|
|
913
|
-
const
|
|
914
|
-
|
|
915
|
-
|
|
1028
|
+
const name = LOCAL_SOURCE_CACHE_PREFIX + (this.localSourceTag ? this.localSourceTag + '-' : '') + hash;
|
|
1029
|
+
const cacheSubdir = path.join(this.cacheFolder, name);
|
|
1030
|
+
this.#localSourceCaches.add(name);
|
|
1031
|
+
if (this.packageCachesReadOnly) {
|
|
1032
|
+
return cacheSubdir;
|
|
1033
|
+
}
|
|
1034
|
+
if (!this.localSourceTag) {
|
|
1035
|
+
await fs.rm(cacheSubdir, { recursive: true, force: true });
|
|
1036
|
+
await fs.mkdir(cacheSubdir, { recursive: true });
|
|
1037
|
+
return cacheSubdir;
|
|
1038
|
+
}
|
|
1039
|
+
const building = `${cacheSubdir}.building-${process.pid}`;
|
|
1040
|
+
const old = `${cacheSubdir}.old-${process.pid}`;
|
|
1041
|
+
await fs.rm(building, { recursive: true, force: true });
|
|
1042
|
+
await fs.rm(old, { recursive: true, force: true });
|
|
1043
|
+
await fs.mkdir(building, { recursive: true });
|
|
1044
|
+
const loader = makeLoader(building);
|
|
1045
|
+
await loader.initialize();
|
|
1046
|
+
const vs = new PackageValueSetProvider(loader);
|
|
1047
|
+
await vs.initialize();
|
|
1048
|
+
await vs.close();
|
|
1049
|
+
const cm = new PackageConceptMapProvider(loader);
|
|
1050
|
+
await cm.initialize();
|
|
1051
|
+
// a directory can't be renamed over a non-empty one: move the old one aside first. A
|
|
1052
|
+
// worker that looks in between finds no folder, and waits for it
|
|
1053
|
+
try {
|
|
1054
|
+
await fs.rename(cacheSubdir, old);
|
|
1055
|
+
} catch (e) {
|
|
1056
|
+
if (e.code !== 'ENOENT') {
|
|
1057
|
+
throw e;
|
|
1058
|
+
}
|
|
1059
|
+
}
|
|
1060
|
+
await fs.rename(building, cacheSubdir);
|
|
1061
|
+
await fs.rm(old, { recursive: true, force: true });
|
|
916
1062
|
return cacheSubdir;
|
|
917
1063
|
}
|
|
918
1064
|
|
|
1065
|
+
/**
|
|
1066
|
+
* Removes the folder-source-* caches left by sources that are no longer loaded (the
|
|
1067
|
+
* name is a hash of the source path, so a moved or temporary folder leaves one behind).
|
|
1068
|
+
*
|
|
1069
|
+
* The cache folder can be shared by more than one Library at a time (a server and the
|
|
1070
|
+
* tests, run from the same checkout), so a cache this load didn't use may be another
|
|
1071
|
+
* one's. Every load rebuilds its caches, so a live one is always recent - only those
|
|
1072
|
+
* untouched for LOCAL_SOURCE_CACHE_MAX_AGE are removed.
|
|
1073
|
+
* @private
|
|
1074
|
+
*/
|
|
1075
|
+
async #pruneLocalSourceCaches() {
|
|
1076
|
+
let entries;
|
|
1077
|
+
try {
|
|
1078
|
+
entries = await fs.readdir(this.cacheFolder, { withFileTypes: true });
|
|
1079
|
+
} catch (e) {
|
|
1080
|
+
return;
|
|
1081
|
+
}
|
|
1082
|
+
const cutoff = Date.now() - LOCAL_SOURCE_CACHE_MAX_AGE;
|
|
1083
|
+
let count = 0;
|
|
1084
|
+
for (const entry of entries) {
|
|
1085
|
+
if (!entry.isDirectory() || !entry.name.startsWith(LOCAL_SOURCE_CACHE_PREFIX) || this.#localSourceCaches.has(entry.name)) {
|
|
1086
|
+
continue;
|
|
1087
|
+
}
|
|
1088
|
+
const dir = path.join(this.cacheFolder, entry.name);
|
|
1089
|
+
try {
|
|
1090
|
+
const stat = await fs.stat(dir);
|
|
1091
|
+
if (stat.mtimeMs < cutoff) {
|
|
1092
|
+
await fs.rm(dir, { recursive: true, force: true });
|
|
1093
|
+
count++;
|
|
1094
|
+
}
|
|
1095
|
+
} catch (e) {
|
|
1096
|
+
this.log.warn(`Unable to remove stale local source cache ${dir}: ${e.message}`);
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
if (count > 0) {
|
|
1100
|
+
this.log.info(`Removed ${count} stale local source cache folder${count === 1 ? '' : 's'}`);
|
|
1101
|
+
}
|
|
1102
|
+
}
|
|
1103
|
+
|
|
1104
|
+
/**
|
|
1105
|
+
* With pruneTerminologyCacheFiles, and in a process that owns the caches (not a tx-worker):
|
|
1106
|
+
* delete what's in the cache folder that wasn't used - the .db and .cache files
|
|
1107
|
+
* (pruneTerminologyCache) and the package folders (prunePackages). Call it once everything is
|
|
1108
|
+
* loaded: the library (load) and every endpoint's core packages (cloneWithFhirVersion)
|
|
1109
|
+
* @returns {Promise<{files: string[], packages: string[]}>} what was deleted
|
|
1110
|
+
*/
|
|
1111
|
+
async pruneUnusedCaches() {
|
|
1112
|
+
if (!this.pruneTerminologyCacheFiles || this.packageCachesReadOnly) {
|
|
1113
|
+
return { files: [], packages: [] };
|
|
1114
|
+
}
|
|
1115
|
+
return { files: await this.pruneTerminologyCache(), packages: await this.prunePackages() };
|
|
1116
|
+
}
|
|
1117
|
+
|
|
1118
|
+
/**
|
|
1119
|
+
* Deletes the package folders (id#version) in the cache folder that the package manager
|
|
1120
|
+
* hasn't returned this run (see PackageManager.usedPackages): packages no longer in the
|
|
1121
|
+
* library, older versions of ones that are, the core packages of FHIR versions no endpoint
|
|
1122
|
+
* has any more, and what failed or interrupted downloads left behind. Their value set and
|
|
1123
|
+
* concept map caches are in them, so go with them. A folder changed in the last
|
|
1124
|
+
* PRUNE_MIN_AGE is kept.
|
|
1125
|
+
* @returns {Promise<string[]>} the names of the folders deleted
|
|
1126
|
+
*/
|
|
1127
|
+
async prunePackages() {
|
|
1128
|
+
let entries;
|
|
1129
|
+
try {
|
|
1130
|
+
entries = await fs.readdir(this.cacheFolder, { withFileTypes: true });
|
|
1131
|
+
} catch (e) {
|
|
1132
|
+
return [];
|
|
1133
|
+
}
|
|
1134
|
+
const used = this.packageManager.usedPackages;
|
|
1135
|
+
const cutoff = Date.now() - PRUNE_MIN_AGE;
|
|
1136
|
+
const deleted = [];
|
|
1137
|
+
let bytes = 0;
|
|
1138
|
+
for (const entry of entries) {
|
|
1139
|
+
if (!entry.isDirectory() || !entry.name.includes('#') || used.has(entry.name)) {
|
|
1140
|
+
continue;
|
|
1141
|
+
}
|
|
1142
|
+
const dir = path.join(this.cacheFolder, entry.name);
|
|
1143
|
+
try {
|
|
1144
|
+
const stat = await fs.stat(dir);
|
|
1145
|
+
if (stat.mtimeMs >= cutoff) {
|
|
1146
|
+
this.log.info(`Not deleting unused package ${entry.name} yet: it changed in the last day`);
|
|
1147
|
+
continue;
|
|
1148
|
+
}
|
|
1149
|
+
const size = await folderSize(dir);
|
|
1150
|
+
await fs.rm(dir, { recursive: true, force: true });
|
|
1151
|
+
deleted.push(entry.name);
|
|
1152
|
+
bytes += size;
|
|
1153
|
+
this.log.info(`Deleted unused package ${entry.name} (${(size / 1024 / 1024).toFixed(1)} MB) from the terminology cache`);
|
|
1154
|
+
} catch (e) {
|
|
1155
|
+
this.log.warn(`Unable to delete unused package ${dir}: ${e.message}`);
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1158
|
+
if (deleted.length > 0) {
|
|
1159
|
+
this.log.info(`Deleted ${deleted.length} unused package${deleted.length === 1 ? '' : 's'} from the terminology cache, ${(bytes / 1024 / 1024).toFixed(1)} MB`);
|
|
1160
|
+
}
|
|
1161
|
+
return deleted;
|
|
1162
|
+
}
|
|
1163
|
+
|
|
1164
|
+
/**
|
|
1165
|
+
* Deletes the .db and .cache files directly in the cache folder (terminology-cache) that this
|
|
1166
|
+
* load didn't use: the downloaded databases and SNOMED CT caches of sources that are no longer
|
|
1167
|
+
* in the library, and the fallback of an "a|b" source whose first file is there. Their SQLite
|
|
1168
|
+
* side files (-wal, -shm, -journal) go with them. Package folders, and the other folders and
|
|
1169
|
+
* files there, are left alone.
|
|
1170
|
+
*
|
|
1171
|
+
* A file changed in the last PRUNE_MIN_AGE is kept: it may be being copied in, or built,
|
|
1172
|
+
* for a library that doesn't name it yet.
|
|
1173
|
+
*
|
|
1174
|
+
* See pruneUnusedCaches, which calls it
|
|
1175
|
+
* @returns {Promise<string[]>} the names of the files deleted
|
|
1176
|
+
*/
|
|
1177
|
+
async pruneTerminologyCache() {
|
|
1178
|
+
let entries;
|
|
1179
|
+
try {
|
|
1180
|
+
entries = await fs.readdir(this.cacheFolder, { withFileTypes: true });
|
|
1181
|
+
} catch (e) {
|
|
1182
|
+
return [];
|
|
1183
|
+
}
|
|
1184
|
+
const cutoff = Date.now() - PRUNE_MIN_AGE;
|
|
1185
|
+
const deleted = [];
|
|
1186
|
+
let bytes = 0;
|
|
1187
|
+
for (const entry of entries) {
|
|
1188
|
+
if (!entry.isFile() || !/\.(db|cache)$/i.test(entry.name) || this.#usedCacheFiles.has(entry.name)) {
|
|
1189
|
+
continue;
|
|
1190
|
+
}
|
|
1191
|
+
const file = path.join(this.cacheFolder, entry.name);
|
|
1192
|
+
try {
|
|
1193
|
+
const stat = await fs.stat(file);
|
|
1194
|
+
if (stat.mtimeMs >= cutoff) {
|
|
1195
|
+
this.log.info(`Not deleting unused ${entry.name} yet: it changed in the last day`);
|
|
1196
|
+
continue;
|
|
1197
|
+
}
|
|
1198
|
+
await fs.rm(file, { force: true });
|
|
1199
|
+
for (const suffix of ['-wal', '-shm', '-journal']) {
|
|
1200
|
+
await fs.rm(file + suffix, { force: true });
|
|
1201
|
+
}
|
|
1202
|
+
deleted.push(entry.name);
|
|
1203
|
+
bytes += stat.size;
|
|
1204
|
+
this.log.info(`Deleted unused ${entry.name} (${(stat.size / 1024 / 1024).toFixed(1)} MB) from the terminology cache`);
|
|
1205
|
+
} catch (e) {
|
|
1206
|
+
this.log.warn(`Unable to delete unused ${file}: ${e.message}`);
|
|
1207
|
+
}
|
|
1208
|
+
}
|
|
1209
|
+
if (deleted.length > 0) {
|
|
1210
|
+
this.log.info(`Deleted ${deleted.length} unused file${deleted.length === 1 ? '' : 's'} from the terminology cache, ${(bytes / 1024 / 1024).toFixed(1)} MB`);
|
|
1211
|
+
}
|
|
1212
|
+
return deleted;
|
|
1213
|
+
}
|
|
1214
|
+
|
|
919
1215
|
/**
|
|
920
1216
|
* Shared loader for folder- and file-sourced content: registers the loader's
|
|
921
1217
|
* CodeSystems, ValueSets and ConceptMaps with this library.
|
|
@@ -972,10 +1268,10 @@ class Library {
|
|
|
972
1268
|
}
|
|
973
1269
|
this.codeSystemProviders.push(cp);
|
|
974
1270
|
|
|
975
|
-
const vs = new PackageValueSetProvider(contentLoader);
|
|
1271
|
+
const vs = new PackageValueSetProvider(contentLoader, { readOnly: this.packageCachesReadOnly });
|
|
976
1272
|
await vs.initialize();
|
|
977
1273
|
this.valueSetProviders.push(vs);
|
|
978
|
-
const cm = new PackageConceptMapProvider(contentLoader);
|
|
1274
|
+
const cm = new PackageConceptMapProvider(contentLoader, { readOnly: this.packageCachesReadOnly });
|
|
979
1275
|
await cm.initialize();
|
|
980
1276
|
this.conceptMapProviders.push(cm);
|
|
981
1277
|
|
|
@@ -1000,10 +1296,12 @@ class Library {
|
|
|
1000
1296
|
|
|
1001
1297
|
const firstPath = path.join(this.cacheFolder, firstName);
|
|
1002
1298
|
if (await this.fileExists(firstPath)) {
|
|
1299
|
+
this.#usedCacheFiles.add(path.basename(firstPath));
|
|
1003
1300
|
return firstPath;
|
|
1004
1301
|
}
|
|
1005
1302
|
}
|
|
1006
1303
|
const filePath = path.join(this.cacheFolder, fileName);
|
|
1304
|
+
this.#usedCacheFiles.add(path.basename(filePath));
|
|
1007
1305
|
|
|
1008
1306
|
// Check if file already exists
|
|
1009
1307
|
if (await this.fileExists(filePath)) {
|
|
@@ -1118,7 +1416,7 @@ class Library {
|
|
|
1118
1416
|
const provider = new Provider();
|
|
1119
1417
|
provider.i18n = this.i18n;
|
|
1120
1418
|
provider.codeSystemFactories = new Map(this.codeSystemFactories); // all of them
|
|
1121
|
-
provider.codeSystems = new
|
|
1419
|
+
provider.codeSystems = new CodeSystemMap();
|
|
1122
1420
|
provider.valueSetProviders = [];
|
|
1123
1421
|
provider.conceptMapProviders = [];
|
|
1124
1422
|
provider.path = path;
|
|
@@ -1142,7 +1440,8 @@ class Library {
|
|
|
1142
1440
|
|
|
1143
1441
|
// Load FHIR packages - these will be added to valueSetProviders first
|
|
1144
1442
|
for (const packageId of fhirPackages) {
|
|
1145
|
-
const contentLoader = await provider.loadNpm(this.packageManager, this.cacheFolder, packageId, false, "npm",
|
|
1443
|
+
const contentLoader = await provider.loadNpm(this.packageManager, this.cacheFolder, packageId, false, "npm",
|
|
1444
|
+
{ readOnly: this.packageCachesReadOnly });
|
|
1146
1445
|
// the core packages are the only version specific content in the syndication store
|
|
1147
1446
|
this.syndicatePackage(contentLoader, { fhirVersion: VersionUtilities.getMajMin(String(fhirVersion)) });
|
|
1148
1447
|
}
|
package/tx/openapi-operations.js
CHANGED
|
@@ -183,13 +183,15 @@ const OPERATIONS = [
|
|
|
183
183
|
methods: ['post'],
|
|
184
184
|
summary: 'Validate many codes against code systems',
|
|
185
185
|
description: 'A batch of CodeSystem $validate-code requests: each `validation` is a ' +
|
|
186
|
-
'Parameters resource with the parameters of one. system,
|
|
187
|
-
'lenient-display-validation at the top level apply to every
|
|
188
|
-
'have its own.',
|
|
186
|
+
'Parameters resource with the parameters of one. url (the code system), version, system, ' +
|
|
187
|
+
'codeSystem, tx-resource and lenient-display-validation at the top level apply to every ' +
|
|
188
|
+
'validation that doesn\'t have its own. Every validation is against a code system.',
|
|
189
189
|
postBody: ['Parameters'],
|
|
190
190
|
in: [
|
|
191
191
|
p('validation', 'Parameters', '*', 'The parameters of one $validate-code', { min: 1 }),
|
|
192
|
-
p('
|
|
192
|
+
p('url', 'uri', '1', 'The code system, for every validation'),
|
|
193
|
+
p('version', 'string', '1', 'The code system version, for every validation'),
|
|
194
|
+
p('system', 'uri', '1', 'The code system, if url isn\'t given, for every validation'),
|
|
193
195
|
p('codeSystem', 'CodeSystem', '1', 'The code system, for every validation'),
|
|
194
196
|
p('lenient-display-validation', 'boolean', '1', 'For every validation'),
|
|
195
197
|
p('tx-resource', 'Resource', '*', 'Resources for every validation')
|