fhirsmith 0.9.5 → 0.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/config-template.json +2 -1
- package/library/folder-content-loader.js +91 -0
- package/library/regex-utilities.js +49 -12
- package/npmprojector/npmprojector.js +2 -6
- package/package.json +2 -2
- package/publisher/publisher.js +105 -12
- package/registry/registry.js +6 -6
- package/server.js +6 -2
- package/test-scripts/repro-re2-wasm-leak.js +8 -7
- package/translations/Messages.properties +1 -1
- package/tx/cs/cs-cs.js +8 -0
- package/tx/cs/cs-loinc.js +13 -12
- package/tx/cs/cs-omop.js +24 -23
- package/tx/cs/cs-provider-list.js +2 -1
- package/tx/cs/cs-snomed.js +142 -59
- package/tx/cs/cs-unii.js +11 -11
- package/tx/data/snomed-testing.cache +0 -0
- package/tx/library/canonical-resource.js +4 -2
- package/tx/library/designations.js +27 -20
- package/tx/library/renderer.js +303 -22
- package/tx/library/ucum-types.js +4 -1
- package/tx/library.js +65 -21
- package/tx/operation-context.js +52 -23
- package/tx/params.js +36 -8
- package/tx/problems.js +0 -4
- package/tx/provider.js +7 -3
- package/tx/tx-html.js +7 -0
- package/tx/tx.js +24 -13
- package/tx/vs/vs-vsac.js +157 -9
- package/tx/workers/expand.js +100 -96
- package/tx/workers/lookup.js +6 -0
- package/tx/workers/read.js +1 -1
- package/tx/workers/translate.js +21 -29
- package/tx/workers/validate.js +18 -10
- package/tx/workers/worker.js +5 -1
- package/tx/xversion/xv-bundle.js +1 -2
- package/tx/xversion/xv-codesystem.js +5 -2
- package/tx/xversion/xv-parameters.js +4 -4
- package/tx/xversion/xv-resource.js +2 -2
- package/tx/xversion/xv-terminologyCapabilities.js +11 -6
- package/tx/xversion/xv-valueset.js +7 -7
- package/publisher/task-draft.js +0 -458
package/tx/operation-context.js
CHANGED
|
@@ -204,6 +204,10 @@ class ExpansionCache {
|
|
|
204
204
|
this.cache = new Map();
|
|
205
205
|
this.maxSize = maxSize;
|
|
206
206
|
this.memoryThresholdBytes = memoryThresholdMB * 1024 * 1024;
|
|
207
|
+
// When true, every expansion is cached regardless of how long it took
|
|
208
|
+
// (bypasses MIN_CACHE_TIME_MS). Used by the test runner to force the cache
|
|
209
|
+
// path so cache correctness (e.g. language in the key) is exercised.
|
|
210
|
+
this.forceCaching = false;
|
|
207
211
|
}
|
|
208
212
|
|
|
209
213
|
/**
|
|
@@ -284,8 +288,9 @@ class ExpansionCache {
|
|
|
284
288
|
* @returns {boolean} True if cached, false if duration too short
|
|
285
289
|
*/
|
|
286
290
|
set(key, expansion, durationMs) {
|
|
287
|
-
// Only cache if expansion took significant time
|
|
288
|
-
|
|
291
|
+
// Only cache if expansion took significant time, unless forceCaching is on
|
|
292
|
+
// (in which case everything is cached regardless of duration).
|
|
293
|
+
if (!this.forceCaching && durationMs < ExpansionCache.MIN_CACHE_TIME_MS) {
|
|
289
294
|
return false;
|
|
290
295
|
}
|
|
291
296
|
|
|
@@ -357,21 +362,6 @@ class ExpansionCache {
|
|
|
357
362
|
return false;
|
|
358
363
|
}
|
|
359
364
|
|
|
360
|
-
/**
|
|
361
|
-
* Force-store an expansion regardless of duration (for testing)
|
|
362
|
-
* @param {string} key - Hash key
|
|
363
|
-
* @param {Object} expansion - The expanded ValueSet
|
|
364
|
-
*/
|
|
365
|
-
forceSet(key, expansion) {
|
|
366
|
-
this.cache.set(key, {
|
|
367
|
-
expansion: expansion,
|
|
368
|
-
createdAt: Date.now(),
|
|
369
|
-
lastUsed: Date.now(),
|
|
370
|
-
durationMs: 0,
|
|
371
|
-
hitCount: 0
|
|
372
|
-
});
|
|
373
|
-
}
|
|
374
|
-
|
|
375
365
|
/**
|
|
376
366
|
* Clear a specific entry
|
|
377
367
|
* @param {string} key - Hash key
|
|
@@ -388,22 +378,22 @@ class ExpansionCache {
|
|
|
388
378
|
}
|
|
389
379
|
|
|
390
380
|
/**
|
|
391
|
-
* Get cache statistics
|
|
381
|
+
* Get cache statistics.
|
|
382
|
+
* NB: named getStats(), not stats() — the `stats` field (the ServerStats
|
|
383
|
+
* passed to the constructor) would shadow a method called `stats`, making it
|
|
384
|
+
* unreachable.
|
|
392
385
|
* @returns {Object} Stats object
|
|
393
386
|
*/
|
|
394
|
-
|
|
387
|
+
getStats() {
|
|
395
388
|
let totalHits = 0;
|
|
396
|
-
let totalDuration = 0;
|
|
397
389
|
for (const entry of this.cache.values()) {
|
|
398
390
|
totalHits += entry.hitCount;
|
|
399
|
-
totalDuration += entry.durationMs;
|
|
400
391
|
}
|
|
401
392
|
return {
|
|
402
393
|
size: this.cache.size,
|
|
403
394
|
maxSize: this.maxSize,
|
|
404
395
|
memoryThresholdMB: this.memoryThresholdBytes > 0 ? this.memoryThresholdBytes / (1024 * 1024) : 0,
|
|
405
|
-
totalHits
|
|
406
|
-
totalDurationSaved: totalHits > 0 ? totalDuration * totalHits : 0
|
|
396
|
+
totalHits
|
|
407
397
|
};
|
|
408
398
|
}
|
|
409
399
|
|
|
@@ -448,6 +438,11 @@ class OperationContext {
|
|
|
448
438
|
this.resourceCache = resourceCache;
|
|
449
439
|
this.expansionCache = expansionCache;
|
|
450
440
|
this.debugging = isDebugging();
|
|
441
|
+
// Providers opened during this operation that need their underlying
|
|
442
|
+
// resources (sqlite connections, etc.) released when the operation ends.
|
|
443
|
+
// Shared by reference with copy()'d contexts so a sub-operation's
|
|
444
|
+
// providers are cleaned up by the parent request's closeProviders().
|
|
445
|
+
this._openProviders = [];
|
|
451
446
|
|
|
452
447
|
this.timeTracker.step('tx-op');
|
|
453
448
|
}
|
|
@@ -476,6 +471,9 @@ class OperationContext {
|
|
|
476
471
|
newContext.logEntries = [...this.logEntries];
|
|
477
472
|
newContext.debugging = this.debugging;
|
|
478
473
|
newContext.usageTracker = this.usageTracker;
|
|
474
|
+
// Share the same provider-cleanup list so providers opened by the copy
|
|
475
|
+
// are released when the parent operation ends.
|
|
476
|
+
newContext._openProviders = this._openProviders;
|
|
479
477
|
return newContext;
|
|
480
478
|
}
|
|
481
479
|
|
|
@@ -624,6 +622,37 @@ class OperationContext {
|
|
|
624
622
|
return this.id;
|
|
625
623
|
}
|
|
626
624
|
|
|
625
|
+
/**
|
|
626
|
+
* Register a code-system provider whose resources (typically a sqlite
|
|
627
|
+
* connection opened by factory.build()) should be released when the
|
|
628
|
+
* operation ends. Providers without a close() method are ignored.
|
|
629
|
+
* @param {Object} provider - The provider returned from factory.build()
|
|
630
|
+
*/
|
|
631
|
+
registerProvider(provider) {
|
|
632
|
+
if (provider && typeof provider.close === 'function') {
|
|
633
|
+
this._openProviders.push(provider);
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
/**
|
|
638
|
+
* Close every provider registered during this operation. Safe to call
|
|
639
|
+
* multiple times — the list is cleared after the first call. Errors
|
|
640
|
+
* from individual close() calls are swallowed so one bad provider can't
|
|
641
|
+
* prevent the others from releasing their resources.
|
|
642
|
+
*/
|
|
643
|
+
async closeProviders() {
|
|
644
|
+
if (!this._openProviders || this._openProviders.length === 0) return;
|
|
645
|
+
const providers = this._openProviders;
|
|
646
|
+
this._openProviders = [];
|
|
647
|
+
for (const p of providers) {
|
|
648
|
+
try {
|
|
649
|
+
await p.close();
|
|
650
|
+
} catch (_e) {
|
|
651
|
+
// Swallow — provider cleanup is best-effort.
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
|
|
627
656
|
/**
|
|
628
657
|
* @type {Languages} languages specified in request
|
|
629
658
|
*/
|
package/tx/params.js
CHANGED
|
@@ -53,6 +53,11 @@ class TxParameters {
|
|
|
53
53
|
|
|
54
54
|
this.FHTTPLanguages = null;
|
|
55
55
|
this.FDisplayLanguages = null;
|
|
56
|
+
// Whether languages were explicitly supplied by the request (vs a
|
|
57
|
+
// synthesised default). Consumed by hasHTTPLanguages/hasDisplayLanguages so
|
|
58
|
+
// the requested language folds into the expansion cache key.
|
|
59
|
+
this.FHasHTTPLanguages = false;
|
|
60
|
+
this.FHasDisplayLanguages = false;
|
|
56
61
|
this.FValueSetVersionRules = null;
|
|
57
62
|
this.FUid = '';
|
|
58
63
|
|
|
@@ -87,11 +92,15 @@ class TxParameters {
|
|
|
87
92
|
if (!params.parameter) {
|
|
88
93
|
return;
|
|
89
94
|
}
|
|
90
|
-
if (
|
|
91
|
-
|
|
95
|
+
if (this.hasParam(params, "__Content-Language")) {
|
|
96
|
+
const lang = this.paramstr(params, "__Content-Language");
|
|
97
|
+
this.HTTPLanguages = Languages.fromAcceptLanguage(lang, this.languageDefinitions, !this.validating);
|
|
98
|
+
if (lang) this.FHasHTTPLanguages = true;
|
|
92
99
|
}
|
|
93
|
-
if (
|
|
94
|
-
|
|
100
|
+
if (this.hasParam(params, "__Accept-Language")) {
|
|
101
|
+
const lang = this.paramstr(params, "__Accept-Language");
|
|
102
|
+
this.HTTPLanguages = Languages.fromAcceptLanguage(lang, this.languageDefinitions, !this.validating);
|
|
103
|
+
if (lang) this.FHasHTTPLanguages = true;
|
|
95
104
|
}
|
|
96
105
|
|
|
97
106
|
for (let p of params.parameter) {
|
|
@@ -124,7 +133,9 @@ class TxParameters {
|
|
|
124
133
|
|
|
125
134
|
case 'displayLanguage': {
|
|
126
135
|
try {
|
|
127
|
-
|
|
136
|
+
const lang = getValuePrimitive(p);
|
|
137
|
+
this.DisplayLanguages = Languages.fromAcceptLanguage(lang, this.languageDefinitions, !this.validating);
|
|
138
|
+
if (lang) this.FHasDisplayLanguages = true;
|
|
128
139
|
} catch (error) {
|
|
129
140
|
throw new Issue("error", "processing", null, 'INVALID_DISPLAY_NAME', this.i18n.translate('INVALID_DISPLAY_NAME', this.HTTPLanguages, [getValuePrimitive(p)]), "invalid-display").handleAsOO(400);
|
|
130
141
|
}
|
|
@@ -139,7 +150,10 @@ class TxParameters {
|
|
|
139
150
|
break;
|
|
140
151
|
}
|
|
141
152
|
case 'no-cache': {
|
|
142
|
-
|
|
153
|
+
// Write FUid (the field the cache key reads via hashSource); writing
|
|
154
|
+
// `this.uid` was a no-op so no-cache=true never busted the cache.
|
|
155
|
+
// Accept both the string ('true') and boolean (valueBoolean) forms.
|
|
156
|
+
if (strToBool(getValuePrimitive(p), false)) this.FUid = crypto.randomUUID();
|
|
143
157
|
break;
|
|
144
158
|
}
|
|
145
159
|
case '_incomplete':
|
|
@@ -292,11 +306,11 @@ class TxParameters {
|
|
|
292
306
|
}
|
|
293
307
|
|
|
294
308
|
get hasHTTPLanguages() {
|
|
295
|
-
return this.
|
|
309
|
+
return this.FHasHTTPLanguages;
|
|
296
310
|
}
|
|
297
311
|
|
|
298
312
|
get hasDisplayLanguages() {
|
|
299
|
-
return this.
|
|
313
|
+
return this.FHasDisplayLanguages;
|
|
300
314
|
}
|
|
301
315
|
|
|
302
316
|
get hasDesignations() {
|
|
@@ -422,6 +436,7 @@ e
|
|
|
422
436
|
if (value) {
|
|
423
437
|
if (name === 'displayLanguage' && (!this.FDisplayLanguages || overwrite)) {
|
|
424
438
|
this.DisplayLanguages = Languages.fromAcceptLanguage(getValuePrimitive(value), this.languageDefinitions, !this.validating)
|
|
439
|
+
if (getValuePrimitive(value)) this.FHasDisplayLanguages = true;
|
|
425
440
|
}
|
|
426
441
|
|
|
427
442
|
if (name === 'designation') {
|
|
@@ -561,6 +576,17 @@ e
|
|
|
561
576
|
if (this.hasDesignations) {
|
|
562
577
|
s = s + this.FDesignations.join(',') + '|';
|
|
563
578
|
}
|
|
579
|
+
if (this.supplements && this.supplements.size > 0) {
|
|
580
|
+
// useSupplement changes the expansion result (and a bad supplement must
|
|
581
|
+
// error), so it must be part of the cache key. Sort for determinism.
|
|
582
|
+
s = s + '$' + [...this.supplements].sort().join(',') + '|';
|
|
583
|
+
}
|
|
584
|
+
// Further result-affecting parameters that were previously omitted from the
|
|
585
|
+
// key: the text filter (changes which codes expand), limited/incomplete
|
|
586
|
+
// expansion handling, whether abstract codes are included, and diagnostics.
|
|
587
|
+
// filter is free text, so JSON.stringify it to avoid delimiter collisions.
|
|
588
|
+
s = s + 'f:' + JSON.stringify(this.filter || '') + '|' +
|
|
589
|
+
b(this.limitedExpansion) + b(this.incompleteOK) + b(this.abstractOk) + b(this.diagnostics);
|
|
564
590
|
for (let t of this.FVersionRules) {
|
|
565
591
|
s = s + t.asString() + '|';
|
|
566
592
|
}
|
|
@@ -623,9 +649,11 @@ e
|
|
|
623
649
|
|
|
624
650
|
if (other.FHTTPLanguages) {
|
|
625
651
|
this.FHTTPLanguages = other.FHTTPLanguages;
|
|
652
|
+
this.FHasHTTPLanguages = this.FHasHTTPLanguages || other.FHasHTTPLanguages;
|
|
626
653
|
}
|
|
627
654
|
if (other.FDisplayLanguages) {
|
|
628
655
|
this.FDisplayLanguages = other.FDisplayLanguages;
|
|
656
|
+
this.FHasDisplayLanguages = this.FHasDisplayLanguages || other.FHasDisplayLanguages;
|
|
629
657
|
}
|
|
630
658
|
}
|
|
631
659
|
|
package/tx/problems.js
CHANGED
|
@@ -2,10 +2,6 @@ const escape = require('escape-html');
|
|
|
2
2
|
|
|
3
3
|
class ProblemFinder {
|
|
4
4
|
|
|
5
|
-
constructor() {
|
|
6
|
-
this.map = new Map();
|
|
7
|
-
}
|
|
8
|
-
|
|
9
5
|
async scanValueSets(provider) {
|
|
10
6
|
let unknownVersions = {}; // system -> Set of versions not known to the server
|
|
11
7
|
for (let vsp of provider.valueSetProviders) {
|
package/tx/provider.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
const { CodeSystem } = require("./library/codesystem");
|
|
2
2
|
const {VersionUtilities} = require("../library/version-utilities");
|
|
3
3
|
const { FhirCodeSystemProvider} = require("./cs/cs-cs");
|
|
4
|
-
const {OperationContext
|
|
4
|
+
const {OperationContext} = require("./operation-context");
|
|
5
|
+
const {TerminologyError} = require("./library/errors");
|
|
5
6
|
const {validateParameter, validateOptionalParameter, validateArrayParameter} = require("../library/utilities");
|
|
6
7
|
const path = require("path");
|
|
7
8
|
const {PackageContentLoader} = require("../library/package-manager");
|
|
@@ -419,6 +420,7 @@ class Provider {
|
|
|
419
420
|
}
|
|
420
421
|
if (factory != null) {
|
|
421
422
|
const csp = await factory.build(opContext, []);
|
|
423
|
+
opContext.registerProvider(csp);
|
|
422
424
|
const c = csp ? csp.locate(code) : null;
|
|
423
425
|
if (c) {
|
|
424
426
|
if (factory.iteratable()) {
|
|
@@ -497,14 +499,16 @@ class Provider {
|
|
|
497
499
|
deleteCodeSystem(cs) {
|
|
498
500
|
this.codeSystems.delete(cs.vurl);
|
|
499
501
|
this.codeSystems.delete(cs.url);
|
|
502
|
+
// If other versions of the SAME url survive, re-point the unversioned [url]
|
|
503
|
+
// entry at the most-recent surviving version. Otherwise leave it deleted.
|
|
500
504
|
let existing = null;
|
|
501
505
|
for (let t of this.codeSystems.values()) {
|
|
502
|
-
if (!existing || t.isMoreRecent(existing)) {
|
|
506
|
+
if (t.url === cs.url && (!existing || t.isMoreRecent(existing))) {
|
|
503
507
|
existing = t;
|
|
504
508
|
}
|
|
505
509
|
}
|
|
506
510
|
if (existing) {
|
|
507
|
-
this.codeSystems.set(
|
|
511
|
+
this.codeSystems.set(existing.url, existing);
|
|
508
512
|
}
|
|
509
513
|
}
|
|
510
514
|
|
package/tx/tx-html.js
CHANGED
|
@@ -19,6 +19,7 @@ const {TerminologyCapabilitiesXML} = require("./xml/terminologycapabilities-xml"
|
|
|
19
19
|
const {ParametersXML} = require("./xml/parameters-xml");
|
|
20
20
|
const {OperationOutcomeXML} = require("./xml/operationoutcome-xml");
|
|
21
21
|
const {debugLog} = require("./operation-context");
|
|
22
|
+
const {InvalidError} = require("./library/errors");
|
|
22
23
|
|
|
23
24
|
const txHtmlLog = Logger.getInstance().child({ module: 'tx-html' });
|
|
24
25
|
|
|
@@ -311,6 +312,12 @@ class TxHtmlRenderer {
|
|
|
311
312
|
return await this.buildHomePage(req);
|
|
312
313
|
} else {
|
|
313
314
|
try {
|
|
315
|
+
if (json === null || json === undefined || typeof json !== 'object' || Array.isArray(json)) {
|
|
316
|
+
throw new InvalidError(`Cannot render: expected a FHIR resource object but got ${json === null ? 'null' : (Array.isArray(json) ? 'an array' : typeof json)}`);
|
|
317
|
+
}
|
|
318
|
+
if (json.resourceType === undefined || json.resourceType === null || typeof json.resourceType !== 'string' || json.resourceType === '') {
|
|
319
|
+
throw new InvalidError(`Cannot render: resource has no resourceType (got ${json.resourceType === undefined ? 'undefined' : JSON.stringify(json.resourceType)})`);
|
|
320
|
+
}
|
|
314
321
|
const _fmt = req?.query?._format || req?.query?.format || req?.body?._format;
|
|
315
322
|
const op = req ? req.path.includes("$") : false;
|
|
316
323
|
const resourceType = json.resourceType;
|
package/tx/tx.js
CHANGED
|
@@ -306,6 +306,18 @@ class TXModule {
|
|
|
306
306
|
req.txI18n = this.i18n;
|
|
307
307
|
req.txLog = this.log;
|
|
308
308
|
|
|
309
|
+
// Release any code-system providers that opened sqlite connections
|
|
310
|
+
// during this request. closeProviders() is idempotent so it's safe
|
|
311
|
+
// for both events to fire. Listeners are sync; the close itself
|
|
312
|
+
// runs fire-and-forget on the event loop.
|
|
313
|
+
const releaseProviders = () => {
|
|
314
|
+
opContext.closeProviders().catch((err) => {
|
|
315
|
+
try { this.log.warn(`closeProviders failed: ${err && err.message}`); } catch (_) { /* ignore */ }
|
|
316
|
+
});
|
|
317
|
+
};
|
|
318
|
+
res.on('finish', releaseProviders);
|
|
319
|
+
res.on('close', releaseProviders);
|
|
320
|
+
|
|
309
321
|
// Add X-Request-Id header to response
|
|
310
322
|
res.setHeader('X-Request-Id', requestId);
|
|
311
323
|
|
|
@@ -370,16 +382,10 @@ class TXModule {
|
|
|
370
382
|
next();
|
|
371
383
|
});
|
|
372
384
|
|
|
373
|
-
// CORS
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
res.header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
|
|
378
|
-
if (req.method === 'OPTIONS') {
|
|
379
|
-
return res.sendStatus(200);
|
|
380
|
-
}
|
|
381
|
-
next();
|
|
382
|
-
});
|
|
385
|
+
// CORS is handled once at the app level (server.js, from config.server.cors).
|
|
386
|
+
// Do not set Access-Control-* headers here - doing so stacks a second
|
|
387
|
+
// CORS layer and produces duplicate, conflicting headers that browsers
|
|
388
|
+
// reject.
|
|
383
389
|
|
|
384
390
|
// JSON body parsing - accept both application/json and application/fhir+json
|
|
385
391
|
// Handle body that may already be read as a Buffer by app-level middleware
|
|
@@ -968,7 +974,9 @@ class TXModule {
|
|
|
968
974
|
});
|
|
969
975
|
|
|
970
976
|
// External source info pages
|
|
971
|
-
|
|
977
|
+
// GET renders the info page; POST lets a source's info() handle a form
|
|
978
|
+
// submission (e.g. VSAC on-demand resync) and re-renders the same page.
|
|
979
|
+
const infoHandler = async (req, res) => {
|
|
972
980
|
const start = Date.now();
|
|
973
981
|
try {
|
|
974
982
|
const source = req.txEndpoint.provider.externalSources.find(s => s.id() === req.params.id);
|
|
@@ -988,7 +996,9 @@ class TXModule {
|
|
|
988
996
|
} finally {
|
|
989
997
|
this.countRequest('info', Date.now() - start);
|
|
990
998
|
}
|
|
991
|
-
}
|
|
999
|
+
};
|
|
1000
|
+
router.get('/info/:id', infoHandler);
|
|
1001
|
+
router.post('/info/:id', infoHandler);
|
|
992
1002
|
}
|
|
993
1003
|
|
|
994
1004
|
/**
|
|
@@ -1119,8 +1129,9 @@ class TXModule {
|
|
|
1119
1129
|
|
|
1120
1130
|
convertResourceToXml(res) {
|
|
1121
1131
|
switch (res.resourceType) {
|
|
1122
|
-
case "CodeSystem" : return CodeSystemXML.
|
|
1132
|
+
case "CodeSystem" : return CodeSystemXML.toXml(res);
|
|
1123
1133
|
case "ValueSet" : return ValueSetXML.toXml(res);
|
|
1134
|
+
case "ConceptMap" : return ConceptMapXML.toXml(res);
|
|
1124
1135
|
case "Bundle" : return BundleXML.toXml(res, this.fhirVersion);
|
|
1125
1136
|
case "CapabilityStatement" : return CapabilityStatementXML.toXml(res, "R5");
|
|
1126
1137
|
case "TerminologyCapabilities" : return TerminologyCapabilitiesXML.toXml(res, "R5");
|
package/tx/vs/vs-vsac.js
CHANGED
|
@@ -7,6 +7,12 @@ const { VersionUtilities } = require('../../library/version-utilities');
|
|
|
7
7
|
const folders = require('../../library/folder-setup');
|
|
8
8
|
const {debugLog} = require("../operation-context");
|
|
9
9
|
|
|
10
|
+
// Persisted watermark for the phase-1b _lastUpdated scan.
|
|
11
|
+
const VSAC_LAST_UPDATED_KEY = 'vsac_last_updated_date';
|
|
12
|
+
|
|
13
|
+
// Canonical URL prefix for VSAC value sets, so operators can enter a bare OID.
|
|
14
|
+
const VSAC_VALUESET_URL_PREFIX = 'http://cts.nlm.nih.gov/fhir/ValueSet/';
|
|
15
|
+
|
|
10
16
|
/**
|
|
11
17
|
* VSAC (Value Set Authority Center) ValueSet provider
|
|
12
18
|
* Fetches and caches ValueSets from the NLM VSAC FHIR server
|
|
@@ -21,6 +27,8 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
|
|
|
21
27
|
* @param {number} [config.refreshIntervalHours=24] - Hours between refresh scans
|
|
22
28
|
* @param {string} [config.baseUrl='http://cts.nlm.nih.gov/fhir'] - Base URL for VSAC FHIR server
|
|
23
29
|
* @param {number} [config.timeoutMs=120000] - HTTP request timeout in milliseconds
|
|
30
|
+
* @param {string} [config.resyncPassword] - If set, enables the operator "resync a ValueSet"
|
|
31
|
+
* form on the /info page, gated by this password. If unset, the form is not offered.
|
|
24
32
|
*/
|
|
25
33
|
constructor(config, stats) {
|
|
26
34
|
super();
|
|
@@ -31,6 +39,8 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
|
|
|
31
39
|
}
|
|
32
40
|
|
|
33
41
|
this.apiKey = config.apiKey;
|
|
42
|
+
// Optional operator password for the on-demand resync form (see info()).
|
|
43
|
+
this.resyncPassword = config.resyncPassword || null;
|
|
34
44
|
this.cacheFolder = folders.ensureFilePath("terminology-cache/vsac");
|
|
35
45
|
this.baseUrl = config.baseUrl || 'http://cts.nlm.nih.gov/fhir';
|
|
36
46
|
this.refreshIntervalHours = config.refreshIntervalHours || 24;
|
|
@@ -127,6 +137,7 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
|
|
|
127
137
|
return;
|
|
128
138
|
}
|
|
129
139
|
this.queue = [];
|
|
140
|
+
this._pendingLastUpdated = null;
|
|
130
141
|
|
|
131
142
|
this.isRefreshing = true;
|
|
132
143
|
const runId = await this.database.startRun();
|
|
@@ -185,6 +196,9 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
|
|
|
185
196
|
this.queue = [...new Set(this.queue)];
|
|
186
197
|
|
|
187
198
|
let tracking = { totalFetched: 0, totalNew: 0, totalUpdated: 0, count: 0, newCount : 0 };
|
|
199
|
+
// URLs that fail even after a requeue are permanently dropped this run; we
|
|
200
|
+
// hold the watermark back when this is non-zero so they get re-scanned.
|
|
201
|
+
let permanentFailures = 0;
|
|
188
202
|
// phase 2: query for history & content
|
|
189
203
|
this.requeue = [];
|
|
190
204
|
for (let q of this.queue) {
|
|
@@ -204,6 +218,7 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
|
|
|
204
218
|
try {
|
|
205
219
|
await this.processContentAndHistory(q, tracking, this.requeue.length);
|
|
206
220
|
} catch (error) {
|
|
221
|
+
permanentFailures++;
|
|
207
222
|
debugLog(error);
|
|
208
223
|
this.stats.task('VSAC Sync', error.message);
|
|
209
224
|
}
|
|
@@ -212,6 +227,20 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
|
|
|
212
227
|
|
|
213
228
|
// Reload map with fresh data
|
|
214
229
|
await this._reloadMap();
|
|
230
|
+
|
|
231
|
+
// Commit the _lastUpdated watermark only now that phase 2 has durably
|
|
232
|
+
// written everything. Hold it back if any URL was permanently dropped, so
|
|
233
|
+
// those URLs are re-scanned next run (never advance past un-applied
|
|
234
|
+
// changes). If the run threw or the process restarted before this point,
|
|
235
|
+
// the watermark is left untouched and the window is re-scanned.
|
|
236
|
+
if (this._pendingLastUpdated && permanentFailures === 0) {
|
|
237
|
+
await this.database.setSetting(VSAC_LAST_UPDATED_KEY, this._pendingLastUpdated);
|
|
238
|
+
} else if (permanentFailures > 0) {
|
|
239
|
+
const holdMsg = `Holding _lastUpdated watermark: ${permanentFailures} URL(s) failed after retry; will re-scan next run`;
|
|
240
|
+
console.log(holdMsg);
|
|
241
|
+
this.stats.task('VSAC Sync', holdMsg);
|
|
242
|
+
}
|
|
243
|
+
|
|
215
244
|
let msg = `VSAC refresh completed. Total: ${tracking.totalFetched} ValueSets, New: ${tracking.totalNew}, Updated: ${tracking.totalUpdated}`;
|
|
216
245
|
this.stats.taskDone('VSAC Sync', msg);
|
|
217
246
|
console.log(msg);
|
|
@@ -579,9 +608,7 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
|
|
|
579
608
|
* @private
|
|
580
609
|
*/
|
|
581
610
|
async _scanLastUpdated() {
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
let sinceDate = await this.database.getSetting(SETTING_KEY);
|
|
611
|
+
let sinceDate = await this.database.getSetting(VSAC_LAST_UPDATED_KEY);
|
|
585
612
|
if (!sinceDate) {
|
|
586
613
|
// No stored date — default to 10 days ago
|
|
587
614
|
const d = new Date();
|
|
@@ -615,10 +642,13 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
|
|
|
615
642
|
url = this._getNextUrl(bundle);
|
|
616
643
|
}
|
|
617
644
|
|
|
618
|
-
//
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
645
|
+
// Capture the server's date but do NOT commit the watermark here. It must
|
|
646
|
+
// only advance once phase 2 has durably upserted the queued URLs; committing
|
|
647
|
+
// it now (before phase 2) means a phase-2 failure or a process restart would
|
|
648
|
+
// move the watermark past URLs that were never written, stranding them
|
|
649
|
+
// permanently. refreshValueSets() commits this._pendingLastUpdated after
|
|
650
|
+
// phase 2 completes with no dropped URLs.
|
|
651
|
+
this._pendingLastUpdated = serverDate;
|
|
622
652
|
|
|
623
653
|
return count;
|
|
624
654
|
}
|
|
@@ -631,8 +661,123 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
|
|
|
631
661
|
return "history";
|
|
632
662
|
}
|
|
633
663
|
|
|
634
|
-
|
|
664
|
+
/**
|
|
665
|
+
* Whether the on-demand resync form is enabled (a resyncPassword is configured).
|
|
666
|
+
*/
|
|
667
|
+
_resyncEnabled() {
|
|
668
|
+
return !!this.resyncPassword;
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
/**
|
|
672
|
+
* Timing-safe comparison of a provided password against the configured one.
|
|
673
|
+
* @private
|
|
674
|
+
*/
|
|
675
|
+
_passwordMatches(provided) {
|
|
676
|
+
if (!this.resyncPassword) {
|
|
677
|
+
return false;
|
|
678
|
+
}
|
|
679
|
+
const a = Buffer.from(String(provided == null ? '' : provided));
|
|
680
|
+
const b = Buffer.from(String(this.resyncPassword));
|
|
681
|
+
if (a.length !== b.length) {
|
|
682
|
+
return false;
|
|
683
|
+
}
|
|
684
|
+
return crypto.timingSafeEqual(a, b);
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
/**
|
|
688
|
+
* Immediately resync a single ValueSet URL (all versions): fetch from VSAC,
|
|
689
|
+
* upsert, and reload the in-memory map.
|
|
690
|
+
* @param {string} url - the ValueSet canonical url
|
|
691
|
+
* @returns {Promise<number>} number of versions fetched
|
|
692
|
+
*/
|
|
693
|
+
async resyncValueSet(url) {
|
|
694
|
+
const tracking = { totalFetched: 0, totalNew: 0, totalUpdated: 0, count: 0, newCount: 0 };
|
|
695
|
+
await this.processContentAndHistory(url, tracking, 1);
|
|
696
|
+
await this._reloadMap();
|
|
697
|
+
return tracking.totalFetched;
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
/**
|
|
701
|
+
* Handle a POST from the resync form. Validates the password (timing-safe) and,
|
|
702
|
+
* if a full sync isn't already running, resyncs the requested URL. Returns an
|
|
703
|
+
* HTML notice. Never echoes or logs the password.
|
|
704
|
+
* @private
|
|
705
|
+
*/
|
|
706
|
+
async _handleResyncRequest(req) {
|
|
707
|
+
const escape = require('escape-html');
|
|
708
|
+
if (!this._resyncEnabled()) {
|
|
709
|
+
return '';
|
|
710
|
+
}
|
|
711
|
+
const body = req.body || {};
|
|
712
|
+
const url = this._expandOidOrUrl(body.url);
|
|
713
|
+
|
|
714
|
+
if (!this._passwordMatches(body.password)) {
|
|
715
|
+
// Small delay to blunt brute-forcing; reveal nothing else.
|
|
716
|
+
await new Promise(r => setTimeout(r, 1000));
|
|
717
|
+
return '<p style="color:#a00"><strong>Incorrect password — no action taken.</strong></p>';
|
|
718
|
+
}
|
|
719
|
+
if (!url) {
|
|
720
|
+
return '<p style="color:#a00">Enter a ValueSet URL to resync.</p>';
|
|
721
|
+
}
|
|
722
|
+
if (this.isRefreshing) {
|
|
723
|
+
return '<p style="color:#a60">A full sync is currently running; please retry in a few minutes.</p>';
|
|
724
|
+
}
|
|
725
|
+
try {
|
|
726
|
+
const n = await this.resyncValueSet(url);
|
|
727
|
+
console.log(`Manual resync of ${url}: ${n} version(s)`);
|
|
728
|
+
this.stats.task('VSAC Sync', `Manual resync of ${url}: ${n} version(s)`);
|
|
729
|
+
return `<p style="color:#070"><strong>Resynced ${escape(url)}: ${n} version(s).</strong></p>`;
|
|
730
|
+
} catch (error) {
|
|
731
|
+
debugLog(error);
|
|
732
|
+
return `<p style="color:#a00">Resync of ${escape(url)} failed: ${escape(error.message)}</p>`;
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
/**
|
|
737
|
+
* Accept either a full canonical URL or a bare VSAC OID. A dotted-decimal OID
|
|
738
|
+
* (optionally with a urn:oid: prefix) is expanded to the VSAC ValueSet URL.
|
|
739
|
+
* @private
|
|
740
|
+
*/
|
|
741
|
+
_expandOidOrUrl(input) {
|
|
742
|
+
let s = (input == null ? '' : String(input)).trim();
|
|
743
|
+
if (s.toLowerCase().startsWith('urn:oid:')) {
|
|
744
|
+
s = s.substring('urn:oid:'.length);
|
|
745
|
+
}
|
|
746
|
+
if (/^[0-9]+(\.[0-9]+)+$/.test(s)) {
|
|
747
|
+
return VSAC_VALUESET_URL_PREFIX + s;
|
|
748
|
+
}
|
|
749
|
+
return s;
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
/**
|
|
753
|
+
* The resync form HTML, or '' when the feature is disabled (no password set).
|
|
754
|
+
* @private
|
|
755
|
+
*/
|
|
756
|
+
_resyncFormHtml() {
|
|
757
|
+
if (!this._resyncEnabled()) {
|
|
758
|
+
return '';
|
|
759
|
+
}
|
|
760
|
+
return `<form method="post" autocomplete="off" style="margin:0 0 1em 0; padding:0.75em; border:1px solid #ccc; background:#f7f7f7">
|
|
761
|
+
<strong>Resync a ValueSet</strong>
|
|
762
|
+
<div style="margin-top:0.5em">
|
|
763
|
+
<label>ValueSet URL or OID: <input type="text" name="url" size="70" autocomplete="off"></label>
|
|
764
|
+
</div>
|
|
765
|
+
<div style="margin-top:0.5em">
|
|
766
|
+
<label>Password: <input type="password" name="password" autocomplete="off"></label>
|
|
767
|
+
<button type="submit">Resync</button>
|
|
768
|
+
</div>
|
|
769
|
+
</form>`;
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
async info(req) {
|
|
635
773
|
const escape = require('escape-html');
|
|
774
|
+
|
|
775
|
+
// Operator action: resync a specific ValueSet (POST from the form below).
|
|
776
|
+
let resyncNotice = '';
|
|
777
|
+
if (req && req.method === 'POST') {
|
|
778
|
+
resyncNotice = await this._handleResyncRequest(req);
|
|
779
|
+
}
|
|
780
|
+
|
|
636
781
|
const db = await this.database._getReadConnection();
|
|
637
782
|
|
|
638
783
|
const rows = await new Promise((resolve, reject) => {
|
|
@@ -689,7 +834,10 @@ class VSACValueSetProvider extends AbstractValueSetProvider {
|
|
|
689
834
|
? new Date(ts * 1000).toISOString().replace('T', ' ').substring(0, 19) + ' UTC'
|
|
690
835
|
: '—';
|
|
691
836
|
|
|
692
|
-
let html = '
|
|
837
|
+
let html = '';
|
|
838
|
+
html += this._resyncFormHtml();
|
|
839
|
+
html += resyncNotice;
|
|
840
|
+
html += '<h3>VSAC Sync History</h3>';
|
|
693
841
|
html += '<table class="grid">';
|
|
694
842
|
html += '<thead><tr><th>Time</th><th>Event</th><th>Detail</th></tr></thead>';
|
|
695
843
|
html += '<tbody>';
|