fhirsmith 0.11.0 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/package.json +1 -1
- package/stats.js +5 -1
- package/tx/html/home-metrics.liquid +16 -1
- package/tx/operation-context.js +91 -2
- package/tx/tx.js +18 -2
- package/tx/workers/expand.js +103 -47
- package/tx/workers/worker.js +9 -0
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,40 @@ All notable changes to the Health Intersections Node Server will be documented i
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [0.11.1] - 2026-07-30
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- Cooperative yielding for long terminology operations: after every 25ms of continuous compute the
|
|
13
|
+
operation yields the event loop (`checkAndYield`), so a heavy $expand can no longer stall every
|
|
14
|
+
other request on the server. Operations whose client has disconnected are aborted at their next
|
|
15
|
+
yield point instead of computing a response nobody will read
|
|
16
|
+
- The operation time limit is now configurable (`operationTimeLimit`, seconds) and defaults to 20s
|
|
17
|
+
(below common client timeouts, so clients see the too-costly OperationOutcome rather than a
|
|
18
|
+
socket timeout), and is measured against compute time consumed rather than wall-clock, so
|
|
19
|
+
concurrent operations time-sharing the event loop don't abort each other
|
|
20
|
+
- The Node Blocking chart on the status page now shows the worst single event-loop stall per
|
|
21
|
+
window as well as the mean
|
|
22
|
+
|
|
23
|
+
### Fixed
|
|
24
|
+
|
|
25
|
+
- fix whole-code-system SNOMED CT expansions blocking the server for 30 seconds and then failing:
|
|
26
|
+
the hierarchy walk read a non-existent `.code` property, so every concept was treated as a
|
|
27
|
+
duplicate of the first and no expansion size guard could ever stop the walk. This is what caused
|
|
28
|
+
the rash of client timeouts on tx.fhir.org after the 0.11.0 upgrade
|
|
29
|
+
- the hierarchy walk now visits each concept once rather than once per path to it (SNOMED CT is a
|
|
30
|
+
poly-hierarchy: the old walk made ~15.8M visits for ~520k concepts)
|
|
31
|
+
- restore the up-front too-costly refusal for expansions that cannot fit the expansion limit (the
|
|
32
|
+
totalCount size guard was silently dead - method read as a property); expansions with a
|
|
33
|
+
count/offset window inside the limit still succeed as partial, unclosed expansions
|
|
34
|
+
- re-enable the CPT 1000-code expansion cap (same method-read-as-property bug)
|
|
35
|
+
- fix parameter misalignment in the no-details expansion path (19 arguments passed to a
|
|
36
|
+
20-parameter function)
|
|
37
|
+
|
|
38
|
+
### Tx Conformance Statement
|
|
39
|
+
|
|
40
|
+
FHIRsmith passed all 2729 HL7 terminology service tests (modes tx.fhir.org+omop+general+snomed, tests v1.9.2, runner v6.9.12)
|
|
41
|
+
|
|
8
42
|
## [0.11.0] - 2026-mm-dd
|
|
9
43
|
|
|
10
44
|
### Added
|
package/package.json
CHANGED
package/stats.js
CHANGED
|
@@ -37,6 +37,10 @@ class ServerStats {
|
|
|
37
37
|
const requestsPerMin = minutesSinceStart > 0 ? requestsDelta / minutesSinceStart : 0;
|
|
38
38
|
|
|
39
39
|
const loopDelay = this.eventLoopMonitor.mean / 1e6;
|
|
40
|
+
// Worst single stall in the window - the mean dilutes a multi-second
|
|
41
|
+
// event-loop block into noise; the max is what shows request-freezing
|
|
42
|
+
// stalls (see the 2026-07-30 tx.fhir.org incident).
|
|
43
|
+
const loopMax = this.eventLoopMonitor.max / 1e6;
|
|
40
44
|
// Two distinct caches: the expansion cache (count entries) and the client
|
|
41
45
|
// (resource) cache (count concepts held - a sense of how much it's carrying).
|
|
42
46
|
let expansionItems = 0;
|
|
@@ -50,7 +54,7 @@ class ServerStats {
|
|
|
50
54
|
}
|
|
51
55
|
}
|
|
52
56
|
|
|
53
|
-
this.history.push({time: now, mem: currentMem - this.startMem, rpm: requestsPerMin, tat: requestsTat, block: loopDelay, expansion: expansionItems, clientConcepts: clientConcepts});
|
|
57
|
+
this.history.push({time: now, mem: currentMem - this.startMem, rpm: requestsPerMin, tat: requestsTat, block: loopDelay, blockMax: loopMax, expansion: expansionItems, clientConcepts: clientConcepts});
|
|
54
58
|
|
|
55
59
|
this.eventLoopMonitor.reset();
|
|
56
60
|
this.requestCountSnapshot = combinedCount;
|
|
@@ -221,23 +221,38 @@
|
|
|
221
221
|
data: {
|
|
222
222
|
labels: historyData.map(d => formatTime(d.time)),
|
|
223
223
|
datasets: [{
|
|
224
|
+
label: 'mean',
|
|
224
225
|
data: historyData.map(d => d.block),
|
|
225
226
|
borderColor: '#2845a7',
|
|
226
227
|
backgroundColor: 'rgba(40, 69, 167, 0.1)',
|
|
227
228
|
fill: true,
|
|
228
229
|
tension: 0.3,
|
|
229
230
|
pointRadius: 2
|
|
231
|
+
}, {
|
|
232
|
+
// Worst single event-loop stall in the window: the mean dilutes a
|
|
233
|
+
// multi-second block into noise, so this is the line that shows
|
|
234
|
+
// request-freezing stalls.
|
|
235
|
+
label: 'max',
|
|
236
|
+
data: historyData.map(d => d.blockMax),
|
|
237
|
+
borderColor: '#a72828',
|
|
238
|
+
fill: false,
|
|
239
|
+
tension: 0.3,
|
|
240
|
+
pointRadius: 2
|
|
230
241
|
}]
|
|
231
242
|
},
|
|
232
243
|
options: {
|
|
233
244
|
...chartOptions,
|
|
245
|
+
plugins: {
|
|
246
|
+
...chartOptions.plugins,
|
|
247
|
+
legend: { display: true }
|
|
248
|
+
},
|
|
234
249
|
scales: {
|
|
235
250
|
...chartOptions.scales,
|
|
236
251
|
y: {
|
|
237
252
|
...chartOptions.scales.y,
|
|
238
253
|
title: {
|
|
239
254
|
display: true,
|
|
240
|
-
text: '
|
|
255
|
+
text: 'ms'
|
|
241
256
|
}
|
|
242
257
|
}
|
|
243
258
|
}
|
package/tx/operation-context.js
CHANGED
|
@@ -556,6 +556,15 @@ const MEMORY_LIMIT = readMemoryLimit();
|
|
|
556
556
|
const MEMORY_FRACTION = 0.98;
|
|
557
557
|
const MEMORY_THRESHOLD = MEMORY_LIMIT > 0 ? MEMORY_LIMIT * MEMORY_FRACTION : 0; // 90% of cgroup limit
|
|
558
558
|
const CHECK_FREQUENCY = 100;
|
|
559
|
+
// How long an operation may compute without yielding the event loop (ms).
|
|
560
|
+
// Node runs all JS on one thread: while a long operation executes, no other
|
|
561
|
+
// request is serviced - awaiting in-memory promises does not help, because
|
|
562
|
+
// resolved promises only bounce through the microtask queue, which runs before
|
|
563
|
+
// I/O. checkAndYield() awaits a real setImmediate every YIELD_INTERVAL ms of
|
|
564
|
+
// compute, so pending I/O (new connections, /metadata, timers) runs between
|
|
565
|
+
// slices and one heavy $expand can no longer time out every other client.
|
|
566
|
+
const YIELD_INTERVAL = 25;
|
|
567
|
+
const { setImmediate: yieldToEventLoop } = require('timers/promises');
|
|
559
568
|
|
|
560
569
|
class OperationContext {
|
|
561
570
|
// Shared counter across all instances — only check RSS every CHECK_FREQUENCY calls
|
|
@@ -584,6 +593,24 @@ class OperationContext {
|
|
|
584
593
|
// Shared by reference with copy()'d contexts so a sub-operation's
|
|
585
594
|
// providers are cleaned up by the parent request's closeProviders().
|
|
586
595
|
this._openProviders = [];
|
|
596
|
+
// Compute-time accounting for the deadline and for yielding. One object,
|
|
597
|
+
// shared by reference with copy()'d contexts, so an operation and its
|
|
598
|
+
// sub-operations (e.g. batch entries) draw down a single budget:
|
|
599
|
+
// - compute: ms of compute consumed in completed slices
|
|
600
|
+
// - sliceStart: when the current compute slice began
|
|
601
|
+
// - lastYield: when we last gave the event loop a turn
|
|
602
|
+
// - clientGone: set when the client disconnects before the response is sent
|
|
603
|
+
// The deadline is measured in COMPUTE time, not wall-clock: once operations
|
|
604
|
+
// time-share the event loop, wall-clock deadlines make concurrency itself
|
|
605
|
+
// cause aborts (N ops each take N x as long), which turns client retries
|
|
606
|
+
// into a failure amplifier. An operation that waited is not too costly;
|
|
607
|
+
// one that consumed its compute budget is.
|
|
608
|
+
this._clock = {
|
|
609
|
+
compute: 0,
|
|
610
|
+
sliceStart: this.startTime,
|
|
611
|
+
lastYield: this.startTime,
|
|
612
|
+
clientGone: false
|
|
613
|
+
};
|
|
587
614
|
|
|
588
615
|
this.timeTracker.step('tx-op');
|
|
589
616
|
}
|
|
@@ -616,6 +643,8 @@ class OperationContext {
|
|
|
616
643
|
// Share the same provider-cleanup list so providers opened by the copy
|
|
617
644
|
// are released when the parent operation ends.
|
|
618
645
|
newContext._openProviders = this._openProviders;
|
|
646
|
+
// Share the compute clock: sub-operations spend the parent's budget.
|
|
647
|
+
newContext._clock = this._clock;
|
|
619
648
|
return newContext;
|
|
620
649
|
}
|
|
621
650
|
|
|
@@ -646,8 +675,10 @@ class OperationContext {
|
|
|
646
675
|
}
|
|
647
676
|
OperationContext._checkCounter = 0;
|
|
648
677
|
|
|
649
|
-
// Time check
|
|
650
|
-
|
|
678
|
+
// Time check - against compute time consumed, not wall clock (see _clock in
|
|
679
|
+
// the constructor). For operations that never yield the two are identical,
|
|
680
|
+
// so this changes nothing for paths that don't use checkAndYield().
|
|
681
|
+
const elapsed = this.computeElapsed();
|
|
651
682
|
if (elapsed > this.timeLimit) {
|
|
652
683
|
const timeInSeconds = Math.round(this.timeLimit / 1000);
|
|
653
684
|
this.log(`Operation took too long @ ${place} (${this.constructor.name})`);
|
|
@@ -674,6 +705,64 @@ class OperationContext {
|
|
|
674
705
|
return false;
|
|
675
706
|
}
|
|
676
707
|
|
|
708
|
+
/**
|
|
709
|
+
* Compute time this operation has consumed (ms): completed slices plus the
|
|
710
|
+
* slice currently executing. Equals wall-clock elapsed for operations that
|
|
711
|
+
* never yield.
|
|
712
|
+
* @returns {number}
|
|
713
|
+
*/
|
|
714
|
+
computeElapsed() {
|
|
715
|
+
return this._clock.compute + (performance.now() - this._clock.sliceStart);
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
/**
|
|
719
|
+
* Mark that the client for this operation disconnected before the response
|
|
720
|
+
* was sent. The operation is aborted at its next yield point - there is no
|
|
721
|
+
* one left to send the result to. (Wired up by the request middleware from
|
|
722
|
+
* the response 'close' event.)
|
|
723
|
+
*/
|
|
724
|
+
markClientGone() {
|
|
725
|
+
this._clock.clientGone = true;
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
/**
|
|
729
|
+
* deadCheck plus cooperative yielding: call this (awaited) from long-running
|
|
730
|
+
* loops in async code. Runs the normal deadCheck, and after every
|
|
731
|
+
* YIELD_INTERVAL ms of continuous compute awaits a real setImmediate so the
|
|
732
|
+
* event loop can service pending I/O - other requests keep getting answered
|
|
733
|
+
* while this operation works. On resuming from a yield, aborts if the client
|
|
734
|
+
* has disconnected in the meantime.
|
|
735
|
+
*
|
|
736
|
+
* Sync call sites keep using deadCheck(); the async loops that surround them
|
|
737
|
+
* yield on their behalf, which caps unbroken compute at one loop iteration.
|
|
738
|
+
*
|
|
739
|
+
* @param {string} place - Location identifier for debugging
|
|
740
|
+
*/
|
|
741
|
+
async checkAndYield(place = 'unknown') {
|
|
742
|
+
this.deadCheck(place);
|
|
743
|
+
if (this.debugging) {
|
|
744
|
+
return;
|
|
745
|
+
}
|
|
746
|
+
const now = performance.now();
|
|
747
|
+
if (now - this._clock.lastYield >= YIELD_INTERVAL) {
|
|
748
|
+
// Close off this compute slice before suspending, so time spent waiting
|
|
749
|
+
// for our next turn is not charged against the deadline.
|
|
750
|
+
this._clock.compute += now - this._clock.sliceStart;
|
|
751
|
+
await yieldToEventLoop();
|
|
752
|
+
const resumed = performance.now();
|
|
753
|
+
this._clock.sliceStart = resumed;
|
|
754
|
+
this._clock.lastYield = resumed;
|
|
755
|
+
if (this._clock.clientGone) {
|
|
756
|
+
this.log(`Operation abandoned @ ${place}: client disconnected`);
|
|
757
|
+
const error = new Issue("error", "too-costly", null,
|
|
758
|
+
`Operation abandoned at ${place}: the client disconnected before the response was ready`);
|
|
759
|
+
error.abandoned = true;
|
|
760
|
+
error.diagnostics = this.diagnostics();
|
|
761
|
+
throw error;
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
|
|
677
766
|
unSeeAll() {
|
|
678
767
|
this.contexts = [];
|
|
679
768
|
}
|
package/tx/tx.js
CHANGED
|
@@ -291,9 +291,15 @@ class TXModule {
|
|
|
291
291
|
// Get Accept-Language header for language preferences
|
|
292
292
|
const acceptLanguage = req.get('Accept-Language') || 'en';
|
|
293
293
|
|
|
294
|
-
// Create operation context with language, ID, time limit, and caches
|
|
294
|
+
// Create operation context with language, ID, time limit, and caches.
|
|
295
|
+
// The time limit is a compute-time budget (see OperationContext._clock)
|
|
296
|
+
// and is configurable (config.operationTimeLimit, seconds). The default
|
|
297
|
+
// sits deliberately under common client timeouts (okhttp reads give up at
|
|
298
|
+
// 10-30s): if the server's limit is >= the client's, the client hangs up
|
|
299
|
+
// first and the carefully-built too-costly OperationOutcome is written to
|
|
300
|
+
// a socket nobody is reading.
|
|
295
301
|
const opContext = new OperationContext(
|
|
296
|
-
acceptLanguage, this.i18n, requestId,
|
|
302
|
+
acceptLanguage, this.i18n, requestId, this.config.operationTimeLimit ?? 20,
|
|
297
303
|
endpointInfo.resourceCache, endpointInfo.expansionCache
|
|
298
304
|
);
|
|
299
305
|
opContext.usageTracker = this.usageTracker;
|
|
@@ -325,6 +331,16 @@ class TXModule {
|
|
|
325
331
|
res.on('finish', releaseProviders);
|
|
326
332
|
res.on('close', releaseProviders);
|
|
327
333
|
|
|
334
|
+
// If the connection closes before the response was completed, the client
|
|
335
|
+
// has given up (timeout, cancelled build, dropped connection). Mark the
|
|
336
|
+
// operation so it aborts at its next yield point instead of computing a
|
|
337
|
+
// response nobody will read.
|
|
338
|
+
res.on('close', () => {
|
|
339
|
+
if (!res.writableEnded) {
|
|
340
|
+
opContext.markClientGone();
|
|
341
|
+
}
|
|
342
|
+
});
|
|
343
|
+
|
|
328
344
|
// Add X-Request-Id header to response
|
|
329
345
|
res.setHeader('X-Request-Id', requestId);
|
|
330
346
|
|
package/tx/workers/expand.js
CHANGED
|
@@ -314,14 +314,20 @@ class ValueSetExpander {
|
|
|
314
314
|
return null;
|
|
315
315
|
}
|
|
316
316
|
|
|
317
|
-
|
|
318
|
-
|
|
317
|
+
// expandLimitation() caps how many codes a single code system may contribute
|
|
318
|
+
// (e.g. CPT limits enumeration to 1000 codes by agreement with the AMA).
|
|
319
|
+
// NB: expandLimitation and system are methods - reading them as properties
|
|
320
|
+
// (cs.expandLimitation > 0) compares the function object and is always false,
|
|
321
|
+
// which silently disabled this cap.
|
|
322
|
+
const expandLimit = cs != null ? cs.expandLimitation() : 0;
|
|
323
|
+
if (expandLimit > 0) {
|
|
324
|
+
let cnt = this.csCounter.get(cs.system());
|
|
319
325
|
if (cnt == null) {
|
|
320
326
|
cnt = new ValueSetCounter();
|
|
321
|
-
this.csCounter.set(cs.system, cnt);
|
|
327
|
+
this.csCounter.set(cs.system(), cnt);
|
|
322
328
|
}
|
|
323
329
|
cnt.increment();
|
|
324
|
-
if (cnt.count >
|
|
330
|
+
if (cnt.count > expandLimit) {
|
|
325
331
|
return null;
|
|
326
332
|
}
|
|
327
333
|
}
|
|
@@ -566,7 +572,7 @@ class ValueSetExpander {
|
|
|
566
572
|
this.checkResourceCanonicalStatus(expansion, vs, this.valueSet);
|
|
567
573
|
|
|
568
574
|
for (const c of vs.expansion.contains || []) {
|
|
569
|
-
this.worker.
|
|
575
|
+
await this.worker.checkAndYield('importValueSet');
|
|
570
576
|
count += await this.importValueSetItem(null, c, imports, offset);
|
|
571
577
|
}
|
|
572
578
|
return count;
|
|
@@ -574,7 +580,7 @@ class ValueSetExpander {
|
|
|
574
580
|
|
|
575
581
|
async importValueSetItem(p, c, imports, offset) {
|
|
576
582
|
let count = 0;
|
|
577
|
-
this.worker.
|
|
583
|
+
await this.worker.checkAndYield('importValueSetItem');
|
|
578
584
|
const s = this.keyC(c);
|
|
579
585
|
if (this.passesImports(imports, c.system, c.code, offset) && !this.map.has(s) && !this.isExcluded(c.system, c.version, c.code)) {
|
|
580
586
|
count++;
|
|
@@ -588,7 +594,7 @@ class ValueSetExpander {
|
|
|
588
594
|
this.map.set(s, c);
|
|
589
595
|
}
|
|
590
596
|
for (const cc of c.contains || []) {
|
|
591
|
-
this.worker.
|
|
597
|
+
await this.worker.checkAndYield('importValueSetItem');
|
|
592
598
|
count += await this.importValueSetItem(c, cc, imports, offset);
|
|
593
599
|
}
|
|
594
600
|
return count;
|
|
@@ -610,11 +616,11 @@ class ValueSetExpander {
|
|
|
610
616
|
}
|
|
611
617
|
|
|
612
618
|
async checkSource(cset, exp, filter, srcURL, ts, vsInfo , source) {
|
|
613
|
-
this.worker.
|
|
619
|
+
await this.worker.checkAndYield('checkSource');
|
|
614
620
|
Extensions.checkNoModifiers(cset, 'ValueSetExpander.checkSource', 'set', srcURL);
|
|
615
621
|
let imp = false;
|
|
616
622
|
for (const u of cset.valueSet || []) {
|
|
617
|
-
this.worker.
|
|
623
|
+
await this.worker.checkAndYield('checkSource');
|
|
618
624
|
const s = this.worker.pinValueSet(u);
|
|
619
625
|
await this.checkCanExpandValueSet(s, '', source);
|
|
620
626
|
imp = true;
|
|
@@ -663,8 +669,21 @@ class ValueSetExpander {
|
|
|
663
669
|
} else if (filter.isNull) {
|
|
664
670
|
// The unclosed marking for a grammar code system is added by
|
|
665
671
|
// processCodes (the actual expansion); checkSource only guards size.
|
|
666
|
-
|
|
667
|
-
|
|
672
|
+
// Refuse up-front only when enumeration could not succeed anyway:
|
|
673
|
+
// count=0 is answered by counting (no enumeration), and a count/offset
|
|
674
|
+
// window that fits inside limitCount is answered by partial
|
|
675
|
+
// enumeration (includeCode stops at limitCount). Otherwise the walk
|
|
676
|
+
// would only ever end in VALUESET_TOO_COSTLY at limitCount codes -
|
|
677
|
+
// throw the same error now without doing the work.
|
|
678
|
+
// NB: totalCount is a method - as a property read (cs.totalCount) the
|
|
679
|
+
// comparison was against the function object, always false, so this
|
|
680
|
+
// guard was silently dead.
|
|
681
|
+
if (!imp && this.count !== 0 && this.limitCount > 0 && await cs.totalCount() > this.limitCount
|
|
682
|
+
&& !(this.count > 0 && this.count + Math.max(this.offset, 0) <= this.limitCount)) {
|
|
683
|
+
// Report the versioned url (vurl), matching the message the
|
|
684
|
+
// enumeration path produces when it hits the same limit - this
|
|
685
|
+
// guard is just that refusal moved earlier.
|
|
686
|
+
throw new Issue("error", "too-costly", null, 'VALUESET_TOO_COSTLY', this.worker.i18n.translate('VALUESET_TOO_COSTLY', this.params.httpLanguages, [source && source.vurl ? source.vurl : srcURL, '>' + this.limitCount]), null, 422).withDiagnostics(this.worker.opContext.diagnostics());
|
|
668
687
|
}
|
|
669
688
|
}
|
|
670
689
|
}
|
|
@@ -684,7 +703,7 @@ class ValueSetExpander {
|
|
|
684
703
|
}
|
|
685
704
|
|
|
686
705
|
async includeCodes(cset, path, vsSrc, compose, filter, expansion, excludeInactive, notClosed) {
|
|
687
|
-
this.worker.
|
|
706
|
+
await this.worker.checkAndYield('processCodes#1');
|
|
688
707
|
const valueSets = [];
|
|
689
708
|
|
|
690
709
|
Extensions.checkNoModifiers(cset, 'ValueSetExpander.processCodes', 'set', vsSrc.vurl);
|
|
@@ -695,7 +714,7 @@ class ValueSetExpander {
|
|
|
695
714
|
|
|
696
715
|
if (!cset.system) {
|
|
697
716
|
for (const u of cset.valueSet) {
|
|
698
|
-
this.worker.
|
|
717
|
+
await this.worker.checkAndYield('processCodes#2');
|
|
699
718
|
const s = this.worker.pinValueSet(u);
|
|
700
719
|
this.worker.opContext.log('import value set ' + s);
|
|
701
720
|
let vs = await this.worker.findValueSet(s, '', vsSrc);
|
|
@@ -722,7 +741,7 @@ class ValueSetExpander {
|
|
|
722
741
|
this.addParamUri(expansion, 'used-codesystem', sv);
|
|
723
742
|
|
|
724
743
|
for (const u of cset.valueSet || []) {
|
|
725
|
-
this.worker.
|
|
744
|
+
await this.worker.checkAndYield('processCodes#2');
|
|
726
745
|
const s = this.worker.pinValueSet(u);
|
|
727
746
|
this.worker.opContext.log('import value set ' + s);
|
|
728
747
|
let vs = await this.worker.findValueSet(s, '', vsSrc);
|
|
@@ -774,11 +793,14 @@ class ValueSetExpander {
|
|
|
774
793
|
|
|
775
794
|
}
|
|
776
795
|
let tcount = 0;
|
|
796
|
+
// One visited set across all roots: in a poly-hierarchy the walks
|
|
797
|
+
// from different roots can reach the same concepts.
|
|
798
|
+
const visited = new Set();
|
|
777
799
|
let c = await cs.nextContext(iter);
|
|
778
800
|
while (c) {
|
|
779
|
-
this.worker.
|
|
801
|
+
await this.worker.checkAndYield('processCodes#3a');
|
|
780
802
|
if (await this.passesFilters(cs, c, prep, filters, 0)) {
|
|
781
|
-
tcount += await this.includeCodeAndDescendants(cs, c, expansion, valueSets, null, excludeInactive, vsSrc.vurl);
|
|
803
|
+
tcount += await this.includeCodeAndDescendants(cs, c, expansion, valueSets, null, excludeInactive, vsSrc.vurl, visited);
|
|
782
804
|
}
|
|
783
805
|
c = await cs.nextContext(iter);
|
|
784
806
|
}
|
|
@@ -800,7 +822,7 @@ class ValueSetExpander {
|
|
|
800
822
|
this.noTotal();
|
|
801
823
|
this.worker.opContext.log('iterate filters');
|
|
802
824
|
while (await cs.filterMore(ctxt, set[0])) {
|
|
803
|
-
this.worker.
|
|
825
|
+
await this.worker.checkAndYield('processCodes#4');
|
|
804
826
|
const c = await cs.filterConcept(ctxt, set[0]);
|
|
805
827
|
if (await this.passesFilters(cs, c, prep, set, 1)) {
|
|
806
828
|
const cds = new Designations(this.worker.i18n.languageDefinitions);
|
|
@@ -830,7 +852,7 @@ class ValueSetExpander {
|
|
|
830
852
|
const cds = new Designations(this.worker.i18n.languageDefinitions);
|
|
831
853
|
|
|
832
854
|
for (const cc of cset.concept) {
|
|
833
|
-
this.worker.
|
|
855
|
+
await this.worker.checkAndYield('processCodes#3');
|
|
834
856
|
cds.clear();
|
|
835
857
|
Extensions.checkNoModifiers(cc, 'ValueSetExpander.processCodes', 'set concept reference', vsSrc.vurl);
|
|
836
858
|
const cctxt = await cs.locate(cc.code, this.allAltCodes);
|
|
@@ -875,7 +897,7 @@ class ValueSetExpander {
|
|
|
875
897
|
}
|
|
876
898
|
|
|
877
899
|
for (let i = 0; i < fcl.length; i++) {
|
|
878
|
-
this.worker.
|
|
900
|
+
await this.worker.checkAndYield('processCodes#4a');
|
|
879
901
|
const fc = fcl[i];
|
|
880
902
|
if (!fc.value) {
|
|
881
903
|
throw new Issue('error', 'invalid', path + ".filter[" + i + "]", 'UNABLE_TO_HANDLE_SYSTEM_FILTER_WITH_NO_VALUE', this.worker.i18n.translate('UNABLE_TO_HANDLE_SYSTEM_FILTER_WITH_NO_VALUE', this.params.httpLanguages, [cs.system(), fc.property, fc.op]), 'vs-invalid', 400);
|
|
@@ -895,7 +917,7 @@ class ValueSetExpander {
|
|
|
895
917
|
this.addToTotal(0);
|
|
896
918
|
const cds = new Designations(this.worker.i18n.languageDefinitions);
|
|
897
919
|
while (await cs.filterMore(prep, fset[0])) {
|
|
898
|
-
this.worker.
|
|
920
|
+
await this.worker.checkAndYield('processCodes#5');
|
|
899
921
|
const c = await cs.filterConcept(prep, fset[0]);
|
|
900
922
|
const ok = (!this.params.activeOnly || !await cs.isInactive(c)) && (await this.passesFilters(cs, c, prep, fset, 1));
|
|
901
923
|
if (ok) {
|
|
@@ -944,7 +966,7 @@ class ValueSetExpander {
|
|
|
944
966
|
}
|
|
945
967
|
|
|
946
968
|
async excludeCodes(cset, path, vsSrc, filter, expansion, excludeInactive, notClosed) {
|
|
947
|
-
this.worker.
|
|
969
|
+
await this.worker.checkAndYield('processCodes#1');
|
|
948
970
|
const valueSets = [];
|
|
949
971
|
|
|
950
972
|
Extensions.checkNoModifiers(cset, 'ValueSetExpander.processCodes', 'set', vsSrc.vurl);
|
|
@@ -958,7 +980,7 @@ class ValueSetExpander {
|
|
|
958
980
|
this.noTotal();
|
|
959
981
|
for (const u of cset.valueSet) {
|
|
960
982
|
const s = this.worker.pinValueSet(u);
|
|
961
|
-
this.worker.
|
|
983
|
+
await this.worker.checkAndYield('processCodes#2');
|
|
962
984
|
let vs = await this.worker.findValueSet(s, '', vsSrc);
|
|
963
985
|
const ivs = new ImportedValueSet(await this.expandValueSet(s, '', vs, filter, notClosed));
|
|
964
986
|
this.checkResourceCanonicalStatus(expansion, ivs.valueSet, this.valueSet);
|
|
@@ -984,7 +1006,7 @@ class ValueSetExpander {
|
|
|
984
1006
|
this.addParamUri(expansion, 'used-codesystem', sv);
|
|
985
1007
|
|
|
986
1008
|
for (const u of cset.valueSet || []) {
|
|
987
|
-
this.worker.
|
|
1009
|
+
await this.worker.checkAndYield('processCodes#3');
|
|
988
1010
|
const s = this.worker.pinValueSet(u);
|
|
989
1011
|
this.worker.opContext.log('import value set ' + s);
|
|
990
1012
|
let vs = await this.worker.findValueSet(s, '', vsSrc);
|
|
@@ -1007,7 +1029,7 @@ class ValueSetExpander {
|
|
|
1007
1029
|
if (iter) {
|
|
1008
1030
|
let c = await cs.nextContext(iter);
|
|
1009
1031
|
while (c) {
|
|
1010
|
-
this.worker.
|
|
1032
|
+
await this.worker.checkAndYield('processCodes#3aa');
|
|
1011
1033
|
this.excludeCode(cs, cs.system(), cs.version(), await cs.code(c), expansion, valueSets, vsSrc.url);
|
|
1012
1034
|
c = await cs.nextContext(iter);
|
|
1013
1035
|
}
|
|
@@ -1031,7 +1053,7 @@ class ValueSetExpander {
|
|
|
1031
1053
|
if (iter) {
|
|
1032
1054
|
let c = await cs.nextContext(iter);
|
|
1033
1055
|
while (c) {
|
|
1034
|
-
this.worker.
|
|
1056
|
+
await this.worker.checkAndYield('processCodes#3a');
|
|
1035
1057
|
if (await this.passesFilters(cs, c, prep, filters, 0)) {
|
|
1036
1058
|
this.excludeCode(cs, cs.system(), cs.version(), await cs.code(c), expansion, valueSets, vsSrc.url);
|
|
1037
1059
|
}
|
|
@@ -1045,7 +1067,7 @@ class ValueSetExpander {
|
|
|
1045
1067
|
this.worker.opContext.log('iterate concepts');
|
|
1046
1068
|
const cds = new Designations(this.worker.i18n.languageDefinitions);
|
|
1047
1069
|
for (const cc of cset.concept) {
|
|
1048
|
-
this.worker.
|
|
1070
|
+
await this.worker.checkAndYield('processCodes#3');
|
|
1049
1071
|
cds.clear();
|
|
1050
1072
|
Extensions.checkNoModifiers(cc, 'ValueSetExpander.processCodes', 'set concept reference', vsSrc.vurl);
|
|
1051
1073
|
const cctxt = await cs.locate(cc.code, this.allAltCodes);
|
|
@@ -1076,7 +1098,7 @@ class ValueSetExpander {
|
|
|
1076
1098
|
|
|
1077
1099
|
let first = true;
|
|
1078
1100
|
for (let fc of cset.filter) {
|
|
1079
|
-
this.worker.
|
|
1101
|
+
await this.worker.checkAndYield('processCodes#4a');
|
|
1080
1102
|
Extensions.checkNoModifiers(fc, 'ValueSetExpander.processCodes', 'filter', vsSrc.vurl);
|
|
1081
1103
|
await cs.filter(prep, first, fc.property, fc.op, fc.value, vsSrc.isCached ? fc : null);
|
|
1082
1104
|
first = false;
|
|
@@ -1089,7 +1111,7 @@ class ValueSetExpander {
|
|
|
1089
1111
|
}
|
|
1090
1112
|
//let count = 0;
|
|
1091
1113
|
while (await cs.filterMore(prep, fset[0])) {
|
|
1092
|
-
this.worker.
|
|
1114
|
+
await this.worker.checkAndYield('processCodes#5');
|
|
1093
1115
|
const c = await cs.filterConcept(prep, fset[0]);
|
|
1094
1116
|
const ok = (!this.params.activeOnly || !await cs.isInactive(c)) && (await this.passesFilters(cs, c, prep, fset, 1));
|
|
1095
1117
|
if (ok) {
|
|
@@ -1105,9 +1127,30 @@ class ValueSetExpander {
|
|
|
1105
1127
|
}
|
|
1106
1128
|
}
|
|
1107
1129
|
|
|
1108
|
-
async includeCodeAndDescendants(cs, context, expansion, imports, parent, excludeInactive, srcUrl) {
|
|
1130
|
+
async includeCodeAndDescendants(cs, context, expansion, imports, parent, excludeInactive, srcUrl, visited = null) {
|
|
1109
1131
|
let result = 0;
|
|
1110
|
-
this.worker.
|
|
1132
|
+
await this.worker.checkAndYield('processCodeAndDescendants');
|
|
1133
|
+
|
|
1134
|
+
// In a poly-hierarchy (e.g. SNOMED CT) a concept is reachable through every
|
|
1135
|
+
// one of its parents; without a visited set the recursion walks every *path*
|
|
1136
|
+
// rather than every *node* - for the full SNOMED International edition that
|
|
1137
|
+
// is ~15.8M visits for ~520k concepts, which is what turned an aborted
|
|
1138
|
+
// expansion into a 30-second event-loop stall. Reaching a concept a second
|
|
1139
|
+
// time means it has multiple parents, so (as when includeCode detects a
|
|
1140
|
+
// duplicate) the expansion cannot be represented as a tree.
|
|
1141
|
+
if (visited == null) {
|
|
1142
|
+
visited = new Set();
|
|
1143
|
+
}
|
|
1144
|
+
// NB: contexts do not reliably carry a .code property (SnomedExpressionContext
|
|
1145
|
+
// does not) - cs.code(context) is the accessor. Reading context.code here
|
|
1146
|
+
// used to yield undefined for every SNOMED concept, collapsing the whole
|
|
1147
|
+
// expansion onto one map key so no size guard could ever fire.
|
|
1148
|
+
const code = await cs.code(context);
|
|
1149
|
+
if (visited.has(code)) {
|
|
1150
|
+
this.canBeHierarchy = false;
|
|
1151
|
+
return 0;
|
|
1152
|
+
}
|
|
1153
|
+
visited.add(code);
|
|
1111
1154
|
|
|
1112
1155
|
if (expansion) {
|
|
1113
1156
|
const vs = this.canonical(await cs.system(), await cs.version());
|
|
@@ -1123,12 +1166,12 @@ class ValueSetExpander {
|
|
|
1123
1166
|
const cds = new Designations(this.worker.i18n.languageDefinitions);
|
|
1124
1167
|
let t;
|
|
1125
1168
|
if (this.noDetails) {
|
|
1126
|
-
t = await this.includeCode(cs, null, await cs.system(), await cs.version(),
|
|
1127
|
-
null, expansion, imports, null, null, null, null, excludeInactive, srcUrl);
|
|
1169
|
+
t = await this.includeCode(cs, null, await cs.system(), await cs.version(), code, await cs.isAbstract(context), await cs.isInactive(context), null, null, null,
|
|
1170
|
+
null, null, expansion, imports, null, null, null, null, excludeInactive, srcUrl);
|
|
1128
1171
|
|
|
1129
1172
|
} else {
|
|
1130
1173
|
await this.listDisplaysFromProvider(cds, cs, context);
|
|
1131
|
-
t = await this.includeCode(cs, parent, await cs.system(), await cs.version(),
|
|
1174
|
+
t = await this.includeCode(cs, parent, await cs.system(), await cs.version(), code, await cs.isAbstract(context), await cs.isInactive(context), await cs.isDeprecated(context), await cs.getStatus(context), cds, await cs.definition(context),
|
|
1132
1175
|
await cs.itemWeight(context), expansion, imports, await cs.extensions(context), null, await cs.properties(context), null, excludeInactive, srcUrl);
|
|
1133
1176
|
}
|
|
1134
1177
|
if (t != null) {
|
|
@@ -1145,16 +1188,29 @@ class ValueSetExpander {
|
|
|
1145
1188
|
if (iter) {
|
|
1146
1189
|
let c = await cs.nextContext(iter);
|
|
1147
1190
|
while (c) {
|
|
1148
|
-
this.worker.
|
|
1149
|
-
result += await this.includeCodeAndDescendants(cs, c, expansion, imports, n, excludeInactive, srcUrl);
|
|
1191
|
+
await this.worker.checkAndYield('processCodeAndDescendants#3');
|
|
1192
|
+
result += await this.includeCodeAndDescendants(cs, c, expansion, imports, n, excludeInactive, srcUrl, visited);
|
|
1150
1193
|
c = await cs.nextContext(iter);
|
|
1151
1194
|
}
|
|
1152
1195
|
}
|
|
1153
1196
|
return result;
|
|
1154
1197
|
}
|
|
1155
1198
|
|
|
1156
|
-
async excludeCodeAndDescendants(cs, context, expansion, imports, excludeInactive, srcUrl) {
|
|
1157
|
-
this.worker.
|
|
1199
|
+
async excludeCodeAndDescendants(cs, context, expansion, imports, excludeInactive, srcUrl, visited = null) {
|
|
1200
|
+
await this.worker.checkAndYield('processCodeAndDescendants');
|
|
1201
|
+
|
|
1202
|
+
// Same visited-set protection as includeCodeAndDescendants: walk nodes, not
|
|
1203
|
+
// paths (see that method for why), and same accessor fix - context.code is
|
|
1204
|
+
// not reliably present, and passing the raw context as the code produced
|
|
1205
|
+
// useless "[object Object]" exclusion keys.
|
|
1206
|
+
if (visited == null) {
|
|
1207
|
+
visited = new Set();
|
|
1208
|
+
}
|
|
1209
|
+
const code = await cs.code(context);
|
|
1210
|
+
if (visited.has(code)) {
|
|
1211
|
+
return;
|
|
1212
|
+
}
|
|
1213
|
+
visited.add(code);
|
|
1158
1214
|
|
|
1159
1215
|
if (expansion) {
|
|
1160
1216
|
const vs = this.canonical(await cs.system(), await cs.version());
|
|
@@ -1168,15 +1224,15 @@ class ValueSetExpander {
|
|
|
1168
1224
|
if ((!this.params.excludeNotForUI || !await cs.isAbstract(context)) && (!this.params.activeOnly || !await cs.isInactive(context))) {
|
|
1169
1225
|
const cds = new Designations(this.worker.i18n.languageDefinitions);
|
|
1170
1226
|
await this.listDisplaysFromProvider(cds, cs, context);
|
|
1171
|
-
this.worker.
|
|
1172
|
-
this.excludeCode(cs, await cs.system(), await cs.version(),
|
|
1227
|
+
await this.worker.checkAndYield('processCodeAndDescendants#2');
|
|
1228
|
+
this.excludeCode(cs, await cs.system(), await cs.version(), code, expansion, imports, srcUrl);
|
|
1173
1229
|
}
|
|
1174
1230
|
|
|
1175
1231
|
const iter = await cs.iterator(context);
|
|
1176
1232
|
let c = await cs.nextContext(iter);
|
|
1177
1233
|
while (c) {
|
|
1178
|
-
this.worker.
|
|
1179
|
-
await this.excludeCodeAndDescendants(cs, c, expansion, imports, excludeInactive, srcUrl);
|
|
1234
|
+
await this.worker.checkAndYield('processCodeAndDescendants#3');
|
|
1235
|
+
await this.excludeCodeAndDescendants(cs, c, expansion, imports, excludeInactive, srcUrl, visited);
|
|
1180
1236
|
c = await cs.nextContext(iter);
|
|
1181
1237
|
}
|
|
1182
1238
|
}
|
|
@@ -1187,11 +1243,11 @@ class ValueSetExpander {
|
|
|
1187
1243
|
this.doingVersion = false;
|
|
1188
1244
|
const ts = new Map();
|
|
1189
1245
|
for (const c of source.jsonObj.compose.include || []) {
|
|
1190
|
-
this.worker.
|
|
1246
|
+
await this.worker.checkAndYield('handleCompose#2');
|
|
1191
1247
|
await this.checkSource(c, expansion, filter, source.url, ts, vsInfo, source);
|
|
1192
1248
|
}
|
|
1193
1249
|
for (const c of source.jsonObj.compose.exclude || []) {
|
|
1194
|
-
this.worker.
|
|
1250
|
+
await this.worker.checkAndYield('handleCompose#3');
|
|
1195
1251
|
this.hasExclusions = true;
|
|
1196
1252
|
await this.checkSource(c, expansion, filter, source.url, ts, null, source);
|
|
1197
1253
|
}
|
|
@@ -1205,7 +1261,7 @@ class ValueSetExpander {
|
|
|
1205
1261
|
|
|
1206
1262
|
let i = 0;
|
|
1207
1263
|
for (const c of source.jsonObj.compose.exclude || []) {
|
|
1208
|
-
this.worker.
|
|
1264
|
+
await this.worker.checkAndYield('handleCompose#4');
|
|
1209
1265
|
await this.excludeCodes(c, "ValueSet.compose.exclude[" + i + "]", source, filter, expansion, this.excludeInactives(source), notClosed);
|
|
1210
1266
|
}
|
|
1211
1267
|
|
|
@@ -1218,7 +1274,7 @@ class ValueSetExpander {
|
|
|
1218
1274
|
return 0;
|
|
1219
1275
|
});
|
|
1220
1276
|
for (const c of includes) {
|
|
1221
|
-
this.worker.
|
|
1277
|
+
await this.worker.checkAndYield('handleCompose#5');
|
|
1222
1278
|
await this.includeCodes(c, "ValueSet.compose.include[" + i + "]", source, source.jsonObj.compose, filter, expansion, this.excludeInactives(source), notClosed);
|
|
1223
1279
|
i++;
|
|
1224
1280
|
}
|
|
@@ -1346,7 +1402,7 @@ class ValueSetExpander {
|
|
|
1346
1402
|
}
|
|
1347
1403
|
|
|
1348
1404
|
this.worker.opContext.log('start working');
|
|
1349
|
-
this.worker.
|
|
1405
|
+
await this.worker.checkAndYield('expand');
|
|
1350
1406
|
|
|
1351
1407
|
let notClosed = { value : false};
|
|
1352
1408
|
|
|
@@ -1433,7 +1489,7 @@ class ValueSetExpander {
|
|
|
1433
1489
|
let t = 0;
|
|
1434
1490
|
let o = 0;
|
|
1435
1491
|
for (let i = 0; i < list.length; i++) {
|
|
1436
|
-
this.worker.
|
|
1492
|
+
await this.worker.checkAndYield('expand#1');
|
|
1437
1493
|
const c = list[i];
|
|
1438
1494
|
if (this.map.has(this.keyC(c))) {
|
|
1439
1495
|
o++;
|
package/tx/workers/worker.js
CHANGED
|
@@ -75,6 +75,15 @@ class TerminologyWorker {
|
|
|
75
75
|
this.opContext.deadCheck(place);
|
|
76
76
|
}
|
|
77
77
|
|
|
78
|
+
/**
|
|
79
|
+
* deadCheck plus a cooperative event-loop yield (see OperationContext.checkAndYield).
|
|
80
|
+
* Use this (awaited) from long-running loops in async code.
|
|
81
|
+
* @param {string} place - Location identifier for debugging
|
|
82
|
+
*/
|
|
83
|
+
async checkAndYield(place = 'unknown') {
|
|
84
|
+
await this.opContext.checkAndYield(place);
|
|
85
|
+
}
|
|
86
|
+
|
|
78
87
|
/**
|
|
79
88
|
* Add cost diagnostics to an error
|
|
80
89
|
* @param {TooCostlyError} e - The error to enhance
|