fhirsmith 0.16.1 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +130 -0
- package/README.md +105 -30
- package/config-template.json +130 -4
- package/extension-tracker/extension-tracker.js +111 -3
- package/extension-tracker/readme.md +3 -0
- package/ips/ai-client.js +263 -0
- package/ips/composer.js +805 -0
- package/ips/editor-schema.js +289 -0
- package/ips/editor.js +290 -0
- package/ips/fhirpath-ips.js +397 -0
- package/ips/ips-codeable-concepts.md +1742 -0
- package/ips/ips-template.html +125 -0
- package/ips/ips-valuesets.js +159 -0
- package/ips/ips.js +1736 -0
- package/ips/membership.js +179 -0
- package/ips/pipeline.js +1012 -0
- package/ips/presenter.js +974 -0
- package/ips/profile-editor.js +578 -0
- package/ips/profiles.js +506 -0
- package/ips/readme.md +460 -0
- package/ips/render.js +1482 -0
- package/ips/search-params.js +83 -0
- package/ips/search.js +667 -0
- package/ips/share.js +178 -0
- package/ips/store.js +650 -0
- package/ips/terminology.js +395 -0
- package/ips/tx-client.js +202 -0
- package/ips/valuesets/CodeSystem-absent-unknown-uv-ips.json +206 -0
- package/ips/valuesets/ValueSet-absent-or-unknown-immunizations-uv-ips.json +59 -0
- package/ips/valuesets/ValueSet-ips-allergy-code.json +59 -0
- package/ips/valuesets/ValueSet-ips-allergy-manifestation.json +356 -0
- package/ips/valuesets/ValueSet-ips-allergy-substance.json +37 -0
- package/ips/valuesets/ValueSet-ips-body-site.json +26 -0
- package/ips/valuesets/ValueSet-ips-clinical-finding-or-none.json +35 -0
- package/ips/valuesets/ValueSet-ips-clinical-finding.json +26 -0
- package/ips/valuesets/ValueSet-ips-condition-code.json +72 -0
- package/ips/valuesets/ValueSet-ips-condition-severity.json +30 -0
- package/ips/valuesets/ValueSet-ips-condition-stage.json +26 -0
- package/ips/valuesets/ValueSet-ips-exposure-route.json +26 -0
- package/ips/valuesets/ValueSet-ips-imaging-procedure.json +25 -0
- package/ips/valuesets/ValueSet-ips-immunization-status-reason.json +43 -0
- package/ips/valuesets/ValueSet-ips-international-codes.json +25 -0
- package/ips/valuesets/ValueSet-ips-medication-ingredient.json +26 -0
- package/ips/valuesets/ValueSet-ips-medication-status-reason.json +37 -0
- package/ips/valuesets/ValueSet-ips-medication.json +50 -0
- package/ips/valuesets/ValueSet-ips-observation-code.json +18 -0
- package/ips/valuesets/ValueSet-ips-observation-interpretation.json +18 -0
- package/ips/valuesets/ValueSet-ips-procedure-followup.json +51 -0
- package/ips/valuesets/ValueSet-ips-procedure-outcome.json +30 -0
- package/ips/valuesets/ValueSet-ips-procedure-reason.json +37 -0
- package/ips/valuesets/ValueSet-ips-procedure.json +116 -0
- package/ips/valuesets/ValueSet-ips-route.json +46 -0
- package/ips/valuesets/ValueSet-ips-target-disease.json +171 -0
- package/ips/valuesets/ValueSet-ips-vaccine.json +31 -0
- package/ips/valuesets/elements.json +44 -0
- package/library/daily-report.js +422 -0
- package/library/fhir-search-values.js +138 -0
- package/library/html-server.js +32 -23
- package/library/html.js +9 -0
- package/library/mailer.js +77 -0
- package/library/narrative-check.js +222 -0
- package/library/npm-audit.js +25 -0
- package/library/package-manager.js +4 -2
- package/library/sqlite-session-store.js +29 -1
- package/package.json +8 -2
- package/packages/package-crawler.js +189 -82
- package/packages/packages.js +36 -63
- package/packages/utilities/dedupe-versions.js +22 -8
- package/publisher/publisher.js +29 -0
- package/registry/auth-store.js +366 -0
- package/registry/auth.js +872 -0
- package/registry/crawler.js +48 -1
- package/registry/documentation/api.md +162 -0
- package/registry/documentation/configuration.md +93 -0
- package/registry/documentation/documents.md +74 -0
- package/registry/documentation/internals.md +151 -0
- package/registry/documentation/readme.md +100 -0
- package/registry/documentation/security.md +199 -0
- package/registry/documents.js +309 -0
- package/registry/openapi.yaml +30 -0
- package/registry/registry-template.html +1 -1
- package/registry/registry.js +247 -54
- package/registry/uptime.js +154 -0
- package/release.sh +80 -0
- package/server.js +105 -44
- package/shl/shl.js +64 -9
- package/stats-db.js +93 -0
- package/stats.js +158 -1
- package/testing/readme.md +1 -1
- package/testing/search.js +2 -108
- package/testing/store.js +5 -0
- package/testing/testing.js +20 -0
- package/token/token-template.html +5 -4
- package/token/token.js +17 -10
- package/translations/Messages.properties +9 -0
- package/tx/README.md +52 -0
- package/tx/cm/cm-database.js +62 -0
- package/tx/cm/cm-package.js +20 -3
- package/tx/cs/cs-api.js +90 -14
- package/tx/cs/cs-cs.js +142 -19
- package/tx/cs/cs-icd11.js +366 -112
- package/tx/cs/cs-icf.js +418 -0
- package/tx/cs/cs-loinc.js +79 -39
- package/tx/cs/cs-omop.js +10 -3
- package/tx/cs/cs-snomed.js +918 -55
- package/tx/cs/cs-ucum.js +6 -0
- package/tx/cs/icd11-expressions.js +191 -78
- package/tx/importers/export-icd11-codesystem.js +58 -12
- package/tx/importers/icd11-schema.md +9 -5
- package/tx/library.js +107 -6
- package/tx/ocl/README.md +62 -6
- package/tx/ocl/cm-ocl.cjs +166 -44
- package/tx/ocl/cs-ocl.cjs +163 -13
- package/tx/ocl/resolve/public-access.js +143 -0
- package/tx/ocl/resolve/reference-resolver.js +458 -0
- package/tx/ocl/vs-ocl.cjs +167 -4
- package/tx/openapi-operations.js +4 -0
- package/tx/params.js +8 -1
- package/tx/problems.js +7 -5
- package/tx/provider.js +21 -5
- package/tx/sct/documentation.md +251 -0
- package/tx/sct/ecl.js +758 -46
- package/tx/sct/expressions.js +113 -52
- package/tx/syndication/syndication-feed.js +218 -0
- package/tx/syndication/syndication-store.js +415 -0
- package/tx/tx-html.js +37 -8
- package/tx/tx.fhir.org.yml +2 -3
- package/tx/tx.js +148 -5
- package/tx/vs/vs-database.js +26 -0
- package/tx/vs/vs-package.js +42 -10
- package/tx/vs/vs-vsac.js +25 -6
- package/tx/workers/expand.js +34 -7
- package/tx/workers/lookup.js +61 -6
- package/tx/workers/metadata.js +6 -1
- package/tx/workers/subsumes.js +1 -1
- package/tx/workers/translate.js +166 -15
- package/tx/workers/validate.js +128 -6
- package/utilities/make-sct-test-extension.js +264 -0
- package/utilities/submit-test-report.js +1 -1
- package/xig/xig.js +120 -12
- package/gr.sh +0 -3
- package/registry/readme.md +0 -124
|
@@ -0,0 +1,458 @@
|
|
|
1
|
+
// Client for OCL's $resolveReference operation.
|
|
2
|
+
//
|
|
3
|
+
// Answers "which OCL repo holds this canonical URL?" authoritatively, so callers
|
|
4
|
+
// stop iterating source/collection listings to find a matching canonical_url. See
|
|
5
|
+
// https://docs.openconceptlab.org/en/latest/oclapi/apireference/resolveReference.html
|
|
6
|
+
//
|
|
7
|
+
// Namespace is deliberately NOT supported: FHIR operations carry no namespace
|
|
8
|
+
// parameter, so every resolution here runs in OCL's global namespace. Namespace
|
|
9
|
+
// semantics (multi-tenant discrimination, sandboxing) are an open discussion with
|
|
10
|
+
// the OCL team, not something to encode client-side yet.
|
|
11
|
+
//
|
|
12
|
+
// Plain .js rather than tx/ocl's .cjs+stub convention: jest's collectCoverageFrom
|
|
13
|
+
// globs **/*.js only, so a .cjs module would be invisible to coverage.
|
|
14
|
+
|
|
15
|
+
const { PublicAccessChecker, safeForLog, mapLimit } = require('./public-access');
|
|
16
|
+
|
|
17
|
+
const RESOLVE_PATH = '/$resolveReference/';
|
|
18
|
+
|
|
19
|
+
// The live instance rejects large batches (403 somewhere between 150 and 200
|
|
20
|
+
// references); 100 resolved in ~1.1s. Chunking is transparent to callers.
|
|
21
|
+
const MAX_BATCH_SIZE = 100;
|
|
22
|
+
|
|
23
|
+
// Caps so that client-supplied references cannot grow memory or outbound traffic
|
|
24
|
+
// without bound (the resolver is reachable from any public terminology request):
|
|
25
|
+
// - the cache is a bounded LRU, so distinct references evict the oldest rather
|
|
26
|
+
// than accumulating forever;
|
|
27
|
+
// - references longer than the limit are treated as unresolved and never sent to
|
|
28
|
+
// OCL (nothing legitimate is anywhere near this long).
|
|
29
|
+
const DEFAULT_CACHE_LIMIT = 5000;
|
|
30
|
+
const MAX_REFERENCE_LENGTH = 2048;
|
|
31
|
+
|
|
32
|
+
// A transient 401/403 (rate limit, a brief credential hiccup, an oversized batch
|
|
33
|
+
// on a busy instance) backs the resolver off for a while rather than disabling it
|
|
34
|
+
// for the whole process life. 404 (endpoint not implemented) stays permanent.
|
|
35
|
+
const DEFAULT_AUTH_BACKOFF_MS = 60_000;
|
|
36
|
+
|
|
37
|
+
// Cap on concurrent public_access repo GETs when gating a batch of resolutions.
|
|
38
|
+
const PUBLIC_ACCESS_CONCURRENCY = 8;
|
|
39
|
+
|
|
40
|
+
// Org-only visibility policy: an artifact is expected to live in an organization
|
|
41
|
+
// to be visible through the terminology service. The path must be a concrete
|
|
42
|
+
// /orgs/<org>/(sources|collections)/<id>/ repo path — this both enforces the
|
|
43
|
+
// policy and keeps repoUrl a safe relative OCL path (so it can never redirect an
|
|
44
|
+
// authenticated request, carrying our token, to an arbitrary host).
|
|
45
|
+
const REPO_PATH_PATTERN = /^\/orgs\/[^/]+\/(sources|collections)\/[^/]+\//;
|
|
46
|
+
|
|
47
|
+
// Reference object fields forwarded to OCL. `namespace` is intentionally absent.
|
|
48
|
+
const BODY_FIELDS = [
|
|
49
|
+
'url',
|
|
50
|
+
'version',
|
|
51
|
+
'code',
|
|
52
|
+
'display',
|
|
53
|
+
'id',
|
|
54
|
+
'filter',
|
|
55
|
+
'cascade',
|
|
56
|
+
'includeExclude',
|
|
57
|
+
'resourceType'
|
|
58
|
+
];
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* True for a relative OCL repo path the terminology service may serve — i.e. an
|
|
62
|
+
* organization-owned source or collection (`/orgs/CIEL/sources/CIEL/`). User-owned
|
|
63
|
+
* paths (`/users/joe/...`), absolute URLs, and anything containing path traversal
|
|
64
|
+
* are rejected.
|
|
65
|
+
*/
|
|
66
|
+
function isOclRepoPath(value) {
|
|
67
|
+
const s = String(value == null ? '' : value).trim();
|
|
68
|
+
if (s.includes('..')) {
|
|
69
|
+
// No legitimate OCL repo path contains `..`; reject traversal outright.
|
|
70
|
+
return false;
|
|
71
|
+
}
|
|
72
|
+
return REPO_PATH_PATTERN.test(s);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* A repoUrl we can safely GET against the OCL base URL: a relative, same-host path
|
|
77
|
+
* with no traversal. Absolute or protocol-relative URLs are rejected so an authed
|
|
78
|
+
* request (carrying our token) can never be redirected to an arbitrary host. This
|
|
79
|
+
* is the SSRF guard and is deliberately separate from the org-only policy: a
|
|
80
|
+
* same-host `/users/...` path is "safe" here but still rejected by isOrgOwned.
|
|
81
|
+
*/
|
|
82
|
+
function isSafeRelativeOclPath(value) {
|
|
83
|
+
const s = String(value == null ? '' : value).trim();
|
|
84
|
+
return s.startsWith('/') && !s.startsWith('//') && !s.includes('..');
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Org-only policy check for an OCL repo payload or resolve result. The path must
|
|
89
|
+
* be a valid org repo path AND, when an explicit owner_type is present, it must be
|
|
90
|
+
* an Organization. owner_type alone can never override the path check — a payload
|
|
91
|
+
* claiming owner_type:"Organization" on a non-org (or off-host) URL is not served.
|
|
92
|
+
*/
|
|
93
|
+
function isOrgOwned(repo) {
|
|
94
|
+
if (!repo || typeof repo !== 'object') {
|
|
95
|
+
return false;
|
|
96
|
+
}
|
|
97
|
+
if (!isOclRepoPath(repo.url)) {
|
|
98
|
+
return false;
|
|
99
|
+
}
|
|
100
|
+
const ownerType = repo.owner_type || repo.ownerType || null;
|
|
101
|
+
return ownerType ? ownerType === 'Organization' : true;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Accepts either a relative/canonical URL string or an expanded reference object,
|
|
106
|
+
* returning the request-body form OCL expects.
|
|
107
|
+
*/
|
|
108
|
+
function normalizeReference(ref) {
|
|
109
|
+
if (typeof ref === 'string') {
|
|
110
|
+
const url = ref.trim();
|
|
111
|
+
if (!url) {
|
|
112
|
+
throw new Error('OCL reference string cannot be empty');
|
|
113
|
+
}
|
|
114
|
+
return url;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
if (!ref || typeof ref !== 'object' || Array.isArray(ref)) {
|
|
118
|
+
throw new Error(`Invalid OCL reference: expected a string or object, got ${typeof ref}`);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
const url = String(ref.url == null ? '' : ref.url).trim();
|
|
122
|
+
if (!url) {
|
|
123
|
+
throw new Error('OCL reference object requires a url');
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
const body = {};
|
|
127
|
+
for (const field of BODY_FIELDS) {
|
|
128
|
+
if (ref[field] !== undefined && ref[field] !== null) {
|
|
129
|
+
body[field] = ref[field];
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
return body;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function cacheKey(body) {
|
|
136
|
+
return typeof body === 'string' ? body : JSON.stringify(body);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// Length of the reference's URL, used for the inbound length cap.
|
|
140
|
+
function referenceLength(body) {
|
|
141
|
+
return (typeof body === 'string' ? body : String(body?.url ?? '')).length;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
function unresolved(request) {
|
|
146
|
+
return {
|
|
147
|
+
resolved: false,
|
|
148
|
+
repoUrl: null,
|
|
149
|
+
canonical: null,
|
|
150
|
+
ownerType: null,
|
|
151
|
+
publicAccess: null,
|
|
152
|
+
resolutionUrl: null,
|
|
153
|
+
registryEntry: null,
|
|
154
|
+
referenceType: null,
|
|
155
|
+
request,
|
|
156
|
+
result: null
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function normalizeResult(entry, request) {
|
|
161
|
+
if (!entry || typeof entry !== 'object') {
|
|
162
|
+
return unresolved(request);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
const result = entry.result && typeof entry.result === 'object' ? entry.result : null;
|
|
166
|
+
const repoUrl = result && result.url ? result.url : null;
|
|
167
|
+
|
|
168
|
+
// Only ever hand back a repoUrl that is a safe same-host relative path. Anything
|
|
169
|
+
// absolute or off-host (or containing traversal) is treated as unresolved, so a
|
|
170
|
+
// surprising OCL response can never cause a caller to issue an authenticated
|
|
171
|
+
// request (with our token) to an arbitrary host. The org-only policy is applied
|
|
172
|
+
// separately by the caller (isOrgOwned), so a same-host /users/ path survives to
|
|
173
|
+
// there and is rejected with a logged reason.
|
|
174
|
+
if (repoUrl && !isSafeRelativeOclPath(repoUrl)) {
|
|
175
|
+
return unresolved(entry.request === undefined ? request : entry.request);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
return {
|
|
179
|
+
resolved: Boolean(entry.resolved) && Boolean(repoUrl),
|
|
180
|
+
repoUrl,
|
|
181
|
+
// OCL returns the repo's own canonical_url and owner_type (richer than the
|
|
182
|
+
// documented example). Prefer them over echoing the request back: they are
|
|
183
|
+
// authoritative, the request is just whatever spelling the caller used.
|
|
184
|
+
canonical: result ? (result.canonical_url || result.canonicalUrl || null) : null,
|
|
185
|
+
ownerType: result ? (result.owner_type || result.ownerType || null) : null,
|
|
186
|
+
// public_access is not in the $resolveReference result today (the live capture
|
|
187
|
+
// omits it), so this is usually null and the repo is fetched to check — kept
|
|
188
|
+
// anyway so the gate is free the day OCL does inline it.
|
|
189
|
+
publicAccess: result ? (result.public_access ?? result.publicAccess ?? null) : null,
|
|
190
|
+
resolutionUrl: entry.resolution_url || null,
|
|
191
|
+
// Kept even though every observed response so far carries null: whether a URL
|
|
192
|
+
// Registry entry was involved is exactly what the OCL-team discussion needs.
|
|
193
|
+
registryEntry: entry.url_registry_entry || null,
|
|
194
|
+
referenceType: entry.reference_type || null,
|
|
195
|
+
request: entry.request === undefined ? request : entry.request,
|
|
196
|
+
result
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
class OclReferenceResolver {
|
|
201
|
+
#httpClient;
|
|
202
|
+
#logger;
|
|
203
|
+
#cache = new Map();
|
|
204
|
+
#cacheLimit;
|
|
205
|
+
#enabled;
|
|
206
|
+
#disabledReason = null;
|
|
207
|
+
#authBackoffMs;
|
|
208
|
+
#backoffUntil = 0;
|
|
209
|
+
#publicAccess;
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* @param {object} options
|
|
213
|
+
* @param {object} options.httpClient - axios instance from createOclHttpClient
|
|
214
|
+
* @param {string} [options.token] - when absent the resolver stays disabled:
|
|
215
|
+
* $resolveReference is authenticated on every OCL instance probed, while the
|
|
216
|
+
* listing endpoints it replaces are public, so a tokenless call is a
|
|
217
|
+
* guaranteed 401
|
|
218
|
+
* @param {object} [options.logger] - module logger; defaults to console only as
|
|
219
|
+
* a last resort (callers pass their child logger)
|
|
220
|
+
* @param {number} [options.cacheLimit] - max cached references (bounded LRU)
|
|
221
|
+
* @param {number} [options.authBackoffMs] - cooldown after a transient 401/403
|
|
222
|
+
* @param {PublicAccessChecker} [options.publicAccess] - shared public-access gate
|
|
223
|
+
* (so discovery and the resolver share one repoUrl cache); created if absent
|
|
224
|
+
*/
|
|
225
|
+
constructor({ httpClient, token = null, logger = console, cacheLimit = DEFAULT_CACHE_LIMIT, authBackoffMs = DEFAULT_AUTH_BACKOFF_MS, publicAccess = null } = {}) {
|
|
226
|
+
if (!httpClient) {
|
|
227
|
+
throw new Error('OCL reference resolver requires an http client');
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
this.#httpClient = httpClient;
|
|
231
|
+
this.#logger = logger;
|
|
232
|
+
this.#cacheLimit = cacheLimit > 0 ? cacheLimit : DEFAULT_CACHE_LIMIT;
|
|
233
|
+
this.#authBackoffMs = authBackoffMs >= 0 ? authBackoffMs : DEFAULT_AUTH_BACKOFF_MS;
|
|
234
|
+
this.#publicAccess = publicAccess || new PublicAccessChecker({ httpClient, logger });
|
|
235
|
+
this.#enabled = Boolean(token);
|
|
236
|
+
if (!this.#enabled) {
|
|
237
|
+
this.#disabledReason = 'no token configured';
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
isEnabled() {
|
|
242
|
+
return this.#enabled && Date.now() >= this.#backoffUntil;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
get disabledReason() {
|
|
246
|
+
return this.#disabledReason;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// Permanent: no token, or the endpoint does not exist on this instance.
|
|
250
|
+
#disable(reason) {
|
|
251
|
+
this.#enabled = false;
|
|
252
|
+
this.#disabledReason = reason;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
// Transient: back off for a cooldown, then let requests resume.
|
|
256
|
+
#backOff(reason) {
|
|
257
|
+
this.#backoffUntil = Date.now() + this.#authBackoffMs;
|
|
258
|
+
this.#disabledReason = reason;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
#cacheTouch(key) {
|
|
262
|
+
// Re-insert to mark most-recently-used.
|
|
263
|
+
const value = this.#cache.get(key);
|
|
264
|
+
this.#cache.delete(key);
|
|
265
|
+
this.#cache.set(key, value);
|
|
266
|
+
return value;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
#cachePut(key, value) {
|
|
270
|
+
if (this.#cache.has(key)) {
|
|
271
|
+
this.#cache.delete(key);
|
|
272
|
+
}
|
|
273
|
+
this.#cache.set(key, value);
|
|
274
|
+
if (this.#cache.size > this.#cacheLimit) {
|
|
275
|
+
const oldest = this.#cache.keys().next().value;
|
|
276
|
+
this.#cache.delete(oldest);
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* Resolve one reference. Returns null when the resolver is unavailable, so the
|
|
282
|
+
* caller falls back to its existing path.
|
|
283
|
+
*/
|
|
284
|
+
async resolve(ref) {
|
|
285
|
+
const results = await this.resolveReferences([ref]);
|
|
286
|
+
return results ? results[0] : null;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/**
|
|
290
|
+
* Resolve many references, chunked into POSTs of at most MAX_BATCH_SIZE.
|
|
291
|
+
* Results come back in the caller's order.
|
|
292
|
+
*
|
|
293
|
+
* @param {object} [options]
|
|
294
|
+
* @param {boolean} [options.bypassCache] - re-ask OCL even for cached entries
|
|
295
|
+
* (the cache is still updated). Used by discovery refresh, where a source's
|
|
296
|
+
* default version may have changed since it was last resolved.
|
|
297
|
+
* @returns {Promise<Array|null>} null when the resolver is unavailable (use fallback)
|
|
298
|
+
*/
|
|
299
|
+
async resolveReferences(refs, { bypassCache = false } = {}) {
|
|
300
|
+
if (!this.isEnabled()) {
|
|
301
|
+
return null;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
const list = Array.isArray(refs) ? refs : [refs];
|
|
305
|
+
if (list.length === 0) {
|
|
306
|
+
return [];
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
const bodies = list.map(normalizeReference);
|
|
310
|
+
const output = new Array(bodies.length).fill(null);
|
|
311
|
+
const misses = [];
|
|
312
|
+
|
|
313
|
+
bodies.forEach((body, index) => {
|
|
314
|
+
// Length cap: an over-long reference is never cached nor sent to OCL.
|
|
315
|
+
if (referenceLength(body) > MAX_REFERENCE_LENGTH) {
|
|
316
|
+
output[index] = unresolved(body);
|
|
317
|
+
return;
|
|
318
|
+
}
|
|
319
|
+
const key = cacheKey(body);
|
|
320
|
+
if (!bypassCache && this.#cache.has(key)) {
|
|
321
|
+
output[index] = this.#cacheTouch(key);
|
|
322
|
+
} else {
|
|
323
|
+
misses.push({ body, index });
|
|
324
|
+
}
|
|
325
|
+
});
|
|
326
|
+
|
|
327
|
+
for (let start = 0; start < misses.length; start += MAX_BATCH_SIZE) {
|
|
328
|
+
const chunk = misses.slice(start, start + MAX_BATCH_SIZE);
|
|
329
|
+
const outcome = await this.#resolveChunk(chunk, output);
|
|
330
|
+
if (outcome === null) {
|
|
331
|
+
// Resolver became unavailable; the caller falls back wholesale rather
|
|
332
|
+
// than acting on a half-resolved set.
|
|
333
|
+
return null;
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
return output;
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
async #resolveChunk(chunk, output) {
|
|
341
|
+
let response;
|
|
342
|
+
try {
|
|
343
|
+
response = await this.#httpClient.post(RESOLVE_PATH, chunk.map(m => m.body));
|
|
344
|
+
} catch (error) {
|
|
345
|
+
return this.#handleError(error, chunk, output);
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
const payload = Array.isArray(response?.data)
|
|
349
|
+
? response.data
|
|
350
|
+
: response?.data == null
|
|
351
|
+
? []
|
|
352
|
+
: [response.data];
|
|
353
|
+
|
|
354
|
+
// Results are positional. On a count mismatch we cannot know which result
|
|
355
|
+
// belongs to which reference, so treat the chunk as unresolved rather than
|
|
356
|
+
// silently attributing a resolution to the wrong canonical.
|
|
357
|
+
if (payload.length !== chunk.length) {
|
|
358
|
+
this.#logger.error(
|
|
359
|
+
`$resolveReference returned ${payload.length} result(s) for ${chunk.length} reference(s); discarding to avoid misaligned results`
|
|
360
|
+
);
|
|
361
|
+
for (const { body, index } of chunk) {
|
|
362
|
+
output[index] = unresolved(body);
|
|
363
|
+
}
|
|
364
|
+
return output;
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
// Pass 1 (sync): normalize + org-only policy.
|
|
368
|
+
const prelim = chunk.map(({ body, index }, position) => {
|
|
369
|
+
let value = normalizeResult(payload[position], body);
|
|
370
|
+
// Org-only policy: a canonical resolving to a user-owned repo is treated as
|
|
371
|
+
// unresolved — user artifacts are experimental and not visible through the
|
|
372
|
+
// terminology service. Cached: the policy outcome is deterministic.
|
|
373
|
+
if (value.resolved && !isOrgOwned({ owner_type: value.ownerType, url: value.repoUrl })) {
|
|
374
|
+
this.#logger.info(
|
|
375
|
+
`$resolveReference resolved ${safeForLog(cacheKey(body))} to a user-owned repo (${safeForLog(value.repoUrl)}); org-only policy treats it as unresolved`
|
|
376
|
+
);
|
|
377
|
+
value = unresolved(body);
|
|
378
|
+
}
|
|
379
|
+
return { body, index, value };
|
|
380
|
+
});
|
|
381
|
+
|
|
382
|
+
// Pass 2 (async): public-access gate. A private repo (public_access other than
|
|
383
|
+
// View/Edit) that the configured token can see must NOT be served through the
|
|
384
|
+
// public terminology server, so anything not positively confirmed public is
|
|
385
|
+
// dropped. public_access is not in the resolve result today, so the shared
|
|
386
|
+
// checker fetches the repo once (cached by repoUrl, concurrency-limited). A
|
|
387
|
+
// transient fetch error is NOT cached — that reference is left out of the
|
|
388
|
+
// result cache below and retried next time rather than pinned as unresolved.
|
|
389
|
+
await mapLimit(prelim, PUBLIC_ACCESS_CONCURRENCY, async entry => {
|
|
390
|
+
if (!entry.value.resolved) {
|
|
391
|
+
return;
|
|
392
|
+
}
|
|
393
|
+
const { public: isPublic, determined } = await this.#publicAccess.check(entry.value.repoUrl, entry.value.publicAccess);
|
|
394
|
+
if (!isPublic) {
|
|
395
|
+
this.#logger.info(
|
|
396
|
+
`$resolveReference resolved ${safeForLog(cacheKey(entry.body))} to a non-public repo ` +
|
|
397
|
+
`(${safeForLog(entry.value.repoUrl)}); not served`
|
|
398
|
+
);
|
|
399
|
+
entry.value = unresolved(entry.body);
|
|
400
|
+
entry.skipCache = !determined;
|
|
401
|
+
}
|
|
402
|
+
});
|
|
403
|
+
|
|
404
|
+
for (const { body, index, value, skipCache } of prelim) {
|
|
405
|
+
if (!skipCache) {
|
|
406
|
+
this.#cachePut(cacheKey(body), value);
|
|
407
|
+
}
|
|
408
|
+
output[index] = value;
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
return output;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
#handleError(error, misses, output) {
|
|
415
|
+
const status = error?.response?.status;
|
|
416
|
+
|
|
417
|
+
if (status === 404) {
|
|
418
|
+
this.#disable('endpoint not implemented (404)');
|
|
419
|
+
this.#logger.info(
|
|
420
|
+
`$resolveReference is not available on this instance (404); using listing search instead`
|
|
421
|
+
);
|
|
422
|
+
return null;
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
if (status === 401 || status === 403) {
|
|
426
|
+
// Transient: back off and retry after a cooldown rather than disabling for
|
|
427
|
+
// the life of the process. A busy public instance can 401/403 briefly (rate
|
|
428
|
+
// limiting, an oversized batch) without our credentials being wrong.
|
|
429
|
+
this.#backOff(`not authorised (${status})`);
|
|
430
|
+
this.#logger.warn(
|
|
431
|
+
`$resolveReference rejected our credentials (${status}); backing off, using listing search meanwhile`
|
|
432
|
+
);
|
|
433
|
+
return null;
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
if (status === 400) {
|
|
437
|
+
// Our request body is wrong — a bug on this side. Don't disable: a later,
|
|
438
|
+
// well-formed batch may be fine.
|
|
439
|
+
const detail = error?.response?.data?.detail || error.message;
|
|
440
|
+
this.#logger.error(`$resolveReference rejected the request body: ${safeForLog(detail)}`);
|
|
441
|
+
for (const { body, index } of misses) {
|
|
442
|
+
output[index] = unresolved(body);
|
|
443
|
+
}
|
|
444
|
+
return output;
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
this.#logger.warn(`$resolveReference failed: ${safeForLog(error.message)}`);
|
|
448
|
+
return null;
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
module.exports = {
|
|
453
|
+
OclReferenceResolver,
|
|
454
|
+
normalizeReference,
|
|
455
|
+
isOclRepoPath,
|
|
456
|
+
isOrgOwned,
|
|
457
|
+
RESOLVE_PATH
|
|
458
|
+
};
|
package/tx/ocl/vs-ocl.cjs
CHANGED
|
@@ -9,6 +9,8 @@ const { TxParameters } = require('../params');
|
|
|
9
9
|
const { OCLSourceCodeSystemFactory, OCLBackgroundJobQueue } = require('./cs-ocl');
|
|
10
10
|
const { PAGE_SIZE, CONCEPT_PAGE_SIZE, FILTERED_CONCEPT_PAGE_SIZE, COLD_CACHE_FRESHNESS_MS } = require('./shared/constants');
|
|
11
11
|
const { createOclHttpClient } = require('./http/client');
|
|
12
|
+
const { OclReferenceResolver, isOrgOwned } = require('./resolve/reference-resolver');
|
|
13
|
+
const { PublicAccessChecker } = require('./resolve/public-access');
|
|
12
14
|
const { CACHE_VS_DIR, getCacheFilePath } = require('./cache/cache-paths');
|
|
13
15
|
const { ensureCacheDirectories, getColdCacheAgeMs, formatCacheAgeMinutes } = require('./cache/cache-utils');
|
|
14
16
|
const { computeValueSetExpansionFingerprint } = require('./fingerprint/fingerprint');
|
|
@@ -44,6 +46,16 @@ class OCLValueSetProvider extends AbstractValueSetProvider {
|
|
|
44
46
|
this.baseUrl = http.baseUrl;
|
|
45
47
|
this.httpClient = http.client;
|
|
46
48
|
|
|
49
|
+
// Resolves a canonical to its OCL repo authoritatively. Disabled without a
|
|
50
|
+
// token, in which case the search paths below run exactly as before.
|
|
51
|
+
this.publicAccess = new PublicAccessChecker({ httpClient: this.httpClient, logger: oclVsLog });
|
|
52
|
+
this.referenceResolver = new OclReferenceResolver({
|
|
53
|
+
httpClient: this.httpClient,
|
|
54
|
+
token: options.token || null,
|
|
55
|
+
logger: oclVsLog,
|
|
56
|
+
publicAccess: this.publicAccess
|
|
57
|
+
});
|
|
58
|
+
|
|
47
59
|
this.valueSetMap = new Map();
|
|
48
60
|
this._idMap = new Map();
|
|
49
61
|
this.collectionMeta = new Map();
|
|
@@ -726,6 +738,11 @@ class OCLValueSetProvider extends AbstractValueSetProvider {
|
|
|
726
738
|
}
|
|
727
739
|
|
|
728
740
|
const sourceKeys = await this.#fetchCollectionSourceKeys(meta.conceptsUrl, meta.owner || null);
|
|
741
|
+
// Resolve every source's canonical in a single $resolveReference batch and seed
|
|
742
|
+
// the per-source cache, replacing one sequential GET per source. The loop below
|
|
743
|
+
// then reads from cache; anything the batch could not resolve falls through to
|
|
744
|
+
// the individual GET in #getSourceCanonicalUrl, so behaviour is unchanged.
|
|
745
|
+
await this.#primeSourceCanonicalsBatch(sourceKeys);
|
|
729
746
|
for (const { owner, source } of sourceKeys) {
|
|
730
747
|
const systemUrl = normalizeCanonicalSystem(await this.#getSourceCanonicalUrl(owner, source));
|
|
731
748
|
if (systemUrl && !seen.has(systemUrl)) {
|
|
@@ -1185,6 +1202,15 @@ class OCLValueSetProvider extends AbstractValueSetProvider {
|
|
|
1185
1202
|
}
|
|
1186
1203
|
|
|
1187
1204
|
const pending = (async () => {
|
|
1205
|
+
// Ask OCL which repo holds this canonical. An authoritative answer skips
|
|
1206
|
+
// the per-org text search below; without one (no token, or the endpoint is
|
|
1207
|
+
// unavailable) we fall through to it.
|
|
1208
|
+
const resolvedCollection = await this.#resolveCollectionViaReference(canonicalUrl, version);
|
|
1209
|
+
if (resolvedCollection) {
|
|
1210
|
+
this.collectionByCanonicalCache.set(lookupKey, resolvedCollection);
|
|
1211
|
+
return resolvedCollection;
|
|
1212
|
+
}
|
|
1213
|
+
|
|
1188
1214
|
const organizations = await this.#fetchOrganizationIds();
|
|
1189
1215
|
const endpoints = organizations.length > 0
|
|
1190
1216
|
? organizations.map(orgId => `/orgs/${encodeURIComponent(orgId)}/collections/`)
|
|
@@ -1251,6 +1277,56 @@ class OCLValueSetProvider extends AbstractValueSetProvider {
|
|
|
1251
1277
|
}
|
|
1252
1278
|
}
|
|
1253
1279
|
|
|
1280
|
+
/**
|
|
1281
|
+
* Resolve a ValueSet canonical to its OCL collection via $resolveReference,
|
|
1282
|
+
* then fetch the collection object so it matches the shape the text-search path
|
|
1283
|
+
* returns. Returns null when the resolver is disabled (no token), the endpoint
|
|
1284
|
+
* is unavailable, OCL cannot resolve it, or the resolved repo is not a
|
|
1285
|
+
* collection -- in every case the caller falls back to the search.
|
|
1286
|
+
*/
|
|
1287
|
+
async #resolveCollectionViaReference(canonicalUrl, version) {
|
|
1288
|
+
let resolved;
|
|
1289
|
+
try {
|
|
1290
|
+
resolved = await this.referenceResolver.resolve(
|
|
1291
|
+
version ? { url: canonicalUrl, version } : canonicalUrl
|
|
1292
|
+
);
|
|
1293
|
+
} catch (error) {
|
|
1294
|
+
oclVsLog.warn(`$resolveReference lookup failed for ${canonicalUrl}: ${error.message}`);
|
|
1295
|
+
return null;
|
|
1296
|
+
}
|
|
1297
|
+
|
|
1298
|
+
if (!resolved?.resolved || !resolved.repoUrl) {
|
|
1299
|
+
return null;
|
|
1300
|
+
}
|
|
1301
|
+
|
|
1302
|
+
// $resolveReference resolves any repo type; only collections are ValueSets.
|
|
1303
|
+
if (!/[/]collections[/]/.test(resolved.repoUrl)) {
|
|
1304
|
+
return null;
|
|
1305
|
+
}
|
|
1306
|
+
|
|
1307
|
+
let collection;
|
|
1308
|
+
try {
|
|
1309
|
+
const response = await this.httpClient.get(resolved.repoUrl);
|
|
1310
|
+
collection = response?.data || null;
|
|
1311
|
+
} catch (error) {
|
|
1312
|
+
oclVsLog.warn(`failed to fetch resolved collection ${resolved.repoUrl}: ${error.message}`);
|
|
1313
|
+
return null;
|
|
1314
|
+
}
|
|
1315
|
+
|
|
1316
|
+
if (!collection || typeof collection !== 'object' || !collection.id) {
|
|
1317
|
+
return null;
|
|
1318
|
+
}
|
|
1319
|
+
|
|
1320
|
+
// Honour an explicit version: the resolved repo is HEAD unless OCL matched a
|
|
1321
|
+
// version, so a mismatch means fall back to the version-aware search.
|
|
1322
|
+
if (version && (collection.version || null) !== version) {
|
|
1323
|
+
return null;
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
oclVsLog.info(`$resolveReference resolved ${canonicalUrl} -> ${resolved.repoUrl}`);
|
|
1327
|
+
return collection;
|
|
1328
|
+
}
|
|
1329
|
+
|
|
1254
1330
|
#valueSetMetadataSignature(vs) {
|
|
1255
1331
|
const meta = this.#getCollectionMeta(vs);
|
|
1256
1332
|
const payload = {
|
|
@@ -1311,10 +1387,37 @@ class OCLValueSetProvider extends AbstractValueSetProvider {
|
|
|
1311
1387
|
}
|
|
1312
1388
|
|
|
1313
1389
|
async #fetchCollectionsForDiscovery() {
|
|
1390
|
+
// Prefer the global listing: one paginated crawl instead of one listing per
|
|
1391
|
+
// org (N+1 requests). Org-only policy: user-owned collections are
|
|
1392
|
+
// experimental by convention and not visible to the terminology service.
|
|
1393
|
+
try {
|
|
1394
|
+
const all = await this.#fetchAllPages('/collections/');
|
|
1395
|
+
if (Array.isArray(all) && all.length > 0) {
|
|
1396
|
+
const seen = new Set();
|
|
1397
|
+
const orgOwned = [];
|
|
1398
|
+
for (const collection of all) {
|
|
1399
|
+
if (!collection || typeof collection !== 'object' || !isOrgOwned(collection)) {
|
|
1400
|
+
continue;
|
|
1401
|
+
}
|
|
1402
|
+
const key = this.#collectionIdentity(collection);
|
|
1403
|
+
if (seen.has(key)) {
|
|
1404
|
+
continue;
|
|
1405
|
+
}
|
|
1406
|
+
seen.add(key);
|
|
1407
|
+
orgOwned.push(collection);
|
|
1408
|
+
}
|
|
1409
|
+
if (orgOwned.length > 0) {
|
|
1410
|
+
// Public-only: a private collection the token can see must not be served.
|
|
1411
|
+
return await this.publicAccess.filterPublic(orgOwned);
|
|
1412
|
+
}
|
|
1413
|
+
}
|
|
1414
|
+
} catch (error) {
|
|
1415
|
+
oclVsLog.warn(`Global /collections/ listing failed (${error.message}); falling back to per-org discovery`);
|
|
1416
|
+
}
|
|
1417
|
+
|
|
1314
1418
|
const organizations = await this.#fetchOrganizationIds();
|
|
1315
1419
|
if (organizations.length === 0) {
|
|
1316
|
-
|
|
1317
|
-
return await this.#fetchAllPages('/collections/');
|
|
1420
|
+
return [];
|
|
1318
1421
|
}
|
|
1319
1422
|
|
|
1320
1423
|
const allCollections = [];
|
|
@@ -1324,7 +1427,9 @@ class OCLValueSetProvider extends AbstractValueSetProvider {
|
|
|
1324
1427
|
const endpoint = `/orgs/${encodeURIComponent(orgId)}/collections/`;
|
|
1325
1428
|
const collections = await this.#fetchAllPages(endpoint);
|
|
1326
1429
|
for (const collection of collections) {
|
|
1327
|
-
|
|
1430
|
+
// Org-only policy applies to the per-org fallback too, not just the global
|
|
1431
|
+
// listing (an org listing can still surface non-org-owned entries).
|
|
1432
|
+
if (!collection || typeof collection !== 'object' || !isOrgOwned(collection)) {
|
|
1328
1433
|
continue;
|
|
1329
1434
|
}
|
|
1330
1435
|
const key = this.#collectionIdentity(collection);
|
|
@@ -1336,7 +1441,8 @@ class OCLValueSetProvider extends AbstractValueSetProvider {
|
|
|
1336
1441
|
}
|
|
1337
1442
|
}
|
|
1338
1443
|
|
|
1339
|
-
|
|
1444
|
+
// Public-only: a private collection the token can see must not be served.
|
|
1445
|
+
return await this.publicAccess.filterPublic(allCollections);
|
|
1340
1446
|
}
|
|
1341
1447
|
|
|
1342
1448
|
async #fetchOrganizationIds() {
|
|
@@ -1783,6 +1889,63 @@ class OCLValueSetProvider extends AbstractValueSetProvider {
|
|
|
1783
1889
|
}
|
|
1784
1890
|
}
|
|
1785
1891
|
|
|
1892
|
+
/**
|
|
1893
|
+
* Resolve many sources' canonicals in one $resolveReference batch, seeding
|
|
1894
|
+
* sourceCanonicalCache. A no-op when the resolver is disabled (no token) or the
|
|
1895
|
+
* batch fails: #getSourceCanonicalUrl then falls back to its per-source GET, so
|
|
1896
|
+
* this only ever saves round trips, never changes the result.
|
|
1897
|
+
*/
|
|
1898
|
+
async #primeSourceCanonicalsBatch(sourceKeys) {
|
|
1899
|
+
if (!this.referenceResolver.isEnabled()) {
|
|
1900
|
+
return;
|
|
1901
|
+
}
|
|
1902
|
+
|
|
1903
|
+
// Only sources not already cached, deduplicated by owner|source.
|
|
1904
|
+
const pending = [];
|
|
1905
|
+
const seen = new Set();
|
|
1906
|
+
for (const { owner, source } of sourceKeys || []) {
|
|
1907
|
+
if (!owner || !source) {
|
|
1908
|
+
continue;
|
|
1909
|
+
}
|
|
1910
|
+
const key = `${owner}|${source}`;
|
|
1911
|
+
if (this.sourceCanonicalCache.has(key) || seen.has(key)) {
|
|
1912
|
+
continue;
|
|
1913
|
+
}
|
|
1914
|
+
seen.add(key);
|
|
1915
|
+
pending.push({ key, ref: `/orgs/${encodeURIComponent(owner)}/sources/${encodeURIComponent(source)}/` });
|
|
1916
|
+
}
|
|
1917
|
+
|
|
1918
|
+
if (pending.length === 0) {
|
|
1919
|
+
return;
|
|
1920
|
+
}
|
|
1921
|
+
|
|
1922
|
+
let results;
|
|
1923
|
+
try {
|
|
1924
|
+
results = await this.referenceResolver.resolveReferences(pending.map(p => p.ref));
|
|
1925
|
+
} catch (error) {
|
|
1926
|
+
oclVsLog.warn(`$resolveReference batch failed: ${error.message}`);
|
|
1927
|
+
return;
|
|
1928
|
+
}
|
|
1929
|
+
|
|
1930
|
+
// null => resolver disabled or unavailable mid-flight; leave the cache untouched
|
|
1931
|
+
// so the per-source GET fallback runs.
|
|
1932
|
+
if (!Array.isArray(results)) {
|
|
1933
|
+
return;
|
|
1934
|
+
}
|
|
1935
|
+
|
|
1936
|
+
let resolvedCount = 0;
|
|
1937
|
+
for (let i = 0; i < pending.length; i++) {
|
|
1938
|
+
const canonical = results[i]?.resolved ? results[i].canonical : null;
|
|
1939
|
+
if (canonical) {
|
|
1940
|
+
this.sourceCanonicalCache.set(pending[i].key, canonical);
|
|
1941
|
+
resolvedCount++;
|
|
1942
|
+
}
|
|
1943
|
+
}
|
|
1944
|
+
if (resolvedCount > 0) {
|
|
1945
|
+
oclVsLog.info(`$resolveReference batch resolved ${resolvedCount}/${pending.length} source canonical(s) in one request`);
|
|
1946
|
+
}
|
|
1947
|
+
}
|
|
1948
|
+
|
|
1786
1949
|
#matches(json, params) {
|
|
1787
1950
|
for (const [name, value] of Object.entries(params)) {
|
|
1788
1951
|
if (!value) {
|