fhirsmith 0.16.1 → 0.17.0

package/tx/ocl/README.md

@@ -39,14 +39,70 @@ In FHIRsmith terms, these providers are loaded by `tx/library.js` when a source
 ## Runtime flow
 ### Metadata discovery
 CodeSystems (`cs-ocl.js`):
-- discover orgs via `/orgs/`
-- for each org, discover sources via `/orgs/{org}/sources/`
-- fallback to `/sources/` if org listing is unavailable
+- prefer one paginated crawl of the global `/sources/` listing (filtered to
+  organization-owned entries)
+- fallback: discover orgs via `/orgs/`, then `/orgs/{org}/sources/` per org
 
 ValueSets (`vs-ocl.js`):
-- discover orgs via `/orgs/`
-- discover collections via `/orgs/{org}/collections/`
-- fallback to `/collections/`
+- prefer one paginated crawl of the global `/collections/` listing (filtered to
+  organization-owned entries)
+- fallback: discover orgs via `/orgs/`, then `/orgs/{org}/collections/` per org
+
+### Visibility policy (org-only + public-only)
+An artifact is expected to live in an **organization** to be visible through the
+terminology service. User-owned repos (`/users/{user}/...`) are experimental by
+convention and are excluded from both discovery and `$resolveReference` results
+(a canonical resolving to a user-owned repo is treated as unresolved).
+
+In addition, a repo is only served when it is **publicly viewable**
+(`public_access` is `View`/`Edit`; `None`/unknown is treated as not servable,
+fail-closed). Because the configured token can see every repo its OCL account has
+access to — and both `$resolveReference` and discovery answer anonymous terminology
+requests — without this gate a *private* repo in the token's org could be surfaced
+publicly. This gate is applied in **both** paths:
+
+- **`$resolveReference`**: a resolved repo that is not public is treated as unresolved.
+- **Discovery**: the global (`/sources/`, `/collections/`) and per-org listing crawls
+  drop non-public (and non-org-owned) repos before they are registered.
+
+`public_access` is not returned by the listing endpoints or by `$resolveReference`
+today, so a shared `PublicAccessChecker` fetches each repo once to read it, caching
+the verdict by `repoUrl` (bounded LRU) and running batch checks with a concurrency
+limit. A check that *errors* is not cached, so a transient blip is retried rather
+than pinning a repo as private. The resolver also rejects any `repoUrl` that is not
+a safe same-host relative path, so an unexpected response can never redirect an
+authenticated request off-host.
+
+> **Operational guidance (defense-in-depth):** configure the `token=` account with
+> the **least privilege** needed — ideally a service account that can see only
+> public repos. The `public_access` gate enforces public-only serving in code
+> regardless, but a least-privilege token means a misconfiguration can never
+> expose private content.
+
+### Canonical resolution via `$resolveReference`
+With a `token=` configured on the `ocl:` source line, the providers resolve
+"which repo holds this canonical URL?" through OCL's
+[`$resolveReference`](https://docs.openconceptlab.org/en/latest/oclapi/apireference/resolveReference.html)
+(global namespace) instead of iterating listings and matching `canonical_url`
+client-side. Used for ConceptMap source-system search, ValueSet lookup by
+canonical, and batched resolution of a collection's compose source canonicals.
+
+Without a token nothing changes: `$resolveReference` is authenticated on every
+instance probed (while the listing endpoints are public), so the resolver is
+constructed disabled and every caller keeps its previous search path. A `404`
+(endpoint not implemented) disables it for the process; a transient `401`/`403`
+backs it off for a cooldown and then retries. The resolution cache is a bounded
+LRU and over-long references are never sent to OCL.
+
+### CodeSystem default versions (release vs HEAD)
+OCL's own resolution treats a source's **latest release** as its default version
+(HEAD only when nothing is released), but discovery listings only ever report
+HEAD. With a token, discovery batch-resolves every canonical through
+`$resolveReference` and registers **both** versions: the release becomes what a
+versionless request gets (matching OCL), and HEAD stays reachable via an
+explicit `version=HEAD`. Only canonicals that are new or changed are re-resolved
+on refresh, so a steady-state cycle costs no extra requests. Without a token,
+discovery stays HEAD-only as before.
 
 ConceptMaps (`cm-ocl.js`):
 - fetch by id via `/mappings/{id}/`

package/tx/ocl/cm-ocl.cjs

@@ -4,9 +4,17 @@ const { PAGE_SIZE } = require('./shared/constants');
 const { createOclHttpClient } = require('./http/client');
 const { fetchAllPages, extractItemsAndNext } = require('./http/pagination');
 const { stripIdPrefix } = require('../library/resource-ids');
+const { OclReferenceResolver, isOclRepoPath } = require('./resolve/reference-resolver');
+const Logger = require('../../library/logger');
+const oclCmLog = Logger.getInstance().child({ module: 'ocl-cm' });
 
 const DEFAULT_MAX_SEARCH_PAGES = 10;
 
+// OCL map types that relate concepts without mapping one to the other, so they must
+// not surface as $translate matches: CIEL uses Q-AND-A (question -> answer) and
+// CONCEPT-SET (set -> member). Unknown types still map to related-to.
+const NON_TRANSLATION_MAP_TYPES = new Set(['Q-AND-A', 'CONCEPT-SET']);
+
 class OCLConceptMapProvider extends AbstractConceptMapProvider {
   constructor(config = {}) {
     super();
@@ -23,6 +31,15 @@ class OCLConceptMapProvider extends AbstractConceptMapProvider {
     this._sourceCandidatesCache = new Map();
     this._sourceUrlsByCanonical = new Map();
     this._canonicalBySourceUrl = new Map();
+
+    // Asks OCL which repo holds a canonical instead of guessing via text search.
+    // Disabled unless a token is configured, in which case #candidateSourceUrls
+    // keeps using its existing search.
+    this.referenceResolver = new OclReferenceResolver({
+      httpClient: this.httpClient,
+      token: options.token || null,
+      logger: oclCmLog
+    });
   }
 
   prefixIds() {
@@ -114,8 +131,12 @@ class OCLConceptMapProvider extends AbstractConceptMapProvider {
   async searchConceptMaps(searchParams, _elements) {
     this._validateSearchParams(searchParams);
 
+    // Keep the canonical exactly as supplied. Lower-casing it was harmless while
+    // every consumer compared through #norm() (which lower-cases anyway), but
+    // $resolveReference matches the canonical exactly, and OCL has no
+    // .../fhir/codesystem/x -- only .../fhir/CodeSystem/X.
     const params = Object.fromEntries(
-      searchParams.map(({ name, value }) => [name, String(value).toLowerCase()])
+      searchParams.map(({ name, value }) => [name, String(value)])
     );
     const sourceSystem = params['source-system'] || params.source || null;
     const targetSystem = params['target-system'] || params.target || null;
@@ -138,7 +159,7 @@ class OCLConceptMapProvider extends AbstractConceptMapProvider {
   async #collectMappingsForSearch(sourceSystem, targetSystem) {
     const systemUrl = sourceSystem || targetSystem;
     const candidates = await this.#candidateSourceUrls(systemUrl);
-    const sourcePaths = candidates.filter(s => String(s || '').startsWith('/orgs/'));
+    const sourcePaths = candidates.filter(isOclRepoPath);
 
     if (sourcePaths.length === 0) {
       return [];
@@ -147,30 +168,20 @@ class OCLConceptMapProvider extends AbstractConceptMapProvider {
     const allMappings = [];
     for (const sourcePath of sourcePaths) {
       const normalizedPath = this.#normalizeSourcePath(sourcePath);
-      let concepts;
       try {
-        concepts = await this.#fetchAllPages(
-          `${normalizedPath}concepts/`, { limit: PAGE_SIZE }, this.maxSearchPages
+        // Ask the source for its mappings directly. Walking concepts and asking
+        // each one costs a request per concept and, worse, only ever sees the
+        // first maxSearchPages of them -- for LOINC that is 1000 of 184683
+        // concepts (0.5%), so any mapping past that point was silently invisible.
+        // Verified live: both paths return the identical mapping set.
+        const mappings = await this.#fetchAllPages(
+          `${normalizedPath}mappings/`, { limit: PAGE_SIZE }, this.maxSearchPages
         );
+        allMappings.push(...mappings);
       } catch (_err) {
+        // source exposes no mappings endpoint or is inaccessible — skip
         continue;
       }
-
-      for (const concept of concepts) {
-        const code = concept.id || concept.mnemonic;
-        if (!code) {
-          continue;
-        }
-        try {
-          const mappings = await this.#fetchAllPages(
-            `${normalizedPath}concepts/${encodeURIComponent(code)}/mappings/`,
-            { limit: PAGE_SIZE }, 2
-          );
-          allMappings.push(...mappings);
-        } catch (_err) {
-          // concept has no mappings or endpoint inaccessible — skip
-        }
-      }
     }
 
     const sourceUrlsToResolve = new Set();
@@ -194,7 +205,7 @@ class OCLConceptMapProvider extends AbstractConceptMapProvider {
       const sourceCode = mapping.from_concept_code || mapping.fromConceptCode;
       const targetCode = mapping.to_concept_code || mapping.toConceptCode;
 
-      if (!fromSource || !toSource || !sourceCode || !targetCode) {
+      if (!fromSource || !toSource || !sourceCode || !targetCode || !this.#isTranslation(mapping)) {
         continue;
       }
 
@@ -252,8 +263,8 @@ class OCLConceptMapProvider extends AbstractConceptMapProvider {
         name: id,
         title: `${sourceId} to ${targetId}`,
         status: 'active',
-        sourceScopeUri: group.sourceCanonical,
-        targetScopeUri: group.targetCanonical,
+        // No sourceScope/targetScope: a scope is a ValueSet, and $translate resolves it and
+        // checks the code against it. The systems are carried by the group instead.
         group: [{
           source: group.sourceCanonical,
           target: group.targetCanonical,
@@ -290,7 +301,7 @@ class OCLConceptMapProvider extends AbstractConceptMapProvider {
     const targetCandidates = await this.#candidateSourceUrls(targetSystem);
 
     const mappings = [];
-    const sourcePaths = sourceCandidates.filter(s => String(s || '').startsWith('/orgs/'));
+    const sourcePaths = sourceCandidates.filter(isOclRepoPath);
 
     if (sourceCode && sourcePaths.length > 0) {
       for (const sourcePath of sourcePaths) {
@@ -410,7 +421,7 @@ class OCLConceptMapProvider extends AbstractConceptMapProvider {
     const sourceCode = mapping.from_concept_code || mapping.fromConceptCode;
     const targetCode = mapping.to_concept_code || mapping.toConceptCode;
 
-    if (!source || !target || !sourceCode || !targetCode) {
+    if (!source || !target || !sourceCode || !targetCode || !this.#isTranslation(mapping)) {
       return null;
     }
 
@@ -430,8 +441,10 @@ class OCLConceptMapProvider extends AbstractConceptMapProvider {
       name: `mapping-${id}`,
       title: mapping.name || `Mapping ${id}`,
       status: 'active',
-      sourceScopeUri: mapping.from_collection_url || mapping.fromCollectionUrl || source,
-      targetScopeUri: mapping.to_collection_url || mapping.toCollectionUrl || target,
+      // No sourceScope/targetScope: a scope is a ValueSet canonical, which $translate resolves
+      // and checks the code against. OCL only offers repo paths here (from_source_url or
+      // from_collection_url), which resolve to no ValueSet and made every translation fail
+      // with not-found. The systems are carried by the group instead.
       group: [
         {
           source: sourceCanonical,
@@ -461,21 +474,32 @@ class OCLConceptMapProvider extends AbstractConceptMapProvider {
     return new ConceptMap(json, 'R5');
   }
 
+  // OCL map_type expresses "from concept <relation> to concept", which lines up
+  // with the R5 ConceptMapRelationship codes (source <relation> target).
   #toRelationship(mapType) {
-    switch ((mapType || '').toUpperCase()) {
+    switch (this.#normalizeMapType(mapType)) {
       case 'SAME-AS':
         return 'equivalent';
       case 'NARROWER-THAN':
-        return 'narrower-than';
+        return 'source-is-narrower-than-target';
       case 'BROADER-THAN':
-        return 'broader-than';
+        return 'source-is-broader-than-target';
       case 'NOT-EQUIVALENT':
+      case 'NOT-SAME-AS':
         return 'not-related-to';
       default:
         return 'related-to';
     }
   }
 
+  #isTranslation(mapping) {
+    return !NON_TRANSLATION_MAP_TYPES.has(this.#normalizeMapType(mapping.map_type || mapping.mapType));
+  }
+
+  #normalizeMapType(mapType) {
+    return String(mapType || '').trim().toUpperCase().replace(/[\s_]+/g, '-');
+  }
+
   async #searchMappings(params = {}, maxPages = this.maxSearchPages) {
     const endpoint = this.org ? `/orgs/${encodeURIComponent(this.org)}/mappings/` : '/mappings/';
     return await this.#fetchAllPages(endpoint, params, maxPages);
@@ -523,9 +547,16 @@ class OCLConceptMapProvider extends AbstractConceptMapProvider {
       }
     }
 
-    const discovered = await this.#resolveSourceCandidatesFromOcl(systemUrl);
-    for (const item of discovered) {
-      result.add(item);
+    // Ask OCL which repo holds this canonical. An authoritative answer makes the
+    // heuristic text search below unnecessary; without one we fall back to it.
+    const resolved = await this.#resolveViaReferenceResolver(systemUrl);
+    if (resolved) {
+      result.add(resolved);
+    } else {
+      const discovered = await this.#resolveSourceCandidatesFromOcl(systemUrl);
+      for (const item of discovered) {
+        result.add(item);
+      }
     }
 
     const out = Array.from(result);
@@ -533,6 +564,56 @@ class OCLConceptMapProvider extends AbstractConceptMapProvider {
     return out;
   }
 
+  /**
+   * Resolve a canonical to its OCL repo via $resolveReference.
+   * Returns null when the resolver is disabled (no token), the endpoint is
+   * unavailable, or OCL cannot resolve the canonical -- in every case the caller
+   * falls back to the existing search.
+   */
+  async #resolveViaReferenceResolver(systemUrl) {
+    // Say why once, rather than per lookup: for a tokenless deployment this is the
+    // expected steady state, not an error.
+    if (!this.referenceResolver.isEnabled()) {
+      if (!this._resolverDisabledLogged) {
+        this._resolverDisabledLogged = true;
+        oclCmLog.info(
+          `$resolveReference not in use (${this.referenceResolver.disabledReason}); using source search`
+        );
+      }
+      return null;
+    }
+
+    let resolved;
+    try {
+      resolved = await this.referenceResolver.resolve(systemUrl);
+    } catch (error) {
+      oclCmLog.warn(`$resolveReference lookup failed for ${systemUrl}: ${error.message}`);
+      return null;
+    }
+
+    if (!resolved?.resolved || !resolved.repoUrl) {
+      oclCmLog.info(`$resolveReference did not resolve ${systemUrl}; falling back to source search`);
+      return null;
+    }
+
+    // Logged on success too: resolver and search return the same thing, so without
+    // this there is no way -- from outside or from the logs -- to tell which path ran.
+    oclCmLog.info(`$resolveReference resolved ${systemUrl} -> ${resolved.repoUrl}`);
+
+    // Keep the canonical<->repo caches coherent with the search path's bookkeeping.
+    // Index under what was asked, but record OCL's own canonical_url as the repo's
+    // canonical: echoing the request back would store whatever spelling the caller
+    // used rather than the repo's actual identity.
+    const canonicalKey = this.#norm(systemUrl);
+    if (!this._sourceUrlsByCanonical.has(canonicalKey)) {
+      this._sourceUrlsByCanonical.set(canonicalKey, new Set());
+    }
+    this._sourceUrlsByCanonical.get(canonicalKey).add(resolved.repoUrl);
+    this._canonicalBySourceUrl.set(this.#norm(resolved.repoUrl), resolved.canonical || systemUrl);
+
+    return resolved.repoUrl;
+  }
+
   async #resolveSourceCandidatesFromOcl(systemUrl) {
     const endpoint = this.org ? `/orgs/${encodeURIComponent(this.org)}/sources/` : '/sources/';
     const query = this.#queryTokenFromSystem(systemUrl);
@@ -573,17 +654,52 @@ class OCLConceptMapProvider extends AbstractConceptMapProvider {
   }
 
   async #ensureCanonicalForSourceUrls(sourceUrls) {
+    // Deduplicate the repo paths that still need a canonical.
+    const pending = [];
+    const seen = new Set();
     for (const sourceUrl of sourceUrls || []) {
       const sourceKey = this.#norm(sourceUrl);
-      if (!sourceKey || this._canonicalBySourceUrl.has(sourceKey)) {
+      if (!sourceKey || this._canonicalBySourceUrl.has(sourceKey) || seen.has(sourceKey)) {
         continue;
       }
-
       const sourcePath = String(sourceUrl || '').trim();
-      if (!sourcePath.startsWith('/orgs/')) {
+      if (!isOclRepoPath(sourcePath)) {
         continue;
       }
+      seen.add(sourceKey);
+      pending.push(sourcePath);
+    }
+
+    if (pending.length === 0) {
+      return;
+    }
+
+    // One $resolveReference batch instead of one GET per source: the result
+    // carries the repo's canonical_url. Anything the batch cannot resolve falls
+    // through to the per-source GETs below, so behaviour without a token (or on
+    // failure) is unchanged.
+    if (this.referenceResolver.isEnabled()) {
+      try {
+        const results = await this.referenceResolver.resolveReferences(pending);
+        if (Array.isArray(results)) {
+          for (let i = 0; i < pending.length; i++) {
+            const r = results[i];
+            if (!r?.resolved || !r.canonical) {
+              continue;
+            }
+            const resolvedSourceUrl = r.repoUrl || pending[i];
+            this.#recordCanonicalForSource(pending[i], resolvedSourceUrl, r.canonical);
+          }
+        }
+      } catch (error) {
+        oclCmLog.warn(`$resolveReference batch for source canonicals failed: ${error.message}`);
+      }
+    }
 
+    for (const sourcePath of pending) {
+      if (this._canonicalBySourceUrl.has(this.#norm(sourcePath))) {
+        continue;
+      }
       try {
         const response = await this.httpClient.get(sourcePath);
         const source = response.data || {};
@@ -592,13 +708,7 @@ class OCLConceptMapProvider extends AbstractConceptMapProvider {
         if (!canonical) {
           continue;
         }
-
-        const canonicalKey = this.#norm(canonical);
-        if (!this._sourceUrlsByCanonical.has(canonicalKey)) {
-          this._sourceUrlsByCanonical.set(canonicalKey, new Set());
-        }
-        this._sourceUrlsByCanonical.get(canonicalKey).add(resolvedSourceUrl);
-        this._canonicalBySourceUrl.set(this.#norm(resolvedSourceUrl), canonical);
+        this.#recordCanonicalForSource(sourcePath, resolvedSourceUrl, canonical);
       } catch (e) {
         // Ignore source lookup failures and continue resolving remaining sources.
         continue;
@@ -606,6 +716,18 @@ class OCLConceptMapProvider extends AbstractConceptMapProvider {
     }
   }
 
+  #recordCanonicalForSource(requestedPath, resolvedSourceUrl, canonical) {
+    const canonicalKey = this.#norm(canonical);
+    if (!this._sourceUrlsByCanonical.has(canonicalKey)) {
+      this._sourceUrlsByCanonical.set(canonicalKey, new Set());
+    }
+    this._sourceUrlsByCanonical.get(canonicalKey).add(resolvedSourceUrl);
+    this._canonicalBySourceUrl.set(this.#norm(resolvedSourceUrl), canonical);
+    // Also index under the exact path we were asked about, so the "already
+    // resolved" checks hit even when OCL's repo url differs in shape.
+    this._canonicalBySourceUrl.set(this.#norm(requestedPath), canonical);
+  }
+
   #queryTokenFromSystem(systemUrl) {
     const raw = String(systemUrl || '').trim().replace(/\/+$/, '');
     if (!raw) {