fhirsmith 0.6.0 → 0.7.0

package/tx/cs/cs-provider-api.js

@@ -20,7 +20,13 @@ class AbstractCodeSystemProvider {
   }
 
   /**
-  * Returns the list of CodeSystems this provider provides
+   * Returns the list of CodeSystems this provider provides. This is called once at start up.
+   * The code systems should be fully loaded; lazy loading code systems is not considered good
+   * for engineering.
+   *
+   *
+   * Note that unlike value sets, which are accessed from the provider on the fly, code systems
+   * are all preloaded into the kernel (e.g. provider) at start up
    *
   * @param {string} fhirVersion - The FHIRVersion in scope - if relevant (there's always a stated version, though R5 is always used)
   * @param {string} context - The client's stated context - if provided.
@@ -32,6 +38,24 @@ class AbstractCodeSystemProvider {
     throw new Error('listCodeSystems must be implemented by AbstractCodeSystemProvider subclass');
   }
 
+  /**
+   * This is called once a minute to update the code system list that the provider maintains.
+   *
+   * return an object that has three Map<String, CodeSystem>: {added, changed, deleted}
+   *
+   * these use the same key as the
+   *
+   * code systems are identified by url and version
+   *
+   * @param fhirVersion
+   * @param context
+   * @returns {Promise<null>}
+   */
+  // eslint-disable-next-line no-unused-vars
+  async getCodeSystemChanges(fhirVersion, context){
+    return null;
+  }
+
   async close() {
 
   }

package/tx/cs/cs-provider-list.js

@@ -7,7 +7,7 @@ class ListCodeSystemProvider extends AbstractCodeSystemProvider {
   /**
    * {Map<String, CodeSystem>} A list of code system factories that contains all the preloaded native code systems
    */
-  codeSystems = new Map();
+  codeSystems = [];
 
   /**
    * ensure that the ids on the code systems are unique, if they are
@@ -17,7 +17,7 @@ class ListCodeSystemProvider extends AbstractCodeSystemProvider {
    */
   // eslint-disable-next-line no-unused-vars
   assignIds(ids) {
-    for (const cs of this.codeSystems.values()) {
+    for (const cs of this.codeSystems) {
       if (!cs.id || ids.has("CodeSystem/"+cs.id)) {
         cs.id = ""+ids.size;
       }

package/tx/library/canonical-resource.js

@@ -111,7 +111,12 @@ class CanonicalResource {
       const fmt = this.versionAlgorithm() || other.versionAlgorithm() || this.guessVersionAlgorithmFromVersion(this.version);
       switch (fmt) {
         case 'semver':
-          return VersionUtilities.isThisOrLater(other.version, this.version, VersionPrecision.PATCH);
+          try {
+            return VersionUtilities.isThisOrLater(other.version, this.version, VersionPrecision.PATCH);
+          } catch (error) {
+            // other is not semver. Not much we can do
+            return false;
+          }
         case 'date':
           return this.dateIsMoreRecent(this.version, other.version);
         case 'integer':

package/tx/library.js

@@ -31,6 +31,9 @@ const { Provider } = require("./provider");
 const {I18nSupport} = require("../library/i18nsupport");
 const folders = require('../library/folder-setup');
 const {VSACValueSetProvider} = require("./vs/vs-vsac");
+const { OCLCodeSystemProvider, OCLSourceCodeSystemFactory } = require('./ocl/cs-ocl');
+const { OCLValueSetProvider } = require('./ocl/vs-ocl');
+const { OCLConceptMapProvider } = require('./ocl/cm-ocl');
 
 /**
  * This class holds all the loaded content ready for processing
@@ -95,6 +98,8 @@ class Library {
     this.codeSystemProviders = [];
     this.valueSetProviders = [];
     this.conceptMapProviders = [];
+    this.oclProviderSets = new Map();
+    this.oclConfig = {};
 
     // Create package manager for FHIR packages
     const packageServers = ['https://packages2.fhir.org/packages'];
@@ -154,6 +159,7 @@ class Library {
     const yamlContent = await fs.readFile(yamlPath, 'utf8');
     const config = yaml.parse(yamlContent);
     this.baseUrl = config.base.url;
+    this.oclConfig = config.ocl && typeof config.ocl === 'object' ? config.ocl : {};
 
     this.log.info('Fetching Data from '+this.baseUrl);
 
@@ -277,12 +283,127 @@ class Library {
       case 'url/cs':
         await this.loadUrl(packageManager, details, isDefault, mode, true);
         break;
+
+      case 'ocl':
+        await this.loadOcl(details, isDefault, mode);
+        break;
         
       default:
         throw new Error(`Unknown source type: ${type}`);
     }
   }
 
+  parseOclConfig(details) {
+    const text = String(details || '').trim();
+    if (!text) {
+      throw new Error('OCL source requires details, e.g. ocl:https://ocl.example.org');
+    }
+
+    const parts = text.split('|').map(p => p.trim()).filter(Boolean);
+    const baseUrl = this.resolveOclConfigValue(parts[0]);
+    if (!baseUrl) {
+      throw new Error('OCL source requires a base URL');
+    }
+
+    const config = { baseUrl };
+    for (let i = 1; i < parts.length; i++) {
+      const part = parts[i];
+      const eq = part.indexOf('=');
+      if (eq === -1) {
+        continue;
+      }
+      const key = part.substring(0, eq).trim().toLowerCase();
+      const value = this.resolveOclConfigValue(part.substring(eq + 1).trim());
+      if (!value) {
+        continue;
+      }
+      if (key === 'org') {
+        config.org = value;
+      } else if (key === 'token') {
+        config.token = value;
+      } else if (key === 'timeout') {
+        const timeout = Number(value);
+        if (Number.isFinite(timeout) && timeout > 0) {
+          config.timeout = timeout;
+        }
+      }
+    }
+
+    return config;
+  }
+
+  resolveOclConfigValue(value) {
+    const text = String(value || '').trim();
+    if (!text) {
+      return text;
+    }
+
+    if (this.oclConfig && Object.hasOwn(this.oclConfig, text)) {
+      const resolved = this.oclConfig[text];
+      if (typeof resolved === 'string') {
+        return resolved.trim();
+      }
+    }
+
+    return text;
+  }
+
+  async loadOcl(details, isDefault, mode) {
+    const config = this.parseOclConfig(details);
+    const cacheKey = `${config.baseUrl}|${config.org || ''}`;
+
+    let providerSet = this.oclProviderSets.get(cacheKey);
+    if (!providerSet) {
+      const codeSystemProvider = new OCLCodeSystemProvider(config);
+      const valueSetProvider = new OCLValueSetProvider(config);
+      const conceptMapProvider = new OCLConceptMapProvider(config);
+      providerSet = {
+        config,
+        codeSystemProvider,
+        valueSetProvider,
+        conceptMapProvider,
+        csRegistered: false,
+        factoriesRegistered: false,
+        vsRegistered: false,
+        cmRegistered: false
+      };
+      this.oclProviderSets.set(cacheKey, providerSet);
+    }
+
+    if (mode === 'fetch') {
+      return;
+    }
+
+    if (mode === 'cs') {
+      if (!providerSet.csRegistered) {
+        this.codeSystemProviders.push(providerSet.codeSystemProvider);
+        providerSet.csRegistered = true;
+      }
+
+      if (!providerSet.factoriesRegistered) {
+        await providerSet.codeSystemProvider.listCodeSystems('5.0', null);
+        const metas = providerSet.codeSystemProvider.getSourceMetas();
+        for (const meta of metas) {
+          const factory = new OCLSourceCodeSystemFactory(this.i18n, providerSet.codeSystemProvider.httpClient, meta);
+          this.registerProvider(`ocl:${config.baseUrl}`, factory, isDefault);
+        }
+        providerSet.factoriesRegistered = true;
+      }
+      return;
+    }
+
+    if (mode === 'npm') {
+      if (!providerSet.vsRegistered) {
+        this.valueSetProviders.push(providerSet.valueSetProvider);
+        providerSet.vsRegistered = true;
+      }
+      if (!providerSet.cmRegistered) {
+        this.conceptMapProviders.push(providerSet.conceptMapProvider);
+        providerSet.cmRegistered = true;
+      }
+    }
+  }
+
   async loadInternal(details, isDefault, mode) {
     if (isDefault) {
       throw new Error("Default is not supported for internal code system providers");
@@ -464,13 +585,7 @@ class Library {
     for (const resource of resources) {
       const cs = new CodeSystem(await contentLoader.loadFile(resource, contentLoader.fhirVersion()));
       cs.sourcePackage = contentLoader.pid();
-      const existing = cp.codeSystems.get(cs.url);
-      if (!existing || cs.isMoreRecent(existing)) {
-        cp.codeSystems.set(cs.url, cs);
-      }
-      if (cs.version) {
-        cp.codeSystems.set(cs.vurl, cs);
-      }
+      cp.codeSystems.push(cs);
       csc++;
     }
     this.codeSystemProviders.push(cp);
@@ -686,10 +801,12 @@ class Library {
     }
 
 
+    provider.codeSystemProviders = this.codeSystemProviders;
+    provider.context = context;
     for (const cp of this.codeSystemProviders) {
-      const csMap = await cp.listCodeSystems(fhirVersion, context);
-      for (const [key, value] of csMap) {
-        provider.codeSystems.set(key, value);
+      const csList = await cp.listCodeSystems(fhirVersion, context);
+      for (const cs of csList) {
+        provider.addCodeSystem(cs);
       }
     }
     // Don't clone valueSetProviders yet - we'll build it with correct order

package/tx/ocl/README.md

@@ -0,0 +1,236 @@
+# OCL Integration in FHIRsmith
+
+## Overview
+The `tx/ocl` module integrates [Open Concept Lab (OCL)](https://openconceptlab.org/) as a terminology source inside FHIRsmith.
+
+It provides adapters/providers for:
+
+- `CodeSystem` from OCL Sources
+- `ValueSet` from OCL Collections
+- `ConceptMap` from OCL Mappings
+
+In FHIRsmith terms, these providers are loaded by `tx/library.js` when a source entry starts with `ocl:`. OCL metadata is discovered from OCL APIs, and heavy content (concept lists/expansions) is loaded lazily and warmed in background jobs.
+
+## Architecture
+### Main modules
+- `cs-ocl.js`
+  - `OCLCodeSystemProvider`: discovers OCL sources and publishes CodeSystem metadata.
+  - `OCLSourceCodeSystemFactory`: creates runtime CodeSystem providers with shared caches, cold-cache hydration, and background full-load jobs.
+  - `OCLSourceCodeSystemProvider`: runtime concept lookup/filter/search behavior for one source.
+- `vs-ocl.js`
+  - `OCLValueSetProvider`: discovers OCL collections, resolves compose includes, serves ValueSet metadata, and builds cached expansions in background.
+- `cm-ocl.js`
+  - `OCLConceptMapProvider`: resolves OCL mappings for fetch/search/translation candidate discovery.
+
+### Supporting modules under `tx/ocl`
+- `http/client.js`: Axios client creation, base URL normalization, token auth header (`Token` or `Bearer`).
+- `http/pagination.js`: OCL pagination helper (`results/items/data`, `next`, page mode fallback).
+- `cache/cache-paths.js`: cold-cache directories and canonical URL to file path mapping.
+- `cache/cache-utils.js`: cache directory creation, file age detection, friendly age formatting.
+- `fingerprint/fingerprint.js`: deterministic SHA-256 fingerprints for CodeSystem concepts and ValueSet expansions.
+- `jobs/background-queue.js`: singleton keyed queue, size-priority ordering, max concurrency = 2, heartbeat logging every 30s.
+- `mappers/concept-mapper.js`: maps OCL concept payloads to internal concept context shape.
+- `model/concept-filter-context.js`: ranked filter result set used by CodeSystem filters.
+- `shared/constants.js`: defaults and constants (`PAGE_SIZE`, cache freshness window, etc.).
+- `shared/patches.js`:
+  - patches search worker so `CodeSystem?url=...&code=...` on OCL resources only returns matching concept subtree.
+  - patches `TxParameters.hashSource()` to include `filter` for expansion cache key differentiation.
+
+## Runtime flow
+### Metadata discovery
+CodeSystems (`cs-ocl.js`):
+- discover orgs via `/orgs/`
+- for each org, discover sources via `/orgs/{org}/sources/`
+- fallback to `/sources/` if org listing is unavailable
+
+ValueSets (`vs-ocl.js`):
+- discover orgs via `/orgs/`
+- discover collections via `/orgs/{org}/collections/`
+- fallback to `/collections/`
+
+ConceptMaps (`cm-ocl.js`):
+- fetch by id via `/mappings/{id}/`
+- search via `/mappings/` or `/orgs/{org}/mappings/`
+
+### Lazy loading
+- CodeSystem concepts are not fully loaded at metadata discovery time.
+- ValueSet expansions are not built inline by default.
+- Missing concept/page/expansion data triggers background warm-up scheduling.
+
+### Cold cache (disk)
+- Base folder: `data/terminology-cache/ocl`
+- CodeSystems: `data/terminology-cache/ocl/codesystems`
+- ValueSets: `data/terminology-cache/ocl/valuesets`
+
+On startup/initialization:
+- CodeSystem factory hydrates concept cache from cold cache file if present.
+- ValueSet provider loads cached expansions from cold cache files.
+
+Corrupt cache handling:
+- JSON parse/read errors are logged and skipped; provider continues without crashing.
+
+### Hot cache (memory)
+- CodeSystems: shared in-memory concept/page caches per factory.
+- ValueSets: in-memory expansion cache keyed by ValueSet + params hash.
+- ConceptMaps: in-memory map keyed by URL/version/id.
+
+### Background warm-up jobs
+Queue behavior (`OCLBackgroundJobQueue`):
+- singleton job key (skip duplicate key)
+- max concurrency = `2`
+- ordering by `jobSize` (smaller concept count first)
+- heartbeat log every `30s`
+- progress supports `{ processed, total }` and `%`
+
+CodeSystem warm-up:
+- skipped when cold-cache file age is `<= 1 hour`
+- enqueued when stale/no cold cache
+- loads all concept pages
+- computes fingerprint from full concept content
+- if fingerprint changed, replaces cold cache file
+
+ValueSet warm-up:
+- skipped when freshest cold cache age (file mtime or cached timestamp) is `<= 1 hour`
+- enqueued when stale/no cold cache
+- builds expansion by paging collection concepts
+- computes expansion fingerprint
+- if fingerprint changed, replaces cold cache file
+
+### Fingerprint/checksum strategy
+- OCL source checksum is treated as informational only in `cs-ocl.js` comments.
+- Cache replacement decisions use custom fingerprints from concept/expansion content.
+- ValueSet cache validity also checks metadata signature and dependency checksums from referenced CodeSystems.
+
+### ValueSet expansion filtering
+`vs-ocl.js` supports filtered concept retrieval through `valueSet.oclFetchConcepts(...)`:
+- local baseline filtering (code/display/definition text contains)
+- `SearchFilterText` token behavior
+- remote query hint (`q`) generated from normalized filter tokens
+- dedicated smaller page size for filtered calls (`FILTERED_CONCEPT_PAGE_SIZE`)
+
+### `/CodeSystem?url=...&code=...` behavior for OCL
+`shared/patches.js` patches search worker to apply concept subtree filtering only for OCL-marked CodeSystems (extension URL `http://fhir.org/FHIRsmith/StructureDefinition/ocl-codesystem`).
+
+## Configuration
+## Activation in FHIRsmith
+OCL is activated by adding an OCL source line in the TX library YAML (for example `data/library.yml`):
+
+```yaml
+sources:
+  - ocl:https://oclapi2.ips.hsl.org.br
+```
+
+`tx/library.js` loads this via `loadOcl()`.
+
+### Source syntax
+`tx/library.js` parses:
+
+```text
+ocl:<baseUrl>|org=<orgId>|token=<tokenOrAlias>|timeout=<ms>
+```
+
+Parsed keys:
+- `baseUrl` (required)
+- `org` (optional)
+- `token` (optional)
+- `timeout` (optional positive number)
+
+Examples:
+
+```yaml
+sources:
+  - ocl:https://api.openconceptlab.org
+  - ocl:https://ocl.example.org|org=my-org
+  - ocl:https://ocl.example.org|org=my-org|token=my-ocl-token|timeout=45000
+```
+
+### Config value aliasing
+`Library.resolveOclConfigValue()` can resolve symbolic values from top-level YAML `ocl:` object loaded into `this.oclConfig`.
+
+Example pattern:
+
+```yaml
+ocl:
+  ocl-base: https://ocl.example.org
+  ocl-token: Token abc123
+
+sources:
+  - ocl:ocl-base|org=my-org|token=ocl-token
+```
+
+### Credentials and URLs
+- Base URL is required (OCL API root).
+- Token is optional, sent as `Authorization`:
+  - `Token <value>` if no prefix is provided
+  - preserved if already `Token ...` or `Bearer ...`
+
+### Enable/disable integration
+- Enable by including at least one `ocl:` source entry in TX library YAML.
+- Disable by removing/commenting those `ocl:` entries.
+- `modules.tx.enabled` must be true in server config to expose TX endpoints.
+
+## Cache behavior details
+### Startup hydration
+- CodeSystem cold cache is loaded when factory is created (`OCLSourceCodeSystemFactory` constructor path).
+- ValueSet cold cache is loaded during `OCLValueSetProvider.initialize()`.
+
+### 1-hour freshness rule
+- Fresh threshold constant: `COLD_CACHE_FRESHNESS_MS = 60 * 60 * 1000`.
+- If cold cache age is `<= 1 hour`, warm-up scheduling is skipped.
+
+### Hot cache replacing cold cache
+- On successful background refresh, new fingerprint is compared to previous cold-cache fingerprint.
+- If changed, cold cache is overwritten with the refreshed in-memory state.
+
+## Operational notes
+### Logs to expect
+Prefixes:
+- `[OCL]` for CodeSystem/queue/general OCL flow
+- `[OCL-ValueSet]` for ValueSet flow
+
+Typical events:
+- fetched org/source/collection counts
+- cold cache loaded/saved
+- warm-up skipped/enqueued/started/completed
+- fingerprint unchanged/changed
+- queue status + heartbeat snapshots
+
+### Troubleshooting hints
+- If nothing loads, verify OCL base URL and org visibility.
+- If cache never warms, check cold-cache timestamps and 1-hour rule.
+- If searches by `code` behave unexpectedly, confirm OCL marker extension is present on CodeSystem resources.
+- For ValueSet filter behavior, verify `filter` is included in request path through `TxParameters.hashSource` patch and that filter text normalizes as expected.
+
+### Known limitations (from current implementation)
+- OCL checksums are not relied on for cache invalidation.
+- Some source/collection discovery paths depend on OCL endpoint support and can fallback.
+- Missing endpoints returning `404` in some concept fetch paths are treated as empty content for graceful degradation.
+
+## Developer notes
+### Where to extend
+- Add/adjust OCL HTTP behavior: `tx/ocl/http/*`
+- Add cache policy/path behavior: `tx/ocl/cache/*`
+- Add queue policy: `tx/ocl/jobs/background-queue.js`
+- Add mapping logic from OCL payloads: `tx/ocl/mappers/*`
+- Add fingerprint strategy: `tx/ocl/fingerprint/*`
+- Add provider-specific behavior:
+  - CodeSystem: `tx/ocl/cs-ocl.js`
+  - ValueSet: `tx/ocl/vs-ocl.js`
+  - ConceptMap: `tx/ocl/cm-ocl.js`
+
+### Ownership by concern
+- HTTP access: `http/client.js`, `http/pagination.js`
+- Disk cold cache and pathing: `cache/cache-paths.js`, `cache/cache-utils.js`
+- Background jobs and scheduling policy: `jobs/background-queue.js`
+- Fingerprints/checksums: `fingerprint/fingerprint.js` and provider compare logic
+- Worker/runtime patches: `shared/patches.js`
+
+### Test strategy for future changes
+- Keep provider tests deterministic by mocking OCL HTTP responses and cold-cache files.
+- Validate stale/fresh transitions with file mtime control.
+- Validate queue behavior with isolated static state reset between tests.
+- Track coverage with:
+
+```bash
+npm test -- --runInBand tests/ocl --coverage --collectCoverageFrom="tx/ocl/**/*.js"
+```

package/tx/ocl/cache/cache-paths.cjs

@@ -0,0 +1,32 @@
+const path = require('path');
+
+const CACHE_BASE_DIR = path.join(process.cwd(), 'data', 'terminology-cache', 'ocl');
+const CACHE_CS_DIR = path.join(CACHE_BASE_DIR, 'codesystems');
+const CACHE_VS_DIR = path.join(CACHE_BASE_DIR, 'valuesets');
+
+function sanitizeFilename(text) {
+  if (!text || typeof text !== 'string') {
+    return 'unknown';
+  }
+  return text
+    .replace(/[^a-zA-Z0-9._-]/g, '_')
+    .replace(/_+/g, '_')
+    .substring(0, 200);
+}
+
+function getCacheFilePath(baseDir, canonicalUrl, version = null, paramsKey = null) {
+  const filename = sanitizeFilename(canonicalUrl)
+    + (version ? `_${sanitizeFilename(version)}` : '')
+    + (paramsKey && paramsKey !== 'default' ? `_p_${sanitizeFilename(paramsKey)}` : '')
+    + '.json';
+
+  return path.join(baseDir, filename);
+}
+
+module.exports = {
+  CACHE_BASE_DIR,
+  CACHE_CS_DIR,
+  CACHE_VS_DIR,
+  sanitizeFilename,
+  getCacheFilePath
+};

package/tx/ocl/cache/cache-paths.js

@@ -0,0 +1,2 @@
+module.exports = require('./cache-paths.cjs');
+

package/tx/ocl/cache/cache-utils.cjs

@@ -0,0 +1,43 @@
+const fs = require('fs/promises');
+const fsSync = require('fs');
+
+async function ensureCacheDirectories(...dirs) {
+  for (const dir of dirs) {
+    if (!dir) {
+      continue;
+    }
+
+    try {
+      await fs.mkdir(dir, { recursive: true });
+    } catch (error) {
+      console.error('[OCL] Failed to create cache directory:', dir, error.message);
+    }
+  }
+}
+
+function getColdCacheAgeMs(cacheFilePath, logPrefix = '[OCL]') {
+  try {
+    const stats = fsSync.statSync(cacheFilePath);
+    if (!stats || !Number.isFinite(stats.mtimeMs)) {
+      return null;
+    }
+
+    return Math.max(0, Date.now() - stats.mtimeMs);
+  } catch (error) {
+    if (error && error.code !== 'ENOENT') {
+      console.error(`${logPrefix} Failed to inspect cold cache file ${cacheFilePath}: ${error.message}`);
+    }
+    return null;
+  }
+}
+
+function formatCacheAgeMinutes(ageMs) {
+  const minutes = Math.max(1, Math.round(ageMs / 60000));
+  return `${minutes} minute${minutes === 1 ? '' : 's'}`;
+}
+
+module.exports = {
+  ensureCacheDirectories,
+  getColdCacheAgeMs,
+  formatCacheAgeMinutes
+};

package/tx/ocl/cache/cache-utils.js

@@ -0,0 +1,2 @@
+module.exports = require('./cache-utils.cjs');
+