fhirsmith 0.16.1 → 0.18.0

package/token/token.js

@@ -435,12 +435,19 @@ class TokenModule {
     this.router.post('/keys', this.requireAuth.bind(this), this.createApiKey.bind(this));
     this.router.delete('/keys/:id', this.requireAuth.bind(this), this.deleteApiKey.bind(this));
     
-    // JSON API routes for other servers
-    this.router.get('/api/validate/:key', this.validateApiKey.bind(this));
-    this.router.post('/api/usage/:key', this.recordUsage.bind(this));
+    // JSON API routes for other servers. The key goes in a header (Authorization: Bearer, or
+    // X-API-Key), not the path, so that it doesn't end up in access logs
+    this.router.get('/api/validate', this.validateApiKey.bind(this));
+    this.router.post('/api/usage', this.recordUsage.bind(this));
     
     // Statistics
-    this.router.get('/api/stats/:key', this.getUsageStats.bind(this));
+    this.router.get('/api/stats', this.getUsageStats.bind(this));
+  }
+
+  // The API key a request carries: Authorization: Bearer <key>, or X-API-Key: <key>
+  apiKeyFrom(req) {
+    const auth = /^Bearer\s+(\S+)$/i.exec(req.get('Authorization') || '');
+    return auth ? auth[1] : (req.get('X-API-Key') || null);
   }
 
   // Middleware
@@ -627,7 +634,7 @@ class TokenModule {
     try {
 
       try {
-        const apiKey = req.params.key;
+        const apiKey = this.apiKeyFrom(req);
 
         if (!apiKey || !apiKey.startsWith('tk_')) {
           return res.status(400).json({
@@ -700,7 +707,7 @@ class TokenModule {
     try {
 
       try {
-        const apiKey = req.params.key;
+        const apiKey = this.apiKeyFrom(req);
         const {count = 1} = req.body;
 
         if (!apiKey || !apiKey.startsWith('tk_')) {
@@ -737,8 +744,8 @@ class TokenModule {
     try {
 
       try {
-        const apiKey = req.params.key;
-        const days = Math.min(parseInt(req.query.days) || 30, 365); // Max 1 year
+        const apiKey = this.apiKeyFrom(req);
+        const days = Math.max(1, Math.min(parseInt(req.query.days) || 30, 365)); // Max 1 year
 
         if (!apiKey || !apiKey.startsWith('tk_')) {
           return res.status(400).json({error: 'Invalid key format'});
@@ -989,7 +996,7 @@ class TokenModule {
   async getApiKeyUsageStats(keyId, days = 30) {
     return new Promise((resolve, reject) => {
       this.db.all(`
-        SELECT date, request_count, last_request_ip
+        SELECT date, request_count
         FROM usage_stats 
         WHERE api_key_id = ? AND date >= date('now', '-${days} days')
         ORDER BY date DESC
@@ -1134,7 +1141,7 @@ class TokenModule {
             <td><span class="${statusClass}">${status}</span></td>
             <td>
               ${key.is_active ? 
-                `<button class="btn btn-sm btn-danger" onclick="deleteKey(${key.id}, '${escape(key.name)}')">Delete</button>` :
+                `<button class="btn btn-sm btn-danger" onclick="deleteKey(${Number(key.id)})">Delete</button>` :
                 '<span class="text-muted">-</span>'
               }
             </td>

package/translations/Messages.properties

@@ -181,6 +181,9 @@ CONCEPTMAP_GROUP_TARGET_PROPERTY_TYPE_NO_SYSTEM = Since no system has been provi
 CONCEPTMAP_GROUP_TARGET_PROPERTY_TYPE_UNKNOWN_SYSTEM = The code system {0} could not be found, so code values can''t be checked
 CONCEPTMAP_GROUP_TARGET_SERVER_SIDE = Target Code System {0} is only supported on the terminology server, so the target codes are not validated for performance reasons
 CONCEPTMAP_GROUP_TARGET_UNKNOWN = The Target Code System {0} is not fully defined and populated, and no targetScope is specified, so the target code checking will not be performed
+TRANSLATE_TARGET_CS_UNKNOWN = The code system {0} is not known, and the code ''{1}'' has not been validated
+TRANSLATE_SCT_EDITION_TOP_LEVEL_ONE = No translation: the only {0} ancestor of {1} is the top-level hierarchy concept {2}
+TRANSLATE_SCT_EDITION_TOP_LEVEL_MANY = No translation: the only {0} ancestors of {1} are the top-level hierarchy concepts {2}
 CONCEPTMAP_SHAREABLE_EXTRA_MISSING = Published concept maps SHOULD conform to the ShareableConceptMap profile, which says that the element ConceptMap.{0} should be present, but it is not
 CONCEPTMAP_SHAREABLE_EXTRA_MISSING_HL7 = Concept maps published by HL7 SHALL conform to the ShareableConceptMap profile, which says that the element ConceptMap.{0} should be present, but it is not
 CONCEPTMAP_SHAREABLE_MISSING = Published concept maps SHOULD conform to the ShareableConceptMap profile, which says that the element ConceptMap.{0} is mandatory, but it is not present
@@ -377,6 +380,11 @@ HTA_LOINC_DESC = LOINC
 HTA_LOINC_MESSAGE = This material contains content from <a href="http://loinc.org">LOINC</a>. LOINC is copyright &copy; 1995-2020, Regenstrief Institute, Inc. and the Logical Observation Identifiers Names and Codes (LOINC) Committee and is available at no cost under the <a href="http://loinc.org/license">license</a>. LOINC&reg; is a registered United States trademark of Regenstrief Institute, Inc.
 HTA_SCT_DESC = SNOMED Clinical Terms&reg; (SNOMED CT&reg;)
 HTA_SCT_MESSAGE = This material contains content that is copyright of SNOMED International. Implementers of these specifications must have the appropriate SNOMED CT Affiliate license - for more information contact <a href="https://www.snomed.org/get-snomed">https://www.snomed.org/get-snomed</a> or <a href="mailto:info@snomed.org">info@snomed.org</a>.
+ICD11_CLASSIFICATION_CODE_NONE = The classificationCode extension says ''{0}'', but ''{1}'' has no classification code
+ICD11_CLASSIFICATION_CODE_NOT_ID = The classificationCode extension is only used when the code is an entity id, and ''{0}'' is not one (the entity id is ''{1}'')
+ICD11_CLASSIFICATION_CODE_SYSTEM = The classificationCode extension is only defined for ICD-11 MMS and ICF codings, not {0}
+ICD11_CLASSIFICATION_CODE_WRONG = The classificationCode extension says ''{0}'', but the classification code of ''{1}'' is ''{2}''
+ICD11_PC_AXIS_NOT_MULTIPLE = ''{0}'' takes one value on its {1} axis, but has several: {2}
 ICD11_PC_NO_AXES = ''{0}'' cannot be postcoordinated onto ''{1}'': that concept declares no postcoordination axes
 ICD11_PC_VALUE_NOT_ON_AXIS = ''{0}'' is not a permitted value on any postcoordination axis of ''{1}''. That stem declares: {2}
 IG_DEPENDENCY_CAN_VERSION_ALONE = The canonical URL specifies the version ''{0}'', while the dependency does not. If a version is specified, it must be in the explicit version property
@@ -412,6 +420,7 @@ Internal_INT_Bad_Type = Unhandled fixed value type {0}
 INV_FAILED = Constraint failed: {0}: ''{1}''
 INV_FAILED_SOURCE = Constraint failed: {0}: ''{1}'' (defined in {2})
 INVALID_DISPLAY_NAME = Invalid displayLanguage: ''{0}''
+LOOKUP_DISPLAY_LANGUAGE_FALLBACK = There is no display in the language ''{0}'' for the code ''{1}'' in {2}, so the display is in the default language ''{3}''
 Invalid_slicing__there_is_more_than_one_type_slice_at__but_one_of_them__has_min__1_so_the_other_slices_cannot_exist=Invalid slicing: there is more than one type slice at {0}, but one of them ({1}) has min = 1, so the other slices cannot exist
 JSON_COMMA_EXTRA = There is an extra comma at the end of the {0} in the JSON
 JSON_COMMA_MISSING = A Comma is missing in the JSON

package/tx/README.md

@@ -148,6 +148,58 @@ The file is read from disk per request, so editing it is reflected immediately -
 this means the published file can be ahead of what the running server actually loaded, if it
 has been edited since startup.
 
+### Syndication feed
+
+`syndication.enabled` publishes the content the library loads as a
+[Terminology Syndication Feed](https://build.fhir.org/ig/FHIR/fhir-syndication-ig/en/)
+(Atom, NCTS ASF profile), so that Ontoserver, other FHIRsmith servers and people can follow
+what the server loads. It is off by default.
+
+```json
+"syndication": { "enabled": true }
+```
+
+**The store.** As the library loads its sources, each one is recorded in the syndication store,
+`<data>/terminology-cache/syndication` (`syndication.json` plus `files/`). The store is defined
+from the moment syndication is turned on: content loaded before then is not back-filled, but
+since induction happens as content loads, everything the server loads after that is recorded
+the first time the store sees it, with the time it was recorded. The store only grows -
+entries are never changed or removed, there are no retract entries, and a later update to
+the same content is a new entry (so `contentItemVersion` is not unique - a deliberate departure
+from the IG). One store serves all the endpoints.
+
+| Source | Entry | Link | Category |
+|--------|-------|------|----------|
+| `npm:` / `url:` packages, and each endpoint's core package | the package | `https://npm.fhir.org/web/{id}-{version}.tgz` (a `url:` source: its url) | `FHIR_Package` |
+| `file:` / `folder:` | each resource that has a version | a copy in the store | `FHIR_CodeSystem` / `FHIR_ValueSet` / `FHIR_ConceptMap` |
+| `snomed`, `loinc`, `rxnorm`, `ndc`, `unii`, `omop`, `icd11`, `cpt` (fragment only) | the file | `base.url` + file name | `SNOMED_CACHE`, `LOINC_DB` ... in `http://healthintersections.com.au/syndication/fhirsmith/1.0.0` |
+
+Not syndicated: `internal:` sources (including VSAC), OCL, `#current`/build packages, resources
+without a version, the full CPT database, and binaries that are not published at `base.url`
+(checked with a HEAD request - a binary that is only local is a development build). Binary
+entries carry `<rights>`. Every link carries `ncts:sha256Hash`: a package is downloaded once to
+hash it, then discarded. Only the core packages are FHIR version specific; everything else
+appears in every endpoint's feed, with that endpoint's `ncts:fhirVersion`.
+
+Induction runs in the background, one item at a time, so it never holds up startup; a failure
+is logged and tried again the next time the content loads.
+
+**The feed** uses RFC 5005 archived feeds, 30 entries a page:
+
+| URL | Content |
+|-----|---------|
+| `/{path}/syndication` | the newest entries (fewer than 30), linked to the newest archive |
+| `/{path}/syndication/archive/{n}` | archive *n* (1 = oldest), linked back to *n*-1 - never changes once it exists |
+| `/{path}/syndication/files/{name}` | a resource file copied into the store |
+
+The links in the feed are built from the server's `hostName` (top level of config.json) as
+`https://{hostName}{path}`, never from the request's Host header; only archives built that way
+are served as cacheable forever. Without a `hostName` the links follow the request, and
+archives are `no-cache`.
+
+A browser (`Accept: text/html`) gets the feed as a page; `?_format=atom` forces Atom. The IG's
+query-string filters are not supported. A **Syndication** item appears in the tx navigation bar.
+
 ## OpenAPI description
 
 Each R5 endpoint is described by an OpenAPI 3.1 spec (the R4 and R3 endpoints work the same

package/tx/cm/cm-database.js

@@ -2,6 +2,9 @@ const fs = require('fs').promises;
 const sqlite3 = require('sqlite3').verbose();
 const { VersionUtilities } = require('../../library/version-utilities');
 const { ConceptMap } = require('../library/conceptmap');
+// Key of the row written once the cache has been fully populated (see isComplete)
+const CACHE_COMPLETE_KEY = 'cache-complete';
+
 // Columns that can be returned directly without parsing JSON
 const INDEXED_COLUMNS = ['id', 'url', 'version', 'date', 'description', 'name', 'publisher', 'status', 'title'];
 
@@ -113,6 +116,14 @@ class ConceptMapDatabase {
           db.run('CREATE INDEX idx_jurisdictions_code ON conceptmap_jurisdictions(code)');
           db.run('CREATE INDEX idx_systems_system ON conceptmap_systems(system)');
 
+          // Cache status (key-value) - holds the marker that population finished
+          db.run(`
+              CREATE TABLE cache_status (
+                                          key TEXT PRIMARY KEY,
+                                          value TEXT
+              )
+          `);
+
           db.close((err) => {
             if (err) {
               reject(new Error(`Failed to close database after creation: ${err.message}`));
@@ -125,6 +136,57 @@ class ConceptMapDatabase {
     });
   }
 
+  /**
+   * True if this cache was fully populated: markComplete() was reached. A cache
+   * whose population was cut short (the process killed part way through) has no
+   * marker - nor does one built before the marker existed, or a file too damaged
+   * to read - so it reports false.
+   * @returns {Promise<boolean>}
+   */
+  async isComplete() {
+    return new Promise((resolve) => {
+      const db = new sqlite3.Database(this.dbPath, sqlite3.OPEN_READONLY, (err) => {
+        if (err) {
+          resolve(false);
+          return;
+        }
+        db.get('SELECT value FROM cache_status WHERE key = ?', [CACHE_COMPLETE_KEY], (err, row) => {
+          db.close(() => resolve(!err && row != null));
+        });
+      });
+    });
+  }
+
+  /**
+   * Record that this cache has been fully populated. Called once, after the last
+   * concept map has been written
+   * @returns {Promise<void>}
+   */
+  async markComplete() {
+    return new Promise((resolve, reject) => {
+      const db = new sqlite3.Database(this.dbPath, (err) => {
+        if (err) {
+          reject(new Error(`Failed to open database: ${err.message}`));
+          return;
+        }
+        db.serialize(() => {
+          db.run('CREATE TABLE IF NOT EXISTS cache_status (key TEXT PRIMARY KEY, value TEXT)');
+          db.run(`INSERT INTO cache_status (key, value) VALUES (?, ?)
+                    ON CONFLICT(key) DO UPDATE SET value = excluded.value`,
+            [CACHE_COMPLETE_KEY, new Date().toISOString()], (err) => {
+              db.close((closeErr) => {
+                if (err || closeErr) {
+                  reject(new Error(`Failed to mark cache complete: ${(err || closeErr).message}`));
+                } else {
+                  resolve();
+                }
+              });
+            });
+        });
+      });
+    });
+  }
+
   /**
    * Insert or update a single ConceptMap in the database
    * @param {Object} conceptMap - The ConceptMap resource

package/tx/cm/cm-package.js

@@ -1,4 +1,6 @@
 const path = require('path');
+const fs = require('fs').promises;
+const { assertNoActiveContent } = require('../../library/narrative-check');
 const { AbstractConceptMapProvider } = require('./cm-api');
 const { PackageContentLoader } = require('../../library/package-manager');
 const { ConceptMapDatabase } = require('./cm-database');
@@ -20,7 +22,9 @@ class PackageConceptMapProvider extends AbstractConceptMapProvider {
     super();
     validateParameter(packageLoader, "packageLoader", PackageContentLoader);
     this.packageLoader = packageLoader;
-    this.dbPath = path.join(packageLoader.packageFolder, '.conceptmaps.db');
+    // -2: caches from before concept maps were checked for active content in their
+    // narrative are rebuilt (see initialize)
+    this.dbPath = path.join(packageLoader.packageFolder, '.conceptmaps-2.db');
     this.database = new ConceptMapDatabase(this.dbPath);
     this.conceptMapMap = new Map();
     this.initialized = false;
@@ -36,12 +40,23 @@ class PackageConceptMapProvider extends AbstractConceptMapProvider {
       return;
     }
     await this.packageLoader.initialize();
-
-    const dbExists = await this.database.exists();
+    await fs.rm(path.join(this.packageLoader.packageFolder, '.conceptmaps.db'), { force: true }); // the old cache
+
+    // A cache is only used once its population finished. One that was cut short
+    // (the process killed part way through) would otherwise be trusted forever,
+    // silently missing whatever hadn't been written yet
+    let dbExists = await this.database.exists();
+    if (dbExists && !(await this.database.isComplete())) {
+      require('../../library/logger').getInstance().child({ module: 'tx' })
+        .warn(`Package ${this.packageLoader.pid ? this.packageLoader.pid() : ''}: concept map cache ${this.dbPath} is incomplete - rebuilding it`);
+      await fs.rm(this.dbPath, { force: true });
+      dbExists = false;
+    }
 
     if (!dbExists) {
       await this.database.create();
       await this._populateDatabase();
+      await this.database.markComplete();
     }
 
     this.conceptMapMap = await this.database.loadAllConceptMaps();
@@ -68,6 +83,8 @@ class PackageConceptMapProvider extends AbstractConceptMapProvider {
       const conceptMap = await this.packageLoader.loadFile(entry);
       if (conceptMap.url) {
         try {
+          // its narrative is shown as-is in a page, so it must not carry script
+          assertNoActiveContent(conceptMap);
           conceptMaps.push(new ConceptMap(conceptMap, this.packageLoader.fhirVersion())); // get in converted to R5 format
         } catch (e) {
           // one unacceptable concept map (e.g. one with contained resources, which the

package/tx/cs/cs-api.js

@@ -67,19 +67,89 @@ class CodeSystemProvider {
    */
 
   /**
-   * The display name of the code system - what $lookup reports as its name parameter, and
-   * what a person reads. Every provider should override this; the fallback is the
-   * versioned uri, which is not a name, and a provider that forgets ends up reporting
+   * The name of the code system (CodeSystem.name, or the provider's equivalent). Every
+   * provider should override this; the fallback is the versioned uri, which is not a name,
+   * and a provider that forgets ends up reporting
    * "http://snomed.info/sct|http://snomed.info/sct/731000124108/version/20230301" where
    * the others report "LOINC".
+   * @returns {string} name of the code system
+   */
+  name() { return this.system() + (this.version() ? "|"+this.version() : "") }
+
+  /**
+   * What $lookup reports as its `name` parameter - "a display name for the code system".
+   * That is the title, translated into the request's language where the provider can,
+   * falling back to name(). Only $lookup uses this; everything else keeps using name().
+   * The default has no title to offer, so it is name().
+   * @param {Languages} [languages] - the request's working languages (displayLanguage, else
+   *   Accept-Language)
    * @returns {string} display name for the code system
    */
+  // eslint-disable-next-line no-unused-vars
+  lookupName(languages) { return this.name(); }
+
+  /**
+   * A second, shorter form of the code that some code systems also carry - ICD-11's short
+   * classification code (1A00) beside its code, the entity uri. $validate-code reports it as
+   * the classificationCode output parameter when the request used the code itself, and
+   * $expand always reports it as a contains property. Null (the default) when there is none.
+   * @param {CodeSystemProviderContext} context
+   * @returns {Promise<string|null>}
+   */
+  // eslint-disable-next-line no-unused-vars
+  async classificationCode(context) { return null; }
+
   /**
-   * @param {Languages} [languages] - the request's working languages, for a provider whose name
-   *   is translated; the default ignores them
+   * The classificationCode $validate-code reports for a code as it was given: by default the
+   * concept's classificationCode, whatever spelling was used.
+   * @param {CodeSystemProviderContext} context the concept the code located
+   * @param {string} code the code as given
+   * @returns {Promise<string|null>}
    */
   // eslint-disable-next-line no-unused-vars
-  name(languages) { return this.system() + (this.version() ? "|"+this.version() : "") }
+  async reportedClassificationCode(context, code) { return this.classificationCode(context); }
+
+  /**
+   * The code to put in an expansion for a concept the value set enumerated as `requested`.
+   * The default keeps the value set's own spelling; a code system whose normal form differs
+   * from what a value set may write (ICD-11: short code vs entity uri) returns code().
+   * @param {CodeSystemProviderContext} context
+   * @param {string} requested - the code as the value set wrote it
+   * @returns {Promise<string>}
+   */
+  // eslint-disable-next-line no-unused-vars
+  async expansionCode(context, requested) { return requested; }
+
+  /**
+   * Properties an expansion carries on every contains entry whatever the request asked
+   * for (each must also come back from properties()). None by default.
+   * @returns {string[]}
+   */
+  mandatoryExpansionProperties() { return []; }
+
+  /**
+   * Check the extensions on a Coding that this code system defines - ICD-11's
+   * classificationCode, say. Called by $validate-code once the coding's code has validated.
+   * Extensions the code system does not define are not its business, and are ignored.
+   *
+   * @param {Object} coding the Coding as given
+   * @param {CodeSystemProviderContext} context the concept the code located
+   * @returns {Promise<{index: number, msgId: string, params: string[]}[]>} one entry per problem,
+   *   naming the extension (its index in coding.extension) and the message; none by default
+   */
+  // eslint-disable-next-line no-unused-vars
+  async checkCodingExtensions(coding, context) { return []; }
+
+  /**
+   * Things $validate-code should say about a code that is valid - for a postcoordinated
+   * expression, say, a qualifier the grammar allows but a rule of the code system does not
+   * (ICD-11: two values on an axis that takes one).
+   *
+   * @param {CodeSystemProviderContext} context the concept the code located
+   * @returns {Promise<{severity: string, msgId: string, params: string[]}[]>} none by default
+   */
+  // eslint-disable-next-line no-unused-vars
+  async codeIssues(context) { return []; }
 
   /**
    * @returns {string} uri for the code system
@@ -540,9 +610,11 @@ class CodeSystemProvider {
    * @param {CodeSystemProviderContext} ctxt the context to add properties for
    * @param {string[]} props the properties requested
    * @param {Parameters} params the parameters response to add to
+   * @param {TxParameters} [txParams] the parsed request parameters, for providers whose output
+   *   depends on more than the property list (ICD-11's post-coordination-metadata)
    */
-
-  async extendLookup(ctxt, props, params) { }
+  // eslint-disable-next-line no-unused-vars
+  async extendLookup(ctxt, props, params, txParams) { }
 
   // procedure getCDSInfo(card : TCDSHookCard; langList : THTTPLanguageList; baseURL, code, display : String); virtual;
 
@@ -954,15 +1026,19 @@ class CodeSystemFactoryProvider {
   }
 
   /**
-   * build and return a known concept map from the URL, if there is one.
+   * Add any concept maps this code system implies for translating from source
+   * (a system URI, with the version of the source coding, which may be empty)
+   * to dest (a system URI, possibly with |version).
    *
-   * the conceptmap is never visible to a user; if it has an implicitSource, then
+   * The conceptmap is never visible to a user; if it has an internalSource, then
    * provider.getTranslations will be called when it's actually used
-   * @param url
-   * @param version
-   * @returns {ConceptMap}
+   * @param conceptMaps - the list to add to
+   * @param source - the source system URI
+   * @param sourceVersion - the version of the source coding (may be null)
+   * @param dest - the target system URI, possibly with |version
    */
-  async findImplicitConceptMaps(conceptMaps, source, dest) {
+  // eslint-disable-next-line no-unused-vars
+  async findImplicitConceptMaps(conceptMaps, source, sourceVersion, dest) {
     return null;
   }