fhirsmith 0.7.5 → 0.7.6

package/CHANGELOG.md

@@ -5,6 +5,30 @@ All notable changes to the Health Intersections Node Server will be documented i
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [v0.7.6] - 2026-03-25
+
+### Added
+
+- Dashboard endpoint (see dashboard.html)
+- Initial cs-api documentation
+
+### Changed
+
+- Update package crawler to support archived feed entries
+
+### Fixed
+
+- OCL improvements:
+   - Improve multilingual support and caching for non-OCL expansions
+   - cache compose instead of pre-built expansions
+- Fix ConceptMap rendering
+- Ongoing on work on publishing module
+- Tidy up tx-reg to prevent hanging
+
+### Tx Conformance Statement
+
+FHIRsmith passed all 1464 HL7 terminology service tests (modes tx.fhir.org+omop+general+snomed, tests v1.9.1, runner v6.9.3)
+
 ## [v0.7.5] - 2026-03-19
 
 ### Changed

package/README.md

@@ -17,6 +17,14 @@ This server provides a set of server-side services that are useful for the FHIR
 * [VCL](vcl/readme.md) - **Parse VCL expressions** into FHIR ValueSet resources for http://fhir.org/vcl
 * (Coming) Token services
 
+## Summary Statement
+
+* Maintainers: Grahame Grieve, Italo Macêdo, Josh Mandel, Jose Costa Teixeira 
+* Issues / Discussion: Use github issues
+* License: BSD-3
+* Contribution Policy: Make PRs. PRs have to pass all the tests 
+* Security Information: See [security.md](security.md)
+ 
 ## Build Status
 ![CI Build](https://github.com/HealthIntersections/fhirsmith/actions/workflows/ci.yml/badge.svg)
 [![Release](https://img.shields.io/github/v/release/HealthIntersections/fhirsmith?include_prereleases)](https://github.com/HealthIntersections/fhirsmith/releases)

package/library/html.js

@@ -428,6 +428,10 @@ class XhtmlNode {
     return this.addTag('label').setAttribute('for', forId);
   }
 
+  colspan(width) {
+    return this.attr("colspan", String(width));
+  }
+
   // Conditional
   iff(test) {
     if (test) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fhirsmith",
-  "version": "0.7.5",
+  "version": "0.7.6",
   "description": "A Node.js server that provides a collection of tools to serve the FHIR ecosystem",
   "main": "server.js",
   "engines": {

package/packages/package-crawler.js

@@ -27,6 +27,25 @@ class PackageCrawler {
     this.db.run('PRAGMA busy_timeout = 5000');
   }
 
+  async isFeedPageVisited(url) {
+    return new Promise((resolve, reject) => {
+      this.db.get('SELECT Url FROM FeedPages WHERE Url = ?', [url], (err, row) => {
+        if (err) reject(err);
+        else resolve(!!row);
+      });
+    });
+  }
+
+  async markFeedPageVisited(url) {
+    return new Promise((resolve, reject) => {
+      this.db.run(
+        'INSERT OR REPLACE INTO FeedPages (Url, VisitedAt) VALUES (?, ?)',
+        [url, new Date().toISOString()],
+        (err) => { if (err) reject(err); else resolve(); }
+      );
+    });
+  }
+
   async crawl(log) {
     this.log = log;
     this.packages.clear();
@@ -151,7 +170,8 @@ class PackageCrawler {
         const parser = new XMLParser({
           ignoreAttributes: false,
           attributeNamePrefix: '@_',
-          textNodeName: '#text'
+          textNodeName: '#text',
+          entityExpansionLimit: 100000
         });
         return parser.parse(content);
       } else {
@@ -166,13 +186,13 @@ class PackageCrawler {
         const parser = new XMLParser({
           ignoreAttributes: false,
           attributeNamePrefix: '@_',
-          textNodeName: '#text'
+          textNodeName: '#text',
+          entityExpansionLimit: 100000
         });
-
         return parser.parse(response.data);
       }
     } catch (error) {
-      debugLog(error);
+      debugLog(`Failed to fetch XML from ${url}: ${error.message}`);
       if (error.response && error.response.status === 429) {
         throw new Error(`RATE_LIMITED: Server returned 429 Too Many Requests for ${url}`);
       }
@@ -200,7 +220,7 @@ class PackageCrawler {
         return Buffer.from(response.data);
       }
     } catch (error) {
-      debugLog(error);
+      debugLog(`Failed to fetch ${url}: ${error.message}`);
       if (error.response && error.response.status === 429) {
         throw new Error(`RATE_LIMITED: Server returned 429 Too Many Requests for ${url}`);
       }
@@ -218,62 +238,96 @@ class PackageCrawler {
     this.log.info('Processing feed: ' + url);
     const startTime = Date.now();
 
-    try {
-      const xmlData = await this.fetchXml(url);
-      feedLog.fetchTime = `${Date.now() - startTime}ms`;
+    // The first page (the root feed URL) is always processed — it contains the
+    // latest packages. Subsequent pages (followed via atom:link rel="next") are
+    // historical archives and only need to be visited once.
+    let currentUrl = url;
+    let isFirstPage = true;
 
-      // Navigate the RSS structure
-      let items = [];
-      if (xmlData.rss && xmlData.rss.channel) {
-        const channel = xmlData.rss.channel;
-        items = Array.isArray(channel.item) ? channel.item : [channel.item].filter(Boolean);
+    while (currentUrl) {
+      if (this.abortController?.signal.aborted) break;
+
+      if (!isFirstPage) {
+        const alreadyVisited = await this.isFeedPageVisited(currentUrl);
+        if (alreadyVisited) {
+          this.log.info(`Feed page already visited, stopping: ${currentUrl}`);
+          break;
+        }
       }
 
-      this.log.info(`Found ${items.length} items in feed`);
+      try {
+        this.log.info(`Fetching feed page: ${currentUrl}`);
+        const xmlData = await this.fetchXml(currentUrl);
+        if (isFirstPage) feedLog.fetchTime = `${Date.now() - startTime}ms`;
+
+        let items = [];
+        let nextUrl = null;
+
+        if (xmlData.rss && xmlData.rss.channel) {
+          const channel = xmlData.rss.channel;
+          items = Array.isArray(channel.item) ? channel.item : [channel.item].filter(Boolean);
+
+          // Check for RFC 5005 next-page link: <atom:link rel="next" href="..."/>
+          const atomLinks = channel['atom:link'];
+          if (atomLinks) {
+            const links = Array.isArray(atomLinks) ? atomLinks : [atomLinks];
+            const nextLink = links.find(l => l['@_rel'] === 'next');
+            if (nextLink && nextLink['@_href']) {
+              nextUrl = this.fixUrl(nextLink['@_href']);
+            }
+          }
+        }
 
-      for (let i = 0; i < items.length; i++) {
-        if (this.abortController?.signal.aborted) break;
-        try {
-          await this.updateItem(url, items[i], i, packageRestrictions, feedLog);
-        } catch (itemError) {
-          // Check if this is a 429 error on package download
-          if (itemError.message.includes('RATE_LIMITED')) {
-            this.log.info(`Rate limited while downloading package from ${url}, stopping feed processing`);
-            feedLog.rateLimited = true;
-            feedLog.rateLimitedAt = `item ${i}`;
-            feedLog.rateLimitMessage = itemError.message;
-            break; // Stop processing this feed
+        this.log.info(`Found ${items.length} items in feed page ${currentUrl}`);
+
+        let rateLimited = false;
+        for (let i = 0; i < items.length; i++) {
+          if (this.abortController?.signal.aborted) break;
+          try {
+            await this.updateItem(currentUrl, items[i], i, packageRestrictions, feedLog);
+          } catch (itemError) {
+            if (itemError.message.includes('RATE_LIMITED')) {
+              this.log.info(`Rate limited while downloading package from ${currentUrl}, stopping feed processing`);
+              feedLog.rateLimited = true;
+              feedLog.rateLimitedAt = `item ${i}`;
+              feedLog.rateLimitMessage = itemError.message;
+              rateLimited = true;
+              break;
+            }
+            this.log.error(`Error processing item ${i} from ${currentUrl}:` + itemError.message);
           }
-          // For other errors, log and continue with next item
-          this.log.error(`Error processing item ${i} from ${url}:` + itemError.message);
         }
-      }
 
-      // TODO: Send email if there were errors and email is provided
-      if (this.errors && email && !feedLog.rateLimited) {
-        this.log.info(`Would send error email to ${email} for feed ${url}`);
-      }
+        if (rateLimited) break;
 
-    } catch (error) {
-      debugLog(error);
-      // Check if this is a 429 error on feed fetch
-      if (error.message.includes('RATE_LIMITED')) {
-        this.log.info(`Rate limited while fetching feed ${url}, skipping this feed`);
-        feedLog.rateLimited = true;
-        feedLog.rateLimitMessage = error.message;
-        feedLog.failTime = `${Date.now() - startTime}ms`;
-        return; // Skip this feed entirely
-      }
+        // Mark this page as visited now that we've successfully processed it.
+        // Don't mark the first page — it must always be re-crawled for new entries.
+        if (!isFirstPage) {
+          await this.markFeedPageVisited(currentUrl);
+        }
 
-      feedLog.exception = error.message;
-      feedLog.failTime = `${Date.now() - startTime}ms`;
-      this.log.error(`Exception processing feed ${url}:` + error.message);
+        currentUrl = nextUrl;
+        isFirstPage = false;
 
-      // TODO: Send email notification for non-rate-limit errors
-      if (email) {
-        this.log.info(`Would send exception email to ${email} for feed ${url}`);
+      } catch (error) {
+        debugLog(error);
+        if (error.message.includes('RATE_LIMITED')) {
+          this.log.info(`Rate limited while fetching feed ${currentUrl}, stopping`);
+          feedLog.rateLimited = true;
+          feedLog.rateLimitMessage = error.message;
+          feedLog.failTime = `${Date.now() - startTime}ms`;
+          break;
+        }
+        feedLog.exception = error.message;
+        feedLog.failTime = `${Date.now() - startTime}ms`;
+        this.log.error(`Exception processing feed ${currentUrl}:` + error.message);
+        break;
       }
     }
+
+    if (this.errors && email && !feedLog.rateLimited) {
+      this.log.info(`Would send error email to ${email} for feed ${url}`);
+    }
   }
 
   async updateItem(source, item, index, packageRestrictions, feedLog) {
@@ -699,6 +753,7 @@ class PackageCrawler {
       };
 
     } catch (error) {
+      console.log(error);
       throw new Error(`Failed to extract NPM package from ${source}: ${error.message}`);
     }
   }

package/packages/packages.js

@@ -617,6 +617,13 @@ class PackagesModule {
             this.createTables().then(resolve).catch(reject);
           } else {
             pckLog.info('Packages database already exists');
+            // Run migrations for tables added after initial schema
+            this.db.run(`CREATE TABLE IF NOT EXISTS FeedPages (
+              Url TEXT PRIMARY KEY,
+              VisitedAt TEXT NOT NULL
+            )`, (err) => {
+              if (err) pckLog.error('Failed to create FeedPages table:', err.message);
+            });
             resolve();
           }
         }
@@ -692,6 +699,13 @@ class PackagesModule {
              ManualToken          TEXT(64) NOT NULL,
              Email                TEXT(128) NOT NULL,
              Mask                 TEXT(64)
+         )`,
+
+        // FeedPages table - tracks which paginated feed archive pages have been visited
+        `CREATE TABLE FeedPages
+         (
+             Url       TEXT PRIMARY KEY,
+             VisitedAt TEXT NOT NULL
          )`
       ];
 

package/publisher/publisher.js

@@ -269,6 +269,7 @@ class PublisherModule {
     this.router.post('/tasks', this.requireAuth.bind(this), this.createTask.bind(this));
     this.router.post('/tasks/:id/approve', this.requireAuth.bind(this), this.approveTask.bind(this));
     this.router.post('/tasks/:id/delete', this.requireAdmin.bind(this), this.deleteTask.bind(this));
+    this.router.post('/tasks/:id/retry', this.requireAuth.bind(this), this.retryTask.bind(this));
     this.router.get('/tasks/:id/output', this.getTaskOutput.bind(this));
     this.router.get('/tasks/:id/history', this.getTaskHistory.bind(this));
     this.router.get('/tasks/:id/qa', this.getTaskQA.bind(this));
@@ -295,19 +296,31 @@ class PublisherModule {
 
   // Background Task Processing
   startTaskProcessor() {
-    const pollInterval = this.config.pollInterval || 5000; // Default 5 seconds
+    const pollInterval = this.config.pollInterval || 5000;
 
     this.logger.info('Starting task processor with ' + pollInterval + 'ms poll interval');
+    this.isProcessingStarted = null;
 
     this.taskProcessor = setInterval(async () => {
-      if (!this.isProcessing && !this.shutdownRequested) {
-        await this.processNextTask();
+      if (this.shutdownRequested) return;
+
+      if (this.isProcessing) {
+        const stuckMs = this.isProcessingStarted ? Date.now() - this.isProcessingStarted : 0;
+        if (stuckMs > 60 * 60 * 1000) {
+          this.logger.warn('Task processor appears stuck (' + Math.round(stuckMs / 60000) + ' min) — resetting');
+          this.isProcessing = false;
+        } else {
+          return;
+        }
       }
+
+      await this.processNextTask();
     }, pollInterval);
   }
 
   async processNextTask() {
     this.isProcessing = true;
+    this.isProcessingStarted = Date.now();
 
     try {
       // Look for queued tasks first (draft builds)
@@ -489,16 +502,9 @@ class PublisherModule {
   }
 
   async createTaskDirectory(taskDir) {
-    const rimraf = require('rimraf');
-
     // Remove existing directory if it exists
     if (fs.existsSync(taskDir)) {
-      await new Promise((resolve, reject) => {
-        rimraf(taskDir, (err) => {
-          if (err) reject(err);
-          else resolve();
-        });
-      });
+      await require('fs').promises.rm(taskDir, { recursive: true, force: true });
     }
 
     // Create fresh directory
@@ -1108,7 +1114,8 @@ class PublisherModule {
             content += '<a href="/publisher/tasks/' + task.id + '/history" class="btn btn-sm btn-outline-secondary me-1">History</a>';
 
             if (task.status === 'waiting for approval') {
-              content += '<a href="/publisher/tasks/' + task.id + '/output" class="btn btn-sm btn-outline-info me-1">View Output</a>';
+              content += '<a href="/publisher/tasks/' + task.id + '/output" class="btn btn-sm btn-outline-info me-1">View Log</a>';
+              content += '<a href="/publisher/tasks/' + task.id + '/qa-files/index.html" class="btn btn-sm btn-outline-secondary me-1">View IG</a>';
               content += '<a href="/publisher/tasks/' + task.id + '/qa" class="btn btn-sm btn-outline-secondary me-1">View QA</a>';
               if (canApprove) {
                 content += '<form method="post" action="/publisher/tasks/' + task.id + '/approve" style="display: inline;">';
@@ -1116,9 +1123,12 @@ class PublisherModule {
                 content += '</form>';
               }
             } else {
-              if (task.build_output_path) {
+              if (task.build_output_path || task.local_folder) {
                 content += '<a href="/publisher/tasks/' + task.id + '/output" class="btn btn-sm btn-outline-info me-1">View Output</a>';
               }
+              if (task.announcement) {
+                content += '<a href="/publisher/tasks/' + task.id + '/history" class="btn btn-sm btn-outline-success me-1">📢 Announcement</a>';
+              }
               if (task.failure_reason) {
                 content += '<span class="text-danger small me-1">' + escape(task.failure_reason) + '</span>';
               }
@@ -1130,6 +1140,12 @@ class PublisherModule {
               content += '</form>';
             }
 
+            if (req.session.userId && task.status === 'failed') {
+              content += ' <form method="post" action="/publisher/tasks/' + task.id + '/retry" style="display: inline;">';
+              content += '<button type="submit" class="btn btn-sm btn-warning">Retry</button>';
+              content += '</form>';
+            }
+
             content += '</td>';
             content += '</tr>';
           }
@@ -1272,15 +1288,11 @@ class PublisherModule {
 
         // Remove build output directory
         if (task.local_folder && fs.existsSync(task.local_folder)) {
-          const rimraf = require('rimraf');
-          await new Promise((resolve) => {
-            rimraf(task.local_folder, (err) => {
-              if (err) {
-                this.logger.warn('Failed to remove task directory ' + task.local_folder + ': ' + err.message);
-              }
-              resolve(); // Continue even if directory removal fails
-            });
-          });
+          try {
+            await require('fs').promises.rm(task.local_folder, { recursive: true, force: true });
+          } catch (err) {
+            this.logger.warn('Failed to remove task directory ' + task.local_folder + ': ' + err.message);
+          }
         }
 
         // Delete task logs
@@ -1304,13 +1316,44 @@ class PublisherModule {
         res.redirect('/publisher/tasks');
       } catch (error) {
         this.logger.error('Error deleting task:', error);
-        res.status(500).send('Failed to delete task');
+        res.status(500).send('Failed to delete task: ' + error.message);
       }
     } finally {
       this.stats.countRequest('delete-task', Date.now() - start);
     }
   }
 
+  async retryTask(req, res) {
+    const start = Date.now();
+    try {
+      try {
+        const taskId = req.params.id;
+        const task = await this.getTask(taskId);
+        if (!task) return res.status(404).send('Task not found');
+        if (task.status !== 'failed') return res.status(400).send('Only failed tasks can be retried');
+        const canQueue = await this.userCanQueue(req.session.userId, task.website_id);
+        if (!canQueue) return res.status(403).send('You do not have permission to queue tasks for this website');
+        const existingTask = await this.findActiveTask(task.npm_package_id, task.version);
+        if (existingTask) return res.status(400).send('An active task for this package and version is already in progress.');
+        const newTaskId = await new Promise((resolve, reject) => {
+          this.db.run(
+            'INSERT INTO tasks (user_id, website_id, github_org, github_repo, git_branch, npm_package_id, version) VALUES (?, ?, ?, ?, ?, ?, ?)',
+            [req.session.userId, task.website_id, task.github_org, task.github_repo, task.git_branch, task.npm_package_id, task.version],
+            function (err) { if (err) reject(err); else resolve(this.lastID); }
+          );
+        });
+        this.logUserAction(req.session.userId, 'retry_task', newTaskId.toString(), req.ip);
+        this.logger.info('Task retried: new ID=' + newTaskId + ' from task #' + taskId);
+        res.redirect('/publisher/tasks/' + newTaskId + '/history');
+      } catch (error) {
+        this.logger.error('Error retrying task:', error);
+        res.status(500).send('Failed to retry task: ' + error.message);
+      }
+    } finally {
+      this.stats.countRequest('retry-task', Date.now() - start);
+    }
+  }
+
   async getTaskOutput(req, res) {
     const start = Date.now();
     try {
@@ -1336,6 +1379,19 @@ class PublisherModule {
           }
         }
 
+        // Get publication log if available
+        let publishLog = '';
+        if (task.local_folder) {
+          const publishLogPath = path.join(task.local_folder, 'publication.log');
+          if (fs.existsSync(publishLogPath)) {
+            try {
+              publishLog = fs.readFileSync(publishLogPath, 'utf8');
+            } catch (error) {
+              publishLog = 'Error reading publication log: ' + error.message;
+            }
+          }
+        }
+
         if (req.headers.accept && req.headers.accept.includes('text/html')) {
           const htmlServer = require('../library/html-server');
           let content = '<h3>Task Output: #' + task.id + ' - ' + task.npm_package_id + '#' + task.version + '</h3>';
@@ -1364,6 +1420,13 @@ class PublisherModule {
             content += '</div>';
           }
 
+          // Announcement section
+          if (task.announcement) {
+            content += '<h4>Announcement</h4>';
+            content += '<button onclick="navigator.clipboard.writeText(document.getElementById(\'announcement-text\').innerText)" class="btn btn-sm btn-outline-secondary mb-2">Copy to clipboard</button>';
+            content += '<div id="announcement-text" class="output-viewer" style="white-space: pre-wrap;">' + escape(task.announcement) + '</div>';
+          }
+
           // Build log section
           if (buildLog) {
             content += '<h4>Build Log</h4>';
@@ -1373,6 +1436,15 @@ class PublisherModule {
             content += '<p><em>Build in progress... Log will appear when available.</em></p>';
           }
 
+          // Publication log section
+          if (publishLog) {
+            content += '<h4>Publication Log</h4>';
+            content += '<div class="output-viewer">' + escape(publishLog) + '</div>';
+          } else if (task.status === 'publishing') {
+            content += '<h4>Publication Log</h4>';
+            content += '<p><em>Publication in progress... Log will appear when available.</em></p>';
+          }
+
           content += '<div class="mt-3"><a href="/publisher/tasks" class="btn btn-secondary">Back to Tasks</a></div>';
 
           const html = htmlServer.renderPage('publisher', 'Task Output - FHIR Publisher', content, {
@@ -1410,6 +1482,11 @@ class PublisherModule {
             output += buildLog;
           }
 
+          if (publishLog) {
+            output += '\n--- Publication Log ---\n';
+            output += publishLog;
+          }
+
           res.setHeader('Content-Type', 'text/plain');
           res.send(output);
         }
@@ -1478,8 +1555,11 @@ class PublisherModule {
         // Announcement section (for completed publications)
         if (task.announcement) {
           content += '<div class="card mb-4"><div class="card-body">';
-          content += '<h5>Announcement</h5>';
-          content += '<pre class="mb-0" style="white-space: pre-wrap;">' + escape(task.announcement) + '</pre>';
+          content += '<div class="d-flex justify-content-between align-items-center mb-2">';
+          content += '<h5 class="mb-0">Announcement</h5>';
+          content += '<button onclick="navigator.clipboard.writeText(document.getElementById(\'hist-announcement\').innerText)" class="btn btn-sm btn-outline-secondary">Copy to clipboard</button>';
+          content += '</div>';
+          content += '<pre id="hist-announcement" class="mb-0" style="white-space: pre-wrap;">' + escape(task.announcement) + '</pre>';
           content += '</div></div>';
         }
 
@@ -1578,12 +1658,22 @@ class PublisherModule {
 
         // Links at the bottom
         content += '<div class="mt-3">';
-        if (task.build_output_path) {
-          content += '<a href="/publisher/tasks/' + task.id + '/output" class="btn btn-outline-info me-2">View Build Output</a>';
+        if (task.build_output_path || task.local_folder) {
+          content += '<a href="/publisher/tasks/' + task.id + '/output" class="btn btn-outline-info me-2">View Output</a>';
         }
         if (task.status === 'waiting for approval') {
           content += '<a href="/publisher/tasks/' + task.id + '/qa" class="btn btn-outline-secondary me-2">View QA Report</a>';
         }
+        if (req.session.userId && task.status === 'failed') {
+          content += '<form method="post" action="/publisher/tasks/' + task.id + '/retry" style="display: inline;" class="me-2">';
+          content += '<button type="submit" class="btn btn-warning">Retry</button>';
+          content += '</form>';
+        }
+        if (req.session.isAdmin && task.status === 'failed') {
+          content += '<form method="post" action="/publisher/tasks/' + task.id + '/delete" style="display: inline;" class="me-2" onsubmit="return confirm(\'Delete task #' + task.id + ' and all its build output? This cannot be undone.\')">';
+          content += '<button type="submit" class="btn btn-danger">Delete</button>';
+          content += '</form>';
+        }
         content += '<a href="/publisher/tasks" class="btn btn-secondary">Back to Tasks</a>';
         content += '</div>';