fhirsmith 0.5.6 → 0.7.0

package/packages/package-crawler.js

@@ -9,10 +9,12 @@ const {XMLParser} = require('fast-xml-parser');
 const crypto = require('crypto');
 const fs = require('fs');
 const path = require('path');
+const {debugLog} = require("../tx/operation-context");
 
 class PackageCrawler {
   log;
-  
+  packages = new Set();
+
   constructor(config, db, stats) {
     this.config = config;
     this.db = db;
@@ -20,13 +22,16 @@ class PackageCrawler {
     this.totalBytes = 0;
     this.crawlerLog = {};
     this.errors = '';
+    this.abortController = null;
     this.db.run('PRAGMA journal_mode = WAL');
     this.db.run('PRAGMA busy_timeout = 5000');
   }
 
   async crawl(log) {
     this.log = log;
-    
+    this.packages.clear();
+    this.abortController = new AbortController();
+
     const startTime = Date.now();
     this.crawlerLog = {
       startTime: new Date().toISOString(),
@@ -52,19 +57,36 @@ class PackageCrawler {
 
       // Process each feed
       for (const feedConfig of masterResponse.feeds) {
+        if (this.abortController?.signal.aborted) break;
+        if (!feedConfig.url) {
+          this.log.info('Skipping feed with no URL: '+ feedConfig);
+          continue;
+        }
+        try {
+          let url = this.fixUrl(feedConfig.url)
+          if (!url.includes('simplifier')) {
+            this.stats.task('Package Crawler', 'Running for '+feedConfig.url);
+            await this.updateTheFeed(url, this.config.masterUrl,feedConfig.errors ? feedConfig.errors.replace(/\|/g, '@').replace(/_/g, '.') : '', packageRestrictions);
+          }
+        } catch (feedError) {
+          this.log.error(`Failed to process feed ${feedConfig.url}: `+ feedError.message);
+          // Continue with next feed even if this one fails
+        }
+      }
+      // process simplifier last
+      for (const feedConfig of masterResponse.feeds) {
+        if (this.abortController?.signal.aborted) break;
         if (!feedConfig.url) {
           this.log.info('Skipping feed with no URL: '+ feedConfig);
           continue;
         }
-        this.stats.task('Package Crawler', 'Running for '+feedConfig.url);
 
         try {
-          await this.updateTheFeed(
-            this.fixUrl(feedConfig.url),
-            this.config.masterUrl,
-            feedConfig.errors ? feedConfig.errors.replace(/\|/g, '@').replace(/_/g, '.') : '',
-            packageRestrictions
-          );
+          let url = this.fixUrl(feedConfig.url)
+          if (url.includes('simplifier')) {
+            this.stats.task('Package Crawler', 'Running for '+feedConfig.url);
+            await this.updateTheFeed(url, this.config.masterUrl,feedConfig.errors ? feedConfig.errors.replace(/\|/g, '@').replace(/_/g, '.') : '', packageRestrictions);
+          }
         } catch (feedError) {
           this.log.error(`Failed to process feed ${feedConfig.url}: `+ feedError.message);
           // Continue with next feed even if this one fails
@@ -100,14 +122,21 @@ class PackageCrawler {
 
   async fetchJson(url) {
     try {
-      const response = await axios.get(url, {
-        timeout: 30000,
-        headers: {
-          'User-Agent': 'FHIR Package Crawler/1.0'
-        }
-      });
-      return response.data;
+      if (url.startsWith("/")) {
+        const content = await fs.promises.readFile(url, "utf8");
+        return JSON.parse(content);
+      } else {
+        const response = await axios.get(url, {
+          timeout: 30000,
+          signal: this.abortController?.signal,
+          headers: {
+            'User-Agent': 'FHIR Package Crawler/1.0'
+          }
+        });
+        return response.data;
+      }
     } catch (error) {
+      debugLog(error);
       if (error.response && error.response.status === 429) {
         throw new Error(`RATE_LIMITED: Server returned 429 Too Many Requests for ${url}`);
       }
@@ -117,21 +146,33 @@ class PackageCrawler {
 
   async fetchXml(url) {
     try {
-      const response = await axios.get(url, {
-        timeout: 30000,
-        headers: {
-          'User-Agent': 'FHIR Package Crawler/1.0'
-        }
-      });
+      if (url.startsWith("/")) {
+        const content = await fs.promises.readFile(url, 'utf8');
+        const parser = new XMLParser({
+          ignoreAttributes: false,
+          attributeNamePrefix: '@_',
+          textNodeName: '#text'
+        });
+        return parser.parse(content);
+      } else {
+        const response = await axios.get(url, {
+          timeout: 30000,
+          signal: this.abortController?.signal,
+          headers: {
+            'User-Agent': 'FHIR Package Crawler/1.0'
+          }
+        });
 
-      const parser = new XMLParser({
-        ignoreAttributes: false,
-        attributeNamePrefix: '@_',
-        textNodeName: '#text'
-      });
+        const parser = new XMLParser({
+          ignoreAttributes: false,
+          attributeNamePrefix: '@_',
+          textNodeName: '#text'
+        });
 
-      return parser.parse(response.data);
+        return parser.parse(response.data);
+      }
     } catch (error) {
+      debugLog(error);
       if (error.response && error.response.status === 429) {
         throw new Error(`RATE_LIMITED: Server returned 429 Too Many Requests for ${url}`);
       }
@@ -141,17 +182,25 @@ class PackageCrawler {
 
   async fetchUrl(url) {
     try {
-      const response = await axios.get(url, {
-        timeout: 60000,
-        responseType: 'arraybuffer',
-        headers: {
-          'User-Agent': 'FHIR Package Crawler/1.0'
-        }
-      });
+      if (url.startsWith("/")) {
+        const buffer = await fs.promises.readFile(url);
+        this.totalBytes += buffer.byteLength;
+        return buffer;
+      } else {
+        const response = await axios.get(url, {
+          timeout: 60000,
+          responseType: 'arraybuffer',
+          signal: this.abortController?.signal,
+          headers: {
+            'User-Agent': 'FHIR Package Crawler/1.0'
+          }
+        });
 
-      this.totalBytes += response.data.byteLength;
-      return Buffer.from(response.data);
+        this.totalBytes += response.data.byteLength;
+        return Buffer.from(response.data);
+      }
     } catch (error) {
+      debugLog(error);
       if (error.response && error.response.status === 429) {
         throw new Error(`RATE_LIMITED: Server returned 429 Too Many Requests for ${url}`);
       }
@@ -166,7 +215,7 @@ class PackageCrawler {
     };
     this.crawlerLog.feeds.push(feedLog);
 
-    this.log.info('Processing feed: '+ url);
+    this.log.info('Processing feed: ' + url);
     const startTime = Date.now();
 
     try {
@@ -183,6 +232,7 @@ class PackageCrawler {
       this.log.info(`Found ${items.length} items in feed`);
 
       for (let i = 0; i < items.length; i++) {
+        if (this.abortController?.signal.aborted) break;
         try {
           await this.updateItem(url, items[i], i, packageRestrictions, feedLog);
         } catch (itemError) {
@@ -195,7 +245,7 @@ class PackageCrawler {
             break; // Stop processing this feed
           }
           // For other errors, log and continue with next item
-          this.log.error(`Error processing item ${i} from ${url}:`+ itemError.message);
+          this.log.error(`Error processing item ${i} from ${url}:` + itemError.message);
         }
       }
 
@@ -205,6 +255,7 @@ class PackageCrawler {
       }
 
     } catch (error) {
+      debugLog(error);
       // Check if this is a 429 error on feed fetch
       if (error.message.includes('RATE_LIMITED')) {
         this.log.info(`Rate limited while fetching feed ${url}, skipping this feed`);
@@ -216,7 +267,7 @@ class PackageCrawler {
 
       feedLog.exception = error.message;
       feedLog.failTime = `${Date.now() - startTime}ms`;
-      this.log.error(`Exception processing feed ${url}:`+ error.message);
+      this.log.error(`Exception processing feed ${url}:` + error.message);
 
       // TODO: Send email notification for non-rate-limit errors
       if (email) {
@@ -262,7 +313,7 @@ class PackageCrawler {
       }
 
       // Check package restrictions
-      if (!this.isPackageAllowed(id, source, packageRestrictions)) {
+      if (!this.isPackageAllowed(id, source, packageRestrictions).allowed) {
         if (!source.includes('simplifier.net')) {
           const error = `The package ${id} is not allowed to come from ${source}`;
           this.log.info(error);
@@ -275,6 +326,12 @@ class PackageCrawler {
         return;
       }
 
+      if (this.packages.has(id)) {
+        this.log.info(`Ignoring package ${id} because it's already been seen in another feed`);
+        return;
+      }
+      this.packages.add(id);
+
       // Check if already processed
       if (await this.hasStored(guid)) {
         itemLog.status = 'Already Processed';
@@ -283,11 +340,12 @@ class PackageCrawler {
 
       // Parse publication date
       let pubDate;
+      let pd;
       try {
         let pd = item.pubDate;
         pubDate = this.parsePubDate(pd);
       } catch (error) {
-        itemLog.error = `Invalid date format '{pd}': ${error.message}`;
+        itemLog.error = `Invalid date format '${pd}': ${error.message}`;
         itemLog.status = 'error';
         return;
       }
@@ -301,7 +359,7 @@ class PackageCrawler {
       }
 
       itemLog.url = url;
-      this.log.info('Fetching package: '+ url);
+      this.log.info('Fetching package: ' + url);
 
       const packageContent = await this.fetchUrl(url, 'application/tar+gzip');
       await this.store(source, url, guid, pubDate, packageContent, id, itemLog);
@@ -309,13 +367,14 @@ class PackageCrawler {
       itemLog.status = 'Fetched';
 
     } catch (error) {
-      this.log.error(`Exception processing item ${itemLog.guid || index}:`+ error.message);
+      this.log.error(`Exception processing item ${itemLog.guid || index} from ${source}: `+ error.message);
       itemLog.status = 'Exception';
       itemLog.error = error.message;
       if (error.message.includes('RATE_LIMITED')) {
         throw error;
       }
     }
+
   }
 
   isPackageAllowed(packageId, source, restrictions) {
@@ -336,7 +395,7 @@ class PackageCrawler {
 
       if (this.matchesPattern(fixedPackageId, fixedMask)) {
         // This package matches a restriction - check if source is allowed
-        const allowedFeeds = restriction.feeds.map(feed => feed);
+        const allowedFeeds = restriction.feeds.map(feed => fixUrl(feed));
         const feedList = allowedFeeds.join(', ');
 
         for (const allowedFeed of restriction.feeds) {
@@ -416,18 +475,19 @@ class PackageCrawler {
         itemLog.warning = warning;
       }
 
+      // Validate package data
+      if (!this.isValidPackageId(id)) {
+        throw new Error(`NPM Id "${id}" is not valid from ${source}`);
+      }
+
       // Save to mirror if configured
       if (this.config.mirrorPath) {
-        const filename = `${id}-${version}.tgz`;
+        let fid = this.fixPrefix(id);
+        const filename = `${fid}-${version}.tgz`;
         const filepath = path.join(this.config.mirrorPath, filename);
         fs.writeFileSync(filepath, packageBuffer);
       }
 
-      // Validate package data
-      if (!this.isValidPackageId(id)) {
-        throw new Error(`NPM Id "${id}" is not valid from ${source}`);
-      }
-
       if (!this.isValidSemVersion(version)) {
         throw new Error(`NPM Version "${version}" is not valid from ${source}`);
       }
@@ -437,6 +497,14 @@ class PackageCrawler {
         throw new Error(`NPM Canonical "${canonical}" is not valid from ${source}`);
       }
 
+      const isTemplate = npmPackage.kind === 2; // fhir.template
+      if (npmPackage.hasInstallScripts) {
+        throw new Error(`Package ${idver} rejected: contains install scripts (preinstall/install/postinstall)`);
+      }
+      if (npmPackage.hasJavaScript && !isTemplate) {
+        throw new Error(`Package ${idver} rejected: contains JavaScript files but is not a template package`);
+      }
+
       // Extract URLs from package
       const urls = this.processPackageUrls(npmPackage);
 
@@ -444,6 +512,7 @@ class PackageCrawler {
       await this.commit(packageBuffer, npmPackage, date, guid, id, version, canonical, urls);
 
     } catch (error) {
+      debugLog(error);
       this.log.error(`Error storing package ${guid}:`+ error.message);
       throw error;
     }
@@ -506,6 +575,14 @@ class PackageCrawler {
       }
 
       const packageJson = JSON.parse(files['package.json']);
+      const hasInstallScripts = !!(
+        packageJson.scripts && (
+          packageJson.scripts.preinstall ||
+          packageJson.scripts.install ||
+          packageJson.scripts.postinstall
+        )
+      );
+      const hasJavaScript = Object.keys(files).some(f => f.endsWith('.js') || f.endsWith('.mjs') || f.endsWith('.cjs'));
 
       // Extract basic NPM fields
       const id = packageJson.name || '';
@@ -615,6 +692,8 @@ class PackageCrawler {
         url: homepage,
         dependencies,
         kind,
+        hasInstallScripts,
+        hasJavaScript,
         notForPublication,
         files
       };
@@ -668,7 +747,7 @@ class PackageCrawler {
 
   isValidPackageId(id) {
     // Simple package ID validation
-    return /^[a-z0-9][a-z0-9._-]*$/.test(id);
+    return /^(@[a-z0-9._-]+\/)?[a-z0-9][a-z0-9._-]*$/.test(id);
   }
 
   isValidSemVersion(version) {
@@ -846,6 +925,19 @@ class PackageCrawler {
       });
     });
   }
+
+  fixPrefix(id) {
+    if (id && id.startsWith("@") && id.includes("/")) {
+      return id.replace("@", "$$").replace("/", "$");
+    } else {
+      return id;
+    }
+  }
+  shutdown() {
+    if (this.abortController) {
+      this.abortController.abort();
+    }
+  }
 }
 
 module.exports = PackageCrawler;

package/packages/packages.js

@@ -803,6 +803,7 @@ class PackagesModule {
 
   stopCrawlerJob() {
     if (this.crawlerJob) {
+      this.crawler.shutdown();
       this.crawlerJob.stop();
       this.crawlerJob = null;
       pckLog.info('Package crawler job stopped');
@@ -1067,13 +1068,13 @@ class PackagesModule {
         const {id, version} = req.params;
 
         if (!id || !version ||
-          !/^[a-zA-Z0-9._-]+$/.test(id) ||
+          !/^(@[a-z0-9._-]+\/)?[a-zA-Z0-9._-]+$/.test(id) ||
           !/^[a-zA-Z0-9._-]+$/.test(version)) {
-          return res.status(400).json({error: 'Invalid package id or version format'});
+          return res.status(400).json({error: `Invalid package id or version format: ${id}`});
         }
 
         if (id.length > 100 || version.length > 50) {
-          return res.status(400).json({error: 'Package id or version too long'});
+          return res.status(400).json({error: `Package id or version too long: ${id}`});
         }
 
         next();
@@ -1517,7 +1518,7 @@ class PackagesModule {
       // Check if we should redirect to bucket storage
       if (this.config.bucketPath) {
         let bucketUrl = this.getBucketUrl(secure);
-        const redirectUrl = `${bucketUrl}${id}-${version}.tgz`;
+        const redirectUrl = `${bucketUrl}${this.fixPrefix(id)}-${version}.tgz`;
         res.redirect(redirectUrl);
         return;
       }
@@ -1888,7 +1889,7 @@ class PackagesModule {
       table += `<td>${escape(this.codeForKind(pv.Kind))}</td>`;
       table += `<td>${new Date(pv.PubDate).toLocaleDateString()}</td>`;
       table += `<td>${(pv.DownloadCount || 0).toLocaleString()}</td>`;
-      table += `<td><a href="/packages/${escape(id)}/${escape(pv.Version)}" class="btn btn-sm btn-primary">Download</a></td>`;
+      table += `<td><a href="/packages/${encodeURIComponent(id)}/${escape(pv.Version)}" class="btn btn-sm btn-primary">Download</a></td>`;
       table += '</tr>';
     }
 
@@ -2126,8 +2127,8 @@ class PackagesModule {
 
     for (const pkg of results) {
       table += '<tr>';
-      table += `<td><a href="${escape(pkg.url)}">${escape(pkg.name)}</a></td>`;
-      table += `<td>${escape(pkg.version)} (<a href="/packages/${escape(pkg.name)}">all</a>)</td>`;
+      table += `<td><a href="${escape(this.fixPrefix(pkg.url))}">${escape(pkg.name)}</a></td>`;
+      table += `<td>${escape(pkg.version)} (<a href="/packages/${encodeURIComponent(pkg.name)}">all</a>)</td>`;
       table += `<td>${escape(pkg.fhirVersion)}</td>`;
       table += `<td>${escape(pkg.kind)}</td>`;
       table += `<td>${pkg.date ? new Date(pkg.date).toLocaleDateString() : 'N/A'}</td>`;
@@ -2817,6 +2818,13 @@ class PackagesModule {
     return content;
   }
 
+  fixPrefix(id) {
+    if (id && id.startsWith("@") && id.includes("/")) {
+      return id.replace("@", "$$").replace("/", "$");
+    } else {
+      return id;
+    }
+  }
 }
 
 module.exports = PackagesModule;

package/publisher/publisher.js

@@ -91,6 +91,8 @@ class PublisherModule {
                                                id INTEGER PRIMARY KEY AUTOINCREMENT,
                                                name TEXT NOT NULL,
                                                local_folder TEXT NOT NULL,
+                                               history_templates TEXT NOT NULL,
+                                               web_templates TEXT NOT NULL,
                                                server_update_script TEXT NOT NULL,
                                                is_active BOOLEAN DEFAULT 1,
                                                created_at DATETIME DEFAULT CURRENT_TIMESTAMP
@@ -1605,6 +1607,14 @@ class PublisherModule {
         content += '<input type="text" class="form-control" id="local_folder" name="local_folder" required>';
         content += '</div>';
         content += '<div class="col-md-4">';
+        content += '<label for="history_templates" class="form-label">History Templates</label>';
+        content += '<input type="text" class="form-control" id="history_templates" name="history_templates" required>';
+        content += '</div>';
+        content += '<div class="col-md-4">';
+        content += '<label for="web_templates" class="form-label">Web Templates</label>';
+        content += '<input type="text" class="form-control" id="web_templates" name="web_templates" required>';
+        content += '</div>';
+        content += '<div class="col-md-4">';
         content += '<label for="server_update_script" class="form-label">Update Script</label>';
         content += '<input type="text" class="form-control" id="server_update_script" name="server_update_script" required>';
         content += '</div>';
@@ -1631,6 +1641,8 @@ class PublisherModule {
             content += '<tr>';
             content += '<td>' + website.name + '</td>';
             content += '<td><code>' + website.local_folder + '</code></td>';
+            content += '<td><code>' + website.history_templates + '</code></td>';
+            content += '<td><code>' + website.web_templates + '</code></td>';
             content += '<td><code>' + website.server_update_script + '</code></td>';
             content += '<td>' + (website.is_active ? '✓' : '✗') + '</td>';
             content += '<td>' + new Date(website.created_at).toLocaleString() + '</td>';
@@ -1665,12 +1677,12 @@ class PublisherModule {
     const start = Date.now();
     try {
       try {
-        const {name, local_folder, server_update_script} = req.body;
+        const {name, local_folder, history_templates, web_templates, server_update_script} = req.body;
 
         await new Promise((resolve, reject) => {
           this.db.run(
-            'INSERT INTO websites (name, local_folder, server_update_script) VALUES (?, ?, ?)',
-            [name, local_folder, server_update_script],
+            'INSERT INTO websites (name, local_folder, history_templates, web_templates, server_update_script) VALUES (?, ?, ?, ?, ?)',
+            [name, local_folder, history_templates, web_templates, server_update_script],
             function (err) {
               if (err) reject(err);
               else resolve();