fhirsmith 0.18.2 → 1.0.0

package/packages/package-crawler.js

@@ -17,6 +17,7 @@ const https = require('https');
 const ipaddr = require('ipaddr.js');
 const {debugLog} = require("../tx/operation-context");
 const {Parser: TarParser} = require('tar');
+const {integrityOfSync, verifyPackageSignature} = require('./npm-signing');
 
 const MB = 1024 * 1024;
 
@@ -72,6 +73,16 @@ function isLoopbackAddress(ip) {
   }
 }
 
+// A mirror file is written as <file>.tgz.tmp-<pid> and then renamed into place
+const MIRROR_TMP_MARKER = '.tmp-';
+
+// crawler.signatures: what to do with publishers' package signatures (see signingFor)
+const SIGNATURE_MODES = ['ignore', 'check', 'require'];
+// a signature file is a few KB; anything much bigger isn't one
+const MAX_SIGNATURE_BYTES = 1024 * 1024;
+// A temporary mirror file older than this was left by a crawler that stopped mid-write
+const MIRROR_TMP_MAX_AGE_MS = 60 * 60 * 1000;
+
 class PackageCrawler {
   log;
   packages = new Set();
@@ -84,6 +95,15 @@ class PackageCrawler {
     this.crawlerLog = {};
     this.errors = '';
     this.abortController = null;
+    // the npm signing key (NpmSigner), or null when this server doesn't sign
+    this.signer = null;
+    this.skippedFeeds = new Set();
+    // publisher signatures (fhir:signature in a feed, keys in package-feeds.json): ignore,
+    // check, or require - see signingFor()
+    this.signatureMode = (config.crawler && config.crawler.signatures) || 'check';
+    if (!SIGNATURE_MODES.includes(this.signatureMode)) {
+      throw new Error(`crawler.signatures must be one of ${SIGNATURE_MODES.join(', ')}, not ${this.signatureMode}`);
+    }
     this.db.run('PRAGMA journal_mode = WAL');
     this.db.run('PRAGMA busy_timeout = 5000');
   }
@@ -110,6 +130,7 @@ class PackageCrawler {
   async crawl(log) {
     this.log = log;
     this.packages.clear();
+    this.skippedFeeds = new Set();
     this.abortController = new AbortController();
 
     const startTime = Date.now();
@@ -122,6 +143,11 @@ class PackageCrawler {
     };
 
     this.log.info('Running web crawler for packages using master URL: '+ this.config.masterUrl);
+    try {
+      this.cleanMirrorTempFiles();
+    } catch (e) {
+      this.log.warn(`Could not clean the package mirror: ${e.message}`);
+    }
     this.stats.task('Package Crawler', 'Running');
 
     try {
@@ -144,9 +170,12 @@ class PackageCrawler {
         }
         try {
           let url = this.fixUrl(feedConfig.url)
+          if (this.skipUnsigned(feedConfig)) {
+            continue;
+          }
           if (!url.includes('simplifier')) {
             this.stats.task('Package Crawler', 'Running for '+feedConfig.url);
-            await this.updateTheFeed(url, this.config.masterUrl,feedConfig.errors ? feedConfig.errors.replace(/\|/g, '@').replace(/_/g, '.') : '', packageRestrictions);
+            await this.updateTheFeed(url, this.config.masterUrl,feedConfig.errors ? feedConfig.errors.replace(/\|/g, '@').replace(/_/g, '.') : '', packageRestrictions, this.signingFor(feedConfig.signing));
           }
         } catch (feedError) {
           this.log.error(`Failed to process feed ${feedConfig.url}: `+ feedError.message);
@@ -163,9 +192,12 @@ class PackageCrawler {
 
         try {
           let url = this.fixUrl(feedConfig.url)
+          if (this.skipUnsigned(feedConfig)) {
+            continue;
+          }
           if (url.includes('simplifier')) {
             this.stats.task('Package Crawler', 'Running for '+feedConfig.url);
-            await this.updateTheFeed(url, this.config.masterUrl,feedConfig.errors ? feedConfig.errors.replace(/\|/g, '@').replace(/_/g, '.') : '', packageRestrictions);
+            await this.updateTheFeed(url, this.config.masterUrl,feedConfig.errors ? feedConfig.errors.replace(/\|/g, '@').replace(/_/g, '.') : '', packageRestrictions, this.signingFor(feedConfig.signing));
           }
         } catch (feedError) {
           this.log.error(`Failed to process feed ${feedConfig.url}: `+ feedError.message);
@@ -466,7 +498,7 @@ class PackageCrawler {
     return body;
   }
 
-  async updateTheFeed(url, source, email, packageRestrictions) {
+  async updateTheFeed(url, source, email, packageRestrictions, signing = null) {
     const feedLog = {
       url: url,
       items: []
@@ -522,7 +554,7 @@ class PackageCrawler {
         for (let i = 0; i < items.length; i++) {
           if (this.abortController?.signal.aborted) break;
           try {
-            await this.updateItem(currentUrl, items[i], i, packageRestrictions, feedLog);
+            await this.updateItem(currentUrl, items[i], i, packageRestrictions, feedLog, signing);
           } catch (itemError) {
             if (itemError.message.includes('RATE_LIMITED')) {
               this.log.info(`Rate limited while downloading package from ${currentUrl}, stopping feed processing`);
@@ -568,7 +600,8 @@ class PackageCrawler {
     }
   }
 
-  async updateItem(source, item, index, packageRestrictions, feedLog) {
+  // signing: how this feed's signatures are checked (signingFor), or null
+  async updateItem(source, item, index, packageRestrictions, feedLog, signing = null) {
     const itemLog = {
       status: '??'
     };
@@ -628,6 +661,7 @@ class PackageCrawler {
       // Check if already processed
       if (await this.hasStored(guid)) {
         itemLog.status = 'Already Processed';
+        await this.checkStoredSignature(item, guid, signing, itemLog);
         return;
       }
 
@@ -672,7 +706,8 @@ class PackageCrawler {
       this.log.info('Fetching package: ' + url);
 
       const packageContent = await this.fetchUrl(url, 'application/tar+gzip');
-      const outcome = await this.store(source, url, guid, pubDate, packageContent, id, itemLog);
+      const signature = signing ? await this.fetchSignature(item) : null;
+      const outcome = await this.store(source, url, guid, pubDate, packageContent, id, itemLog, signing, signature);
 
       itemLog.status = {stored: 'Fetched', replaced: 'Replaced Simplifier copy', skipped: 'Already Stored'}[outcome];
 
@@ -735,6 +770,141 @@ class PackageCrawler {
     }
   }
 
+  // --- publisher signatures (see npm-signing.js) --------------------------------------------------
+  //
+  // A feed may list keys in package-feeds.json (feeds[].signing: {required, keys}), and its items
+  // may link a signature (fhir:signature). crawler.signatures says what to do with them:
+  //   ignore  - nothing
+  //   check   - (the default) check each signature of a feed that lists keys; one that fails is
+  //             logged, and the package is stored anyway, unless the feed says signing.required
+  //   require - as check, but an unsigned package, or one whose signature fails, is refused, for
+  //             every feed that lists keys
+  // Feeds that list no keys are never affected. The signature file is kept (PackageSignatures) as
+  // it was when it was checked, valid or not.
+
+  // With crawler.signatures = require, a feed that lists no keys isn't read at all: none of its
+  // packages could be accepted. Noted in the crawl log.
+  skipUnsigned(feedConfig) {
+    if (this.signatureMode !== 'require' || this.signingFor(feedConfig.signing)) {
+      return false;
+    }
+    if (!this.skippedFeeds.has(feedConfig.url)) {
+      this.skippedFeeds.add(feedConfig.url);
+      this.log.info(`Skipping feed ${feedConfig.url}: it lists no signing keys, and signatures are required`);
+      this.crawlerLog.feeds.push({ url: feedConfig.url, skipped: 'it lists no signing keys, and signatures are required', items: [] });
+    }
+    return true;
+  }
+
+  // How a feed's signatures are checked, or null if they aren't
+  signingFor(feedSigning) {
+    if (this.signatureMode === 'ignore' || !feedSigning || !Array.isArray(feedSigning.keys) || feedSigning.keys.length === 0) {
+      return null;
+    }
+    return { keys: feedSigning.keys, required: this.signatureMode === 'require' || feedSigning.required === true };
+  }
+
+  signatureUrl(item) {
+    const s = item['fhir:signature'];
+    const v = s && typeof s === 'object' ? s['#text'] : s;
+    return v ? String(v).trim() : null;
+  }
+
+  // The item's signature file: {url, text}, {url: null} if it has none, or {url, fetchError}
+  async fetchSignature(item) {
+    const url = this.signatureUrl(item);
+    if (!url) {
+      return { url: null };
+    }
+    try {
+      const content = await this.fetchUrl(this.fixUrl(url), 'application/json');
+      if (content.length > MAX_SIGNATURE_BYTES) {
+        return { url, fetchError: `it is too large (${content.length} bytes)` };
+      }
+      return { url, text: content.toString('utf8') };
+    } catch (e) {
+      if (e.message.includes('RATE_LIMITED')) {
+        throw e;
+      }
+      return { url, fetchError: e.message };
+    }
+  }
+
+  // Checks a package's signature. Returns {record} - what to keep in PackageSignatures - and
+  // {refuse} - why the package must not be stored - either or neither
+  checkSignature(signing, signature, packageBuffer, id, version, date, itemLog) {
+    if (!signing) {
+      return {};
+    }
+    if (!signature || !signature.url) {
+      itemLog.signature = 'unsigned';
+      return signing.required ? { refuse: 'it is not signed, and this feed\'s packages must be' } : {};
+    }
+    if (signature.fetchError) {
+      itemLog.signature = `the signature ${signature.url} could not be fetched: ${signature.fetchError}`;
+      this.log.error(`Signature for ${id}#${version} could not be fetched (${signature.url}): ${signature.fetchError}`);
+      return signing.required ? { refuse: itemLog.signature } : {};
+    }
+    const record = { url: signature.url, bundle: signature.text, keyid: null, valid: false, message: null };
+    try {
+      let bundle;
+      try {
+        bundle = JSON.parse(signature.text);
+      } catch (e) {
+        throw new Error('the signature file is not JSON');
+      }
+      const v = verifyPackageSignature(packageBuffer, bundle, signing.keys, id, version, date instanceof Date && !isNaN(date) ? date : null);
+      record.keyid = v.keyid;
+      record.valid = true;
+      itemLog.signature = `valid (${v.keyid})`;
+      return { record };
+    } catch (e) {
+      record.message = e.message;
+      itemLog.signature = `invalid: ${e.message}`;
+      this.log.error(`Signature check failed for ${id}#${version} (${signature.url}): ${e.message}`);
+      return { record, refuse: signing.required ? `its signature is not valid: ${e.message}` : null };
+    }
+  }
+
+  async recordSignature(id, version, record) {
+    await this.run(`INSERT OR REPLACE INTO PackageSignatures (PackageVersionKey, Url, KeyId, Valid, Message, Checked, Bundle)
+                    SELECT PackageVersionKey, ?, ?, ?, ?, datetime('now'), ? FROM PackageVersions WHERE Id = ? AND Version = ?`,
+      [record.url, record.keyid, record.valid ? 1 : 0, record.message, record.bundle, id, version]);
+  }
+
+  // A package stored before its feed was signed (or before this server checked): its signature is
+  // checked against the stored bytes, once. Never refused - it is already held - but a failure is
+  // logged and kept. A signature that can't be fetched is tried again next crawl.
+  async checkStoredSignature(item, guid, signing, itemLog) {
+    if (!signing || !this.signatureUrl(item)) {
+      return;
+    }
+    const row = await new Promise((resolve, reject) => {
+      this.db.get(`SELECT v.PackageVersionKey, v.Id, v.Version, v.PubDate, s.PackageVersionKey AS Checked
+                   FROM PackageVersions v LEFT JOIN PackageSignatures s ON s.PackageVersionKey = v.PackageVersionKey
+                   WHERE v.GUID = ?`, [guid], (err, r) => err ? reject(err) : resolve(r));
+    });
+    if (!row || row.Checked) {
+      return;
+    }
+    const signature = await this.fetchSignature(item);
+    if (signature.fetchError) {
+      itemLog.signature = `the signature ${signature.url} could not be fetched: ${signature.fetchError}`;
+      this.log.warn(`Signature for ${row.Id}#${row.Version} could not be fetched (${signature.url}): ${signature.fetchError}`);
+      return;
+    }
+    const content = await new Promise((resolve, reject) => {
+      this.db.get('SELECT Content FROM PackageVersions WHERE PackageVersionKey = ?', [row.PackageVersionKey], (err, r) => err ? reject(err) : resolve(r));
+    });
+    if (!content) {
+      return;
+    }
+    const verdict = this.checkSignature({ ...signing, required: false }, signature, content.Content, row.Id, row.Version, new Date(row.PubDate), itemLog);
+    if (verdict.record) {
+      await this.recordSignature(row.Id, row.Version, verdict.record);
+    }
+  }
+
   async hasStored(guid) {
     return new Promise((resolve, reject) => {
       this.db.get('SELECT COUNT(*) as count FROM PackageVersions WHERE GUID = ?', [guid], (err, row) => {
@@ -786,6 +956,55 @@ class PackageCrawler {
     });
   }
 
+  // Runs fn inside a database transaction: all of its writes commit, or none do. The crawler
+  // stores one package at a time, so its transactions never nest. The connection is shared
+  // with the web side, so a write it makes meanwhile (a download count) joins the transaction.
+  async transaction(fn) {
+    await this.run('BEGIN IMMEDIATE');
+    try {
+      const result = await fn();
+      await this.run('COMMIT');
+      return result;
+    } catch (e) {
+      try {
+        await this.run('ROLLBACK');
+      } catch (rollbackError) {
+        this.log?.error?.(`Rollback failed: ${rollbackError.message}`);
+      }
+      throw e;
+    }
+  }
+
+  // Removes mirror files that a crawler stopped before renaming into place. Only files old
+  // enough that nothing can still be writing them: the mirror may be shared, and in a
+  // container the pid in the name is no guide.
+  cleanMirrorTempFiles(maxAgeMs = MIRROR_TMP_MAX_AGE_MS) {
+    if (!this.config.mirrorPath || !fs.existsSync(this.config.mirrorPath)) {
+      return 0;
+    }
+    const cutoff = Date.now() - maxAgeMs;
+    let removed = 0;
+    for (const name of fs.readdirSync(this.config.mirrorPath)) {
+      if (!name.includes(MIRROR_TMP_MARKER)) {
+        continue;
+      }
+      const file = path.join(this.config.mirrorPath, name);
+      try {
+        const stat = fs.statSync(file);
+        if (stat.isFile() && stat.mtimeMs < cutoff) {
+          fs.rmSync(file, { force: true });
+          removed++;
+        }
+      } catch (e) {
+        this.log?.warn?.(`Could not remove stale mirror file ${name}: ${e.message}`);
+      }
+    }
+    if (removed > 0) {
+      this.log?.info?.(`Removed ${removed} stale temporary file(s) from the package mirror`);
+    }
+    return removed;
+  }
+
   // Point Packages.CurrentVersion at the most recent version of id
   async refreshCurrentVersion(id) {
     const latest = await new Promise((resolve, reject) => {
@@ -801,7 +1020,7 @@ class PackageCrawler {
 
   // A direct copy replacing the Simplifier copies we held: it takes over the first row (so its
   // key and download count survive), and any further Simplifier rows go.
-  async replaceVersion(keys, packageBuffer, npmPackage, date, guid, id, canonical, urls) {
+  async replaceVersion(keys, packageBuffer, npmPackage, date, guid, id, version, canonical, urls) {
     const [vkey, ...extra] = keys;
     await this.run(`UPDATE PackageVersions
                     SET GUID = ?, PubDate = ?, Indexed = datetime('now'), Kind = ?, Canonical = ?,
@@ -812,7 +1031,7 @@ class PackageCrawler {
         npmPackage.description, this.genHash(packageBuffer), npmPackage.author, npmPackage.license,
         npmPackage.url, packageBuffer, vkey]);
     for (const key of keys) {
-      for (const table of ['PackageFHIRVersions', 'PackageDependencies', 'PackageURLs']) {
+      for (const table of ['PackageFHIRVersions', 'PackageDependencies', 'PackageURLs', 'PackageDist', 'PackageSignatures']) {
         await this.run(`DELETE FROM ${table} WHERE PackageVersionKey = ?`, [key]);
       }
     }
@@ -820,6 +1039,7 @@ class PackageCrawler {
       await this.run('DELETE FROM PackageVersions WHERE PackageVersionKey = ?', [key]);
     }
     await this.insertRelatedData(vkey, npmPackage, urls);
+    await this.storeDist(vkey, id, version, packageBuffer);
     await this.refreshCurrentVersion(id);
   }
 
@@ -848,7 +1068,8 @@ class PackageCrawler {
     return date;
   }
 
-  async store(source, url, guid, date, packageBuffer, idver, itemLog) {
+  // signing and signature: the feed's signature checking (signingFor) and what fetchSignature got
+  async store(source, url, guid, date, packageBuffer, idver, itemLog, signing = null, signature = null) {
     try {
       // Extract and parse the NPM package
       const npmPackage = await this.extractNpmPackage(packageBuffer, `${source}#${guid}`);
@@ -876,48 +1097,63 @@ class PackageCrawler {
         throw new Error(check.reason);
       }
 
-      if (!this.isValidSemVersion(version)) {
-        throw new Error(`NPM Version "${version}" is not valid from ${source}`);
-      }
+      const canonical = this.checkPackageContent(npmPackage, idver, source);
 
-      let canonical = npmPackage.canonical || `http://simplifier.net/packages/${id}`;
-      if (!this.isAbsoluteUrl(canonical)) {
-        throw new Error(`NPM Canonical "${canonical}" is not valid from ${source}`);
-      }
-
-      const isTemplate = npmPackage.kind === 2; // fhir.template
-      if (npmPackage.hasInstallScripts) {
-        throw new Error(`Package ${idver} rejected: contains install scripts (preinstall/install/postinstall)`);
-      }
-      if (npmPackage.hasJavaScript && !isTemplate && id !== 'hl7.fhir.pubpack') {
-        throw new Error(`Package ${idver} rejected: contains JavaScript files but is not a template package`);
-      }
-      // The feed gate (item.notForPublication) only sees the RSS entry. A package whose
-      // feed entry is clean can still carry notForPublication inside the tarball - that is
-      // a draft build that must never enter the registry. Reject it here too.
-      if (npmPackage.notForPublication) {
-        throw new Error(`Package ${idver} rejected: tarball is flagged notForPublication (draft build, not suitable for publication)`);
+      const verdict = this.checkSignature(signing, signature, packageBuffer, id, version, date, itemLog);
+      if (verdict.refuse) {
+        throw new Error(`Package ${idver} rejected: ${verdict.refuse}`);
       }
 
       // Extract URLs from package
       const urls = this.processPackageUrls(npmPackage);
 
-      // Commit to database
-      await this.commit(packageBuffer, npmPackage, date, guid, id, version, canonical, urls, check.keys);
-
-      // Only now write the mirror copy: a package that failed validation or didn't commit
-      // must not replace the file that is served for the copy we hold
+      // The mirror file has to be complete before the registry advertises the package (the
+      // download redirect points at it), and a replacement must not touch the file served for
+      // the copy we hold unless it commits. So: write the new file beside its target first;
+      // a new package's file is put in place before the transaction, a replacement's at the
+      // end of it, just before COMMIT, so a failed rename rolls the replacement back.
+      // Rename within a directory is atomic, so readers see the old file or the whole new one.
+      let filepath = null;
+      let tmpPath = null;
       if (this.config.mirrorPath) {
         let fid = this.fixPrefix(id);
-        const filename = `${fid}-${version}.tgz`;
-        const filepath = path.join(this.config.mirrorPath, filename);
+        filepath = path.join(this.config.mirrorPath, `${fid}-${version}.tgz`);
         // id and version are validated, but the file must land in the mirror whatever they say
         if (path.dirname(path.resolve(filepath)) !== path.resolve(this.config.mirrorPath)) {
           throw new Error(`Package ${id}#${version} rejected: its mirror file name is not a plain file name`);
         }
-        fs.writeFileSync(filepath, packageBuffer);
+        tmpPath = `${filepath}${MIRROR_TMP_MARKER}${process.pid}`;
+      }
+      const replacing = check.action === 'replace';
+      let placed = false;
+      try {
+        if (tmpPath) {
+          fs.writeFileSync(tmpPath, packageBuffer);
+          if (!replacing) {
+            fs.renameSync(tmpPath, filepath);
+            placed = true;
+          }
+        }
+        await this.transaction(async () => {
+          await this.commit(packageBuffer, npmPackage, date, guid, id, version, canonical, urls, check.keys);
+          if (tmpPath && replacing) {
+            fs.renameSync(tmpPath, filepath);
+          }
+        });
+      } catch (e) {
+        if (tmpPath) {
+          fs.rmSync(tmpPath, { force: true });
+        }
+        if (placed) {
+          // a new package that didn't commit isn't advertised, so its file goes too
+          fs.rmSync(filepath, { force: true });
+        }
+        throw e;
       }
 
+      if (verdict.record) {
+        await this.recordSignature(id, version, verdict.record);
+      }
       return check.action === 'replace' ? 'replaced' : 'stored';
 
     } catch (error) {
@@ -927,6 +1163,40 @@ class PackageCrawler {
     }
   }
 
+  // What a package must be to be stored (beyond being new): a valid id, a whole semver version,
+  // an absolute canonical, no install scripts, no JavaScript unless it's a template, and not
+  // flagged notForPublication. Throws saying what's wrong; returns the canonical. Shared with
+  // the feed check (feed-check.js), which stores nothing.
+  checkPackageContent(npmPackage, idver, source) {
+    const {id, version} = npmPackage;
+    if (!this.isValidPackageId(id)) {
+      throw new Error(`NPM Id "${id}" is not valid from ${source}`);
+    }
+    if (!this.isValidSemVersion(version)) {
+      throw new Error(`NPM Version "${version}" is not valid from ${source}`);
+    }
+
+    let canonical = npmPackage.canonical || `http://simplifier.net/packages/${id}`;
+    if (!this.isAbsoluteUrl(canonical)) {
+      throw new Error(`NPM Canonical "${canonical}" is not valid from ${source}`);
+    }
+
+    const isTemplate = npmPackage.kind === 2; // fhir.template
+    if (npmPackage.hasInstallScripts) {
+      throw new Error(`Package ${idver} rejected: contains install scripts (preinstall/install/postinstall)`);
+    }
+    if (npmPackage.hasJavaScript && !isTemplate && id !== 'hl7.fhir.pubpack') {
+      throw new Error(`Package ${idver} rejected: contains JavaScript files but is not a template package`);
+    }
+    // The feed gate (item.notForPublication) only sees the RSS entry. A package whose
+    // feed entry is clean can still carry notForPublication inside the tarball - that is
+    // a draft build that must never enter the registry. Reject it here too.
+    if (npmPackage.notForPublication) {
+      throw new Error(`Package ${idver} rejected: tarball is flagged notForPublication (draft build, not suitable for publication)`);
+    }
+    return canonical;
+  }
+
   // Reads what the crawler needs from a package as the package is unzipped and untarred:
   // package.json, .index.json and ig.ini, the file names, and the url of each resource. Only
   // those files are held, one at a time, so memory is bounded by maxFileMB however large
@@ -1262,9 +1532,19 @@ class PackageCrawler {
     return crypto.createHash('sha1').update(data).digest('hex');
   }
 
+  // The version's dist.integrity, and its registry signature when this server signs
+  // (this.signer, set by the packages module - see npm-signing.js). In their own table,
+  // PackageDist: a column after Content in PackageVersions can only be read by walking the
+  // tarball's overflow pages, and the package document reads these for every version.
+  async storeDist(vkey, id, version, packageBuffer) {
+    const integrity = integrityOfSync(packageBuffer);
+    await this.run('INSERT OR REPLACE INTO PackageDist (PackageVersionKey, Integrity, KeyId, Signature) VALUES (?, ?, ?, ?)',
+      [vkey, integrity, this.signer ? this.signer.keyid : null, this.signer ? this.signer.sign(id, version, integrity) : null]);
+  }
+
   async commit(packageBuffer, npmPackage, date, guid, id, version, canonical, urls, replaceKeys = null) {
     if (replaceKeys && replaceKeys.length > 0) {
-      return this.replaceVersion(replaceKeys, packageBuffer, npmPackage, date, guid, id, canonical, urls);
+      return this.replaceVersion(replaceKeys, packageBuffer, npmPackage, date, guid, id, version, canonical, urls);
     }
     return new Promise((resolve, reject) => {
       // Get next version key
@@ -1298,10 +1578,11 @@ class PackageCrawler {
           }
 
           // Insert FHIR versions, dependencies, and URLs
-          this.insertRelatedData(vkey, npmPackage, urls).then(() => {
+          this.insertRelatedData(vkey, npmPackage, urls)
+            .then(() => this.storeDist(vkey, id, version, packageBuffer))
             // Handle package table (insert or update)
-            this.upsertPackage(id, vkey, canonical).then(resolve).catch(reject);
-          }).catch(reject);
+            .then(() => this.upsertPackage(id, vkey, canonical))
+            .then(resolve).catch(reject);
         });
       });
     });

package/packages/packages-template.html

@@ -54,13 +54,12 @@
 		<div id="stripe"> </div>
 		<div class="container">  <!-- container -->
 		<div style="background-color: #ad1f2f; padding: 6px; color: white;">  <!-- container -->
-  <a href="http://www.hl7.org/fhir" style="color: gold" title="Fast Healthcare Interoperability Resources - Home Page"><img border="0" src="/icon-fhir-16.png" style="vertical-align: text-bottom"/> <b>FHIR</b></a> &copy; HL7.org &nbsp;|&nbsp;
-  <a href="https://github.com/HealthIntersections/FHIRsmith/blob/main/README.md" style="color: gold"><img border="0" src="/FHIRsmith16.png" style="vertical-align: text-bottom"/> FHIRsmith</a> [%ver%] &nbsp;|&nbsp;
   <a href="/" style="color: gold"> Server Home</a> &nbsp;|&nbsp;
   <a href="/packages" style="color: gold">Packages Home</a>  &nbsp;|&nbsp;
   <a href="/packages/updates" style="color: gold">Updates</a>  &nbsp;|&nbsp;
   <a href="/packages/broken" style="color: gold">Broken</a>  &nbsp;|&nbsp;
   <a href="/packages/log" style="color: gold">Crawler Log</a>  &nbsp;|&nbsp;
+  [%feed-check-link%]
   <a href="/packages/openapi" style="color: gold">API</a>
 		</div>  <!-- /container -->
 		</div>  <!-- /container -->
@@ -74,7 +73,7 @@
             	<div class="inner-wrapper">
  <div class="col-9">
 
-  <h2><img border="0" src="../FHIRsmith32.png" style="vertical-align: text-bottom"/> [%title%] </h2>
+  <h2><a href="https://github.com/HealthIntersections/FHIRsmith/blob/main/README.md" title="FHIRsmith [%ver%]"><img border="0" src="/FHIRsmith32.png" alt="FHIRsmith Logo" style="vertical-align: text-bottom"/></a> [%title%] </h2>
 
 [%content%]
 
@@ -92,8 +91,7 @@
 		<div class="container">  <!-- container -->
 			<div class="inner-wrapper">
 				<p>
-                    <a href="http://www.hl7.org/fhir" style="color: gold" title="Fast Healthcare Interoperability Resources - Home Page"><img border="0" src="/icon-fhir-16.png" style="vertical-align: text-bottom"/> <b>FHIR</b></a> &copy; HL7.org 2011+.  &nbsp;|&nbsp;
-                    <a href="https://github.com/HealthIntersections/FHIRsmith/blob/main/README.md" style="color: gold"><img border="0" src="/FHIRsmith16.png" style="vertical-align: text-bottom"/> FHIRsmith</a>  [%ver%]  &copy; HealthIntersections.com.au 2023+ &nbsp;|&nbsp;
+                    <a href="https://github.com/HealthIntersections/FHIRsmith/blob/main/README.md" style="color: gold"><img border="0" src="/FHIRsmith16.png" alt="FHIRsmith Logo" style="vertical-align: text-bottom"/> FHIRsmith</a>  [%ver%]  &copy; HealthIntersections.com.au 2023+ &nbsp;|&nbsp;
                     Package Registry [%crawler-status%] &nbsp;|&nbsp; [%total-packages%] packages &nbsp;|
                     &nbsp; ([%ms%] ms)
                     [%sponsorMessage%]
@@ -117,10 +115,10 @@
 ================================================== -->
   <!-- Placed at the end of the document so the pages load faster -->
 <!-- <script src="/assets/js/jquery.js"/> -->
-<script src="/assets/js/bootstrap.min.js"/>
-<script src="/assets/js/respond.min.js"/>
+<script src="/assets/js/bootstrap.min.js"></script>
+<script src="/assets/js/respond.min.js"></script>
 
-<script src="/assets/js/fhir.js"/>
+<script src="/assets/js/fhir.js"></script>
 
   <!-- Analytics Below
 ================================================== -->