fhirsmith 0.16.0 → 0.17.0

package/packages/package-crawler.js

@@ -16,6 +16,33 @@ const http = require('http');
 const https = require('https');
 const ipaddr = require('ipaddr.js');
 const {debugLog} = require("../tx/operation-context");
+const {Parser: TarParser} = require('tar');
+
+const MB = 1024 * 1024;
+
+// Limits on what a package can make the crawler download, unpack and hold in memory, so
+// that a hostile or broken package (a decompression bomb, say) costs a bounded amount.
+// All can be set in the packages "crawler" config section:
+//   maxDownloadMB  the package file as downloaded (or read from a local feed)
+//   maxUnpackedMB  the total of the package's files once unzipped. The files are streamed,
+//                  not held, so this limits the work, not the memory
+//   maxFileMB      a file the crawler reads: package.json, .index.json, ig.ini, and each
+//                  .json file (for its url). Memory is bounded by this, not by the package
+//   maxFiles       the number of files in the package
+const LIMIT_DEFAULTS = { maxDownloadMB: 250, maxUnpackedMB: 4096, maxFileMB: 50, maxFiles: 250000 };
+
+function crawlerLimits(config) {
+  const c = { ...LIMIT_DEFAULTS, ...(config?.crawler || {}) };
+  return {
+    maxDownloadBytes: c.maxDownloadMB * MB,
+    maxUnpackedBytes: c.maxUnpackedMB * MB,
+    maxFileBytes: c.maxFileMB * MB,
+    maxFiles: c.maxFiles
+  };
+}
+
+// The files in a package that the crawler reads for its metadata
+const META_FILES = new Set(['package.json', '.index.json', 'ig.ini']);
 
 // True if an IP literal is anything other than a normal public (unicast) address:
 // loopback, private, link-local (incl. 169.254.169.254 cloud metadata), unique-local,
@@ -350,8 +377,14 @@ class PackageCrawler {
 
   async fetchUrl(url) {
     try {
+      const limits = crawlerLimits(this.config);
       if (url.startsWith("/")) {
-        const buffer = await fs.promises.readFile(this.resolveLocalReadPath(url));
+        const localPath = this.resolveLocalReadPath(url);
+        const size = (await fs.promises.stat(localPath)).size;
+        if (size > limits.maxDownloadBytes) {
+          throw new Error(`the package is ${Math.round(size / MB)} MB, more than the limit of ${limits.maxDownloadBytes / MB} MB (maxDownloadMB)`);
+        }
+        const buffer = await fs.promises.readFile(localPath);
         this.totalBytes += buffer.byteLength;
         return buffer;
       } else {
@@ -360,6 +393,7 @@ class PackageCrawler {
           responseType: 'arraybuffer',
           // We want the package file's own bytes: decodePackageBody undoes the Content-Encoding
           decompress: false,
+          maxContentLength: limits.maxDownloadBytes,
           signal: this.abortController?.signal,
           headers: {
             'User-Agent': 'FHIR Package Crawler/1.0'
@@ -373,6 +407,9 @@ class PackageCrawler {
       }
     } catch (error) {
       debugLog(`Failed to fetch ${url}: ${error.message}`);
+      if (/maxContentLength/.test(error.message)) {
+        throw new Error(`Failed to fetch ${url}: the package is more than the limit of ${crawlerLimits(this.config).maxDownloadBytes / MB} MB (maxDownloadMB)`);
+      }
       if (error.response && error.response.status === 429) {
         throw new Error(`RATE_LIMITED: Server returned 429 Too Many Requests for ${url}`);
       }
@@ -387,21 +424,41 @@ class PackageCrawler {
   // package unless unzipping it gives another gzip file - a server that really did compress
   // a .tgz for transfer.
   decodePackageBody(body, contentEncoding) {
+    // undoing the transfer encoding gives the package file, so the download limit applies
+    const max = { maxOutputLength: crawlerLimits(this.config).maxDownloadBytes };
+    const tooBig = (e) => e.code === 'ERR_BUFFER_TOO_LARGE'
+      ? new Error(`the package is more than the limit of ${max.maxOutputLength / MB} MB (maxDownloadMB) once its Content-Encoding is undone`) : e;
     const encodings = String(contentEncoding || '').toLowerCase().split(',').map(e => e.trim()).filter(e => e && e !== 'identity');
     for (const enc of encodings.reverse()) {
       if (enc === 'gzip' || enc === 'x-gzip') {
+        let start;
         try {
-          const inner = zlib.gunzipSync(body);
-          if (inner[0] === 0x1f && inner[1] === 0x8b) {
-            body = inner;
-          }
+          // only the start is needed to see whether it is gzip inside: a tar unzipped
+          // from a .tgz can be far larger than the download limit
+          start = zlib.gunzipSync(body.subarray(0, 4096), { finishFlush: zlib.constants.Z_SYNC_FLUSH });
         } catch (e) {
           // not gzip after all: leave it for extractNpmPackage to report
+          continue;
+        }
+        if (start[0] === 0x1f && start[1] === 0x8b) {
+          try {
+            body = zlib.gunzipSync(body, max);
+          } catch (e) {
+            throw tooBig(e);
+          }
         }
       } else if (enc === 'br') {
-        body = zlib.brotliDecompressSync(body);
+        try {
+          body = zlib.brotliDecompressSync(body, max);
+        } catch (e) {
+          throw tooBig(e);
+        }
       } else if (enc === 'deflate') {
-        body = zlib.inflateSync(body);
+        try {
+          body = zlib.inflateSync(body, max);
+        } catch (e) {
+          throw tooBig(e);
+        }
       } else {
         throw new Error(`Unsupported Content-Encoding ${enc}`);
       }
@@ -854,6 +911,10 @@ class PackageCrawler {
         let fid = this.fixPrefix(id);
         const filename = `${fid}-${version}.tgz`;
         const filepath = path.join(this.config.mirrorPath, filename);
+        // id and version are validated, but the file must land in the mirror whatever they say
+        if (path.dirname(path.resolve(filepath)) !== path.resolve(this.config.mirrorPath)) {
+          throw new Error(`Package ${id}#${version} rejected: its mirror file name is not a plain file name`);
+        }
         fs.writeFileSync(filepath, packageBuffer);
       }
 
@@ -866,60 +927,123 @@ class PackageCrawler {
     }
   }
 
-  async extractNpmPackage(packageBuffer, source) {
-    try {
-      const files = {};
-      if (packageBuffer.length < 2 || packageBuffer[0] !== 0x1f || packageBuffer[1] !== 0x8b) {
-        const start = packageBuffer.subarray(0, 8).toString('hex');
-        throw new Error(`not a gzip file (it starts with ${start || 'nothing'})`);
-      }
-
-      // First decompress the gzip
-      const decompressed = zlib.gunzipSync(packageBuffer);
-
-      // Parse tar manually without any file system operations
-      let offset = 0;
-
-      while (offset < decompressed.length) {
-        // Read tar header (512 bytes)
-        if (offset + 512 > decompressed.length) break;
-
-        const header = decompressed.slice(offset, offset + 512);
-
-        // Check if this is the end (null header)
-        if (header[0] === 0) break;
-
-        // Extract filename (first 100 bytes, null-terminated)
-        let filename = '';
-        for (let i = 0; i < 100; i++) {
-          if (header[i] === 0) break;
-          filename += String.fromCharCode(header[i]);
+  // Reads what the crawler needs from a package as the package is unzipped and untarred:
+  // package.json, .index.json and ig.ini, the file names, and the url of each resource. Only
+  // those files are held, one at a time, so memory is bounded by maxFileMB however large
+  // the package unpacks to (see crawlerLimits)
+  async unpackPackage(packageBuffer) {
+    const limits = crawlerLimits(this.config);
+    const meta = {};
+    const fileNames = [];
+    const resourceUrls = [];
+    const skipped = [];
+
+    await new Promise((resolve, reject) => {
+      let unpacked = 0;
+      let failed = false;
+      const gunzip = zlib.createGunzip();
+      const fail = (e) => {
+        if (!failed) {
+          failed = true;
+          gunzip.destroy();
+          reject(e);
         }
+      };
 
-        // Extract file size (12 bytes starting at offset 124, octal)
-        let sizeStr = '';
-        for (let i = 124; i < 136; i++) {
-          if (header[i] === 0 || header[i] === 32) break; // null or space
-          sizeStr += String.fromCharCode(header[i]);
+      const parser = new TarParser({
+        onReadEntry: (entry) => {
+          if (failed || !['File', 'OldFile', 'ContiguousFile'].includes(entry.type)) {
+            entry.resume();
+            return;
+          }
+          const name = entry.path.replace(/^package\//, '');
+          fileNames.push(name);
+          if (fileNames.length > limits.maxFiles) {
+            fail(new Error(`the package has more than ${limits.maxFiles} files (maxFiles)`));
+            entry.resume();
+            return;
+          }
+          const isMeta = META_FILES.has(name);
+          if (!isMeta && !name.endsWith('.json')) {
+            entry.resume();
+            return;
+          }
+          if (entry.size > limits.maxFileBytes) {
+            if (name === 'package.json') {
+              fail(new Error(`package.json is ${Math.round(entry.size / MB)} MB, more than the limit of ${limits.maxFileBytes / MB} MB (maxFileMB)`));
+            } else {
+              skipped.push(name);
+            }
+            entry.resume();
+            return;
+          }
+          const chunks = [];
+          entry.on('data', (chunk) => chunks.push(chunk));
+          entry.on('end', () => {
+            if (failed) {
+              return;
+            }
+            const text = Buffer.concat(chunks).toString('utf8');
+            if (isMeta) {
+              meta[name] = text;
+            }
+            if (name.endsWith('.json') && name !== 'package.json' && name !== '.index.json') {
+              try {
+                const json = JSON.parse(text);
+                if (json.url && json.resourceType) {
+                  resourceUrls.push(json.url);
+                }
+              } catch (e) {
+                // not a resource
+              }
+            }
+          });
         }
-        const fileSize = parseInt(sizeStr, 8) || 0;
-
-        // Move past header
-        offset += 512;
-
-        // Extract file content if we need this file
-        if (fileSize > 0) {
-          const cleanFilename = filename.replace(/^package\//, ''); // Remove package/ prefix
+      });
+      parser.on('warn', (code, message) => {
+        if (code === 'TAR_BAD_ARCHIVE') {
+          fail(new Error(`not a tar file once unzipped (${message})`));
+        }
+      });
+      parser.on('error', fail);
+      parser.on('end', () => {
+        if (!failed) {
+          resolve();
+        }
+      });
 
-          const fileContent = decompressed.slice(offset, offset + fileSize);
-          files[cleanFilename] = fileContent.toString('utf8');
+      gunzip.on('data', (chunk) => {
+        if (failed) {
+          return;
+        }
+        unpacked += chunk.length;
+        if (unpacked > limits.maxUnpackedBytes) {
+          fail(new Error(`the package unzips to more than the limit of ${limits.maxUnpackedBytes / MB} MB (maxUnpackedMB)`));
+          return;
+        }
+        parser.write(chunk);
+      });
+      gunzip.on('end', () => {
+        if (!failed) {
+          parser.end();
         }
+      });
+      gunzip.on('error', (e) => fail(new Error(`could not unzip the package: ${e.message}`)));
+      gunzip.end(packageBuffer);
+    });
+
+    return { meta, fileNames, resourceUrls, skipped };
+  }
 
-        // Move to next file (files are padded to 512-byte boundaries)
-        const paddedSize = Math.ceil(fileSize / 512) * 512;
-        offset += paddedSize;
+  async extractNpmPackage(packageBuffer, source) {
+    try {
+      if (packageBuffer.length < 2 || packageBuffer[0] !== 0x1f || packageBuffer[1] !== 0x8b) {
+        const start = packageBuffer.subarray(0, 8).toString('hex');
+        throw new Error(`not a gzip file (it starts with ${start || 'nothing'})`);
       }
 
+      const { meta: files, fileNames, resourceUrls, skipped } = await this.unpackPackage(packageBuffer);
+
       // Parse package.json (required)
       if (!files['package.json']) {
         throw new Error('package.json not found in extracted package');
@@ -933,7 +1057,7 @@ class PackageCrawler {
           packageJson.scripts.postinstall
         )
       );
-      const hasJavaScript = Object.keys(files).some(f => f.endsWith('.js') || f.endsWith('.mjs') || f.endsWith('.cjs'));
+      const hasJavaScript = fileNames.some(f => f.endsWith('.js') || f.endsWith('.mjs') || f.endsWith('.cjs'));
 
       // Extract basic NPM fields
       const id = packageJson.name || '';
@@ -1025,6 +1149,10 @@ class PackageCrawler {
         }
       }
 
+      if (skipped.length > 0) {
+        this.log.warn(`Package ${id}#${version}: ${skipped.length} file(s) are more than the limit of ${crawlerLimits(this.config).maxFileBytes / MB} MB (maxFileMB), so were not read for their url: ${skipped.slice(0, 5).join(', ')}${skipped.length > 5 ? ', ...' : ''}`);
+      }
+
       // Default fhirVersion if not found
       if (!fhirVersion) {
         fhirVersion = '4.0.1'; // Default to R4
@@ -1046,7 +1174,8 @@ class PackageCrawler {
         hasInstallScripts,
         hasJavaScript,
         notForPublication,
-        files
+        fileNames,
+        resourceUrls
       };
 
     } catch (error) {
@@ -1103,8 +1232,9 @@ class PackageCrawler {
   }
 
   isValidSemVersion(version) {
-    // Simple semantic version validation
-    return /^\d+\.\d+\.\d+/.test(version);
+    // The whole version must be semver: it goes into the database, URLs and the mirror
+    // file name, so a version like 1.0.0/../../x must not get through
+    return /^\d+\.\d+\.\d+(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$/.test(version);
   }
 
   isAbsoluteUrl(url) {
@@ -1117,31 +1247,8 @@ class PackageCrawler {
   }
 
   processPackageUrls(npmPackage) {
-    const urls = [];
-
-    try {
-
-      for (const filename of Object.keys(npmPackage.files)) {
-        try {
-          const bytes = npmPackage.files[filename];
-          if (filename.endsWith('.json')) {
-            try {
-              const jsonContent = JSON.parse(bytes);
-
-              if (jsonContent.url && jsonContent.resourceType) {
-                urls.push(jsonContent.url);
-              }
-            } catch (fileError) {
-              // this.log.warn(`Error processing package file ${npmPackage.name}#${npmPackage.version}/package/${filename}: ${fileError.message}`);
-            }
-          }
-        } catch (fileError) {
-          this.log.warn(`Error processing package file ${npmPackage.name}#${npmPackage.version}/package/${filename}: ${fileError.message}`);
-        }
-      }
-    } catch (error) {
-      this.log.warn(`Error processing package URLs for ${npmPackage.name}#${npmPackage.version}:`, error.message);
-    }
+    // the resources' urls are read as the package is unpacked (unpackPackage)
+    const urls = [...(npmPackage.resourceUrls || [])];
 
     // Include main package URL
     if (npmPackage.url) {

package/packages/packages.js

@@ -944,11 +944,13 @@ class PackagesModule {
     }
   }
 
-  stopCrawlerJob() {
+  // node-cron 4: stop() may return a promise
+  async stopCrawlerJob() {
     if (this.crawlerJob) {
       this.crawler.shutdown();
-      this.crawlerJob.stop();
+      const job = this.crawlerJob;
       this.crawlerJob = null;
+      await job.stop();
       pckLog.info('Package crawler job stopped');
     }
   }
@@ -1024,15 +1026,16 @@ class PackagesModule {
     if (this.config.crawler.enabled) {
       pckLog.info('Starting initial package crawler...');
 
-      // Run crawler in background (non-blocking)
-      setImmediate(async () => {
+      // Run crawler in background (non-blocking); errors are logged inside
+      const crawl = async () => {
         try {
           await this.runCrawler();
           pckLog.info('Initial package crawler completed successfully');
         } catch (error) {
           pckLog.error('Initial package crawler failed:', error.message);
         }
-      });
+      };
+      setImmediate(() => void crawl());
     }
   }
 
@@ -1831,14 +1834,34 @@ class PackagesModule {
     res.send(htmlServer.renderPage('packages', 'npm Registry Check', content, stats));
   }
 
-  // GET /{id}/{version} is npm's version document when the request asks for JSON, and the
-  // tarball otherwise - which is what FHIR clients have always got there, without asking
-  // for JSON. A request carrying the TARBALL_HEADER always gets the tarball.
+  // GET /{id}/{version} is npm's version document when the request prefers JSON, and the
+  // tarball otherwise - which is what FHIR clients have always got there. Preferring JSON
+  // means ranking it above anything else the request accepts: a JSON type with a higher q
+  // than any */*, application/* or tarball type. Many clients send JSON together with */*
+  // at the same q - axios's default, application/json, text/plain, */*, which is what
+  // SUSHI (fhir-package-loader) sends when it downloads a tarball from here - and they get
+  // the tarball. A request carrying the TARBALL_HEADER always gets the tarball.
   wantsVersionDocument(req) {
     if (req.get(TARBALL_HEADER) !== undefined) {
       return false;
     }
-    return /application\/(json|vnd\.npm\.install-v1\+json)/i.test(req.get('accept') || '');
+    let json = 0;
+    let other = 0;
+    for (const part of String(req.get('accept') || '').split(',')) {
+      const [type, ...params] = part.split(';').map(s => s.trim().toLowerCase());
+      const qParam = params.find(p => p.startsWith('q='));
+      const q = qParam ? parseFloat(qParam.substring(2)) : 1;
+      if (!type || Number.isNaN(q)) {
+        continue;
+      }
+      if (type === 'application/json' || type === 'application/vnd.npm.install-v1+json') {
+        json = Math.max(json, q);
+      } else if (['*/*', 'application/*', 'application/tar+gzip', 'application/gzip', 'application/x-gzip',
+        'application/x-tar', 'application/octet-stream'].includes(type)) {
+        other = Math.max(other, q);
+      }
+    }
+    return json > 0 && json > other;
   }
 
   async serveVersionDocument(secure, id, version, req, res) {
@@ -2782,69 +2805,10 @@ class PackagesModule {
     return va.pre.localeCompare(vb.pre, undefined, { numeric: true });
   }
 
-  generateSearchHtml(req, results, params) {
-    // Simplified HTML generation - you'd want to use a proper template engine
-    const {name, dependson, canonicalPkg, canonicalUrl, fhirVersion} = params;
-    return `
-    <!DOCTYPE html>
-    <html>
-    <head>
-      <title>FHIR Package Search</title>
-      <style>
-        body { font-family: Arial, sans-serif; margin: 20px; }
-        .search-form { background: #f5f5f5; padding: 20px; margin-bottom: 20px; }
-        .search-form input, .search-form select { margin: 5px; padding: 5px; }
-        .results { margin-top: 20px; }
-        .package { border: 1px solid #ddd; margin: 10px 0; padding: 15px; }
-        .package-name { font-weight: bold; font-size: 1.2em; }
-        .package-details { color: #666; margin-top: 5px; }
-      </style>
-    </head>
-    <body>
-      <h1>FHIR Package Search</h1>
-      
-      <form class="search-form" method="GET">
-        <input type="text" name="name" placeholder="Package name" value="${escape(name)}">
-        <input type="text" name="dependson" placeholder="Depends on" value="${escape(dependson)}">
-        <input type="text" name="canonicalPkg" placeholder="Canonical package" value="${escape(canonicalPkg)}">
-        <input type="text" name="canonicalUrl" placeholder="Canonical URL" value="${escape(canonicalUrl)}">
-        <select name="fhirVersion">
-          <option value="">Any FHIR version</option>
-          <option value="R2" ${fhirVersion === 'R2' ? 'selected' : ''}>R2</option>
-          <option value="R3" ${fhirVersion === 'R3' ? 'selected' : ''}>R3</option>
-          <option value="R4" ${fhirVersion === 'R4' ? 'selected' : ''}>R4</option>
-          <option value="R5" ${fhirVersion === 'R5' ? 'selected' : ''}>R5</option>
-        </select>
-        <button type="submit">Search</button>
-      </form>
-      
-      <div class="results">
-        <h2>Results (${results.length} packages found)</h2>
-        ${results.map(pkg => `
-          <div class="package">
-            <div class="package-name">
-              <a href="${pkg.url}">${escape(pkg.name)}</a> v${escape(pkg.version)}
-            </div>
-            <div class="package-details">
-              <strong>FHIR Version:</strong> ${escape(pkg.fhirVersion)}<br>
-              <strong>Type:</strong> ${escape(pkg.kind)}<br>
-              <strong>Canonical:</strong> ${escape(pkg.canonical)}<br>
-              ${pkg.description ? `<strong>Description:</strong> ${escape(pkg.description)}<br>` : ''}
-              ${pkg.date ? `<strong>Published:</strong> ${new Date(pkg.date).toLocaleDateString()}<br>` : ''}
-              ${pkg.count ? `<strong>Downloads:</strong> ${pkg.count}<br>` : ''}
-            </div>
-          </div>
-        `).join('')}
-      </div>
-    </body>
-    </html>
-  `;
-  }
-
   async shutdown() {
     pckLog.info('Shutting down Packages module...');
 
-    this.stopCrawlerJob();
+    await this.stopCrawlerJob();
 
     // Close database connection
     if (this.db) {

package/packages/readme.md

@@ -31,7 +31,7 @@ In summary:
 | `GET /packages/catalog` | Search packages: `name`, `canonical`, `pkgcanonical`, `fhirversion`, `dependency`, `dependson`, `sort` |
 | `GET /packages/-/v1/search` | npm search API: `text`, `size`, `from`, plus the `/catalog` parameters |
 | `GET /packages/{id}` | npm package document: all versions, dependencies, tarball URLs |
-| `GET /packages/{id}/{version}` | Download a package tarball; or, when the request's `Accept` asks for JSON, npm's document for that version. A `FHIR-Package-Download` header always gets the tarball (the HL7 Java tools send it) |
+| `GET /packages/{id}/{version}` | Download a package tarball; or, when the request's `Accept` prefers JSON (ranks it above `*/*`), npm's document for that version. A `FHIR-Package-Download` header always gets the tarball (the HL7 Java tools send it) |
 | `GET /packages/updates` | Versions published since a date |
 | `GET /packages/broken` | Package versions whose dependencies this registry doesn't hold |
 | `GET /packages/npm-check` | Check any registry against the npm registry protocol (a form, in a browser) |

package/packages/utilities/dedupe-versions.js

@@ -9,7 +9,7 @@
  * One-off repair of a package server's database and mirror, for a database built before
  * the crawler stored each id#version only once.
  *
- *   node packages/utilities/dedupe-versions.js -db <packages.db> -mirror <mirror folder> [-apply] [-verbose]
+ *   node packages/utilities/dedupe-versions.js -db <packages.db> -mirror <mirror folder> [-apply] [-verbose] [-keep <GUID>]...
  *
  * Without -apply it reports what it would do and changes nothing. Back up the database
  * first. It is safe to run while the server is up, but best between crawler runs. It can be
@@ -19,7 +19,8 @@
  *    crawler's rule: a direct copy beats a Simplifier copy (a secondary echo); otherwise the
  *    first stored wins. Two direct copies with different content are a conflict - only their
  *    Simplifier rows are removed, and they are listed for manual cleanup. Deleted rows'
- *    download counts are added to the kept row.
+ *    download counts are added to the kept row. To settle a conflict, run it again with
+ *    -keep and the GUID (the URL it lists) of the copy to keep, once per conflict.
  * 2. The unique index on (Id, Version), once there are no duplicates left.
  * 3. The mirror. Every row's mirror file must be the bytes that row's Hash describes; a file
  *    that is missing or different is rewritten from the row's Content. Conflicts are skipped.
@@ -59,8 +60,13 @@ function keepOwnership(file, like) {
 }
 const sha1 = buf => crypto.createHash('sha1').update(buf).digest('hex');
 
-// Which rows of one id#version to keep and delete. rows are in PackageVersionKey order.
-function plan(rows) {
+// Which rows of one id#version to keep and delete. rows are in PackageVersionKey order;
+// keepGuids are the copies chosen (-keep) to settle conflicts.
+function plan(rows, keepGuids = new Set()) {
+  const chosen = rows.filter(r => keepGuids.has(r.GUID));
+  if (chosen.length === 1) {
+    return { keep: chosen, conflict: false };
+  }
   const direct = rows.filter(r => !isSimplifier(r.GUID));
   if (direct.length === 0) {
     return { keep: [rows[0]], conflict: false };
@@ -71,7 +77,8 @@ function plan(rows) {
   return { keep: direct, conflict: true };
 }
 
-function dedupe({ dbPath, mirrorPath, apply = false, verbose = false, log = console.log }) {
+function dedupe({ dbPath, mirrorPath, apply = false, verbose = false, keep = [], log = console.log }) {
+  const keepGuids = new Set(keep);
   if (!fs.existsSync(dbPath)) {
     throw new Error(`No database at ${dbPath}`); // DatabaseSync would create an empty one
   }
@@ -87,7 +94,7 @@ function dedupe({ dbPath, mirrorPath, apply = false, verbose = false, log = cons
     const changes = [];
     for (const g of groups) {
       const rows = rowsOf.all(g.Id, g.Version);
-      const { keep, conflict } = plan(rows);
+      const { keep, conflict } = plan(rows, keepGuids);
       const del = rows.filter(r => !keep.includes(r));
       if (conflict) {
         report.conflicts.push({ id: g.Id, version: g.Version, kept: keep.map(r => r.GUID) });
@@ -100,6 +107,12 @@ function dedupe({ dbPath, mirrorPath, apply = false, verbose = false, log = cons
       }
     }
     report.groups = groups.length;
+    const used = new Set(changes.flatMap(c => [c.keep.GUID]));
+    for (const g of keepGuids) {
+      if (!used.has(g)) {
+        log(`-keep ${g}: not a copy of any duplicated id#version, so ignored`);
+      }
+    }
     const toDelete = new Set(changes.flatMap(c => c.del.map(r => r.PackageVersionKey)));
     const conflicted = new Set(report.conflicts.map(c => `${c.id}#${c.version}`));
 
@@ -219,15 +232,16 @@ function main(argv) {
   const dbPath = arg('-db');
   const mirrorPath = arg('-mirror');
   if (!dbPath) {
-    console.error('usage: node packages/utilities/dedupe-versions.js -db <packages.db> [-mirror <mirror folder>] [-apply] [-verbose]');
+    console.error('usage: node packages/utilities/dedupe-versions.js -db <packages.db> [-mirror <mirror folder>] [-apply] [-verbose] [-keep <GUID>]...');
     return 1;
   }
   const apply = argv.includes('-apply');
+  const keep = argv.flatMap((a, i) => a === '-keep' && argv[i + 1] ? [argv[i + 1]] : []);
   if (!apply) {
     console.log('Dry run: nothing will be changed (use -apply)');
   }
   try {
-    dedupe({ dbPath, mirrorPath, apply, verbose: argv.includes('-verbose') });
+    dedupe({ dbPath, mirrorPath, apply, keep, verbose: argv.includes('-verbose') });
   } catch (e) {
     console.error(e.message);
     return 1;

package/registry/crawler.js

@@ -42,7 +42,9 @@ class RegistryCrawler {
       userAgent: config.userAgent || 'HealthIntersections/FhirServer',
       crawlInterval: config.crawlInterval || 5 * 60 * 1000, // 5 minutes default
       apiKeys: config.apiKeys || {}, // Map of server URL or code to API key
-      allowPrivateAddresses: config.allowPrivateAddresses || false // SSRF opt-out for local test servers
+      allowPrivateAddresses: config.allowPrivateAddresses || false, // SSRF opt-out for local test servers
+      // the most a server's response can be, once unzipped: the servers are other people's
+      maxResponseBytes: (config.maxResponseMB || 100) * 1024 * 1024
     };
     this.stats = stats;
 
@@ -588,6 +590,8 @@ class RegistryCrawler {
         headers: headers,
         signal: this.abortController?.signal,
         validateStatus: (status) => status < 500, // Don't throw on 4xx
+        // counted as it is unzipped, so a small response that unzips to a lot is stopped too
+        maxContentLength: this.config.maxResponseBytes,
         httpAgent: this.guardedAgents().httpAgent,
         httpsAgent: this.guardedAgents().httpsAgent
       });
@@ -608,6 +612,9 @@ class RegistryCrawler {
       
     } catch (error) {
       debugLog(error);
+      if (/maxContentLength/.test(error.message)) {
+        throw new Error(`The response from ${url} is more than ${this.config.maxResponseBytes / (1024 * 1024)} MB`);
+      }
       if (error.response) {
         throw new Error(`HTTP ${error.response.status}: ${error.response.statusText}`);
       } else if (error.request) {