fhirsmith 0.16.1 → 0.18.0

package/packages/package-crawler.js

@@ -16,6 +16,33 @@ const http = require('http');
 const https = require('https');
 const ipaddr = require('ipaddr.js');
 const {debugLog} = require("../tx/operation-context");
+const {Parser: TarParser} = require('tar');
+
+const MB = 1024 * 1024;
+
+// Limits on what a package can make the crawler download, unpack and hold in memory, so
+// that a hostile or broken package (a decompression bomb, say) costs a bounded amount.
+// All can be set in the packages "crawler" config section:
+//   maxDownloadMB  the package file as downloaded (or read from a local feed)
+//   maxUnpackedMB  the total of the package's files once unzipped. The files are streamed,
+//                  not held, so this limits the work, not the memory
+//   maxFileMB      a file the crawler reads: package.json, .index.json, ig.ini, and each
+//                  .json file (for its url). Memory is bounded by this, not by the package
+//   maxFiles       the number of files in the package
+const LIMIT_DEFAULTS = { maxDownloadMB: 250, maxUnpackedMB: 4096, maxFileMB: 50, maxFiles: 250000 };
+
+function crawlerLimits(config) {
+  const c = { ...LIMIT_DEFAULTS, ...(config?.crawler || {}) };
+  return {
+    maxDownloadBytes: c.maxDownloadMB * MB,
+    maxUnpackedBytes: c.maxUnpackedMB * MB,
+    maxFileBytes: c.maxFileMB * MB,
+    maxFiles: c.maxFiles
+  };
+}
+
+// The files in a package that the crawler reads for its metadata
+const META_FILES = new Set(['package.json', '.index.json', 'ig.ini']);
 
 // True if an IP literal is anything other than a normal public (unicast) address:
 // loopback, private, link-local (incl. 169.254.169.254 cloud metadata), unique-local,
@@ -350,8 +377,14 @@ class PackageCrawler {
 
   async fetchUrl(url) {
     try {
+      const limits = crawlerLimits(this.config);
       if (url.startsWith("/")) {
-        const buffer = await fs.promises.readFile(this.resolveLocalReadPath(url));
+        const localPath = this.resolveLocalReadPath(url);
+        const size = (await fs.promises.stat(localPath)).size;
+        if (size > limits.maxDownloadBytes) {
+          throw new Error(`the package is ${Math.round(size / MB)} MB, more than the limit of ${limits.maxDownloadBytes / MB} MB (maxDownloadMB)`);
+        }
+        const buffer = await fs.promises.readFile(localPath);
         this.totalBytes += buffer.byteLength;
         return buffer;
       } else {
@@ -360,6 +393,7 @@ class PackageCrawler {
           responseType: 'arraybuffer',
           // We want the package file's own bytes: decodePackageBody undoes the Content-Encoding
           decompress: false,
+          maxContentLength: limits.maxDownloadBytes,
           signal: this.abortController?.signal,
           headers: {
             'User-Agent': 'FHIR Package Crawler/1.0'
@@ -373,6 +407,9 @@ class PackageCrawler {
       }
     } catch (error) {
       debugLog(`Failed to fetch ${url}: ${error.message}`);
+      if (/maxContentLength/.test(error.message)) {
+        throw new Error(`Failed to fetch ${url}: the package is more than the limit of ${crawlerLimits(this.config).maxDownloadBytes / MB} MB (maxDownloadMB)`);
+      }
       if (error.response && error.response.status === 429) {
         throw new Error(`RATE_LIMITED: Server returned 429 Too Many Requests for ${url}`);
       }
@@ -387,21 +424,41 @@ class PackageCrawler {
   // package unless unzipping it gives another gzip file - a server that really did compress
   // a .tgz for transfer.
   decodePackageBody(body, contentEncoding) {
+    // undoing the transfer encoding gives the package file, so the download limit applies
+    const max = { maxOutputLength: crawlerLimits(this.config).maxDownloadBytes };
+    const tooBig = (e) => e.code === 'ERR_BUFFER_TOO_LARGE'
+      ? new Error(`the package is more than the limit of ${max.maxOutputLength / MB} MB (maxDownloadMB) once its Content-Encoding is undone`) : e;
     const encodings = String(contentEncoding || '').toLowerCase().split(',').map(e => e.trim()).filter(e => e && e !== 'identity');
     for (const enc of encodings.reverse()) {
       if (enc === 'gzip' || enc === 'x-gzip') {
+        let start;
         try {
-          const inner = zlib.gunzipSync(body);
-          if (inner[0] === 0x1f && inner[1] === 0x8b) {
-            body = inner;
-          }
+          // only the start is needed to see whether it is gzip inside: a tar unzipped
+          // from a .tgz can be far larger than the download limit
+          start = zlib.gunzipSync(body.subarray(0, 4096), { finishFlush: zlib.constants.Z_SYNC_FLUSH });
         } catch (e) {
           // not gzip after all: leave it for extractNpmPackage to report
+          continue;
+        }
+        if (start[0] === 0x1f && start[1] === 0x8b) {
+          try {
+            body = zlib.gunzipSync(body, max);
+          } catch (e) {
+            throw tooBig(e);
+          }
         }
       } else if (enc === 'br') {
-        body = zlib.brotliDecompressSync(body);
+        try {
+          body = zlib.brotliDecompressSync(body, max);
+        } catch (e) {
+          throw tooBig(e);
+        }
       } else if (enc === 'deflate') {
-        body = zlib.inflateSync(body);
+        try {
+          body = zlib.inflateSync(body, max);
+        } catch (e) {
+          throw tooBig(e);
+        }
       } else {
         throw new Error(`Unsupported Content-Encoding ${enc}`);
       }
@@ -854,6 +911,10 @@ class PackageCrawler {
         let fid = this.fixPrefix(id);
         const filename = `${fid}-${version}.tgz`;
         const filepath = path.join(this.config.mirrorPath, filename);
+        // id and version are validated, but the file must land in the mirror whatever they say
+        if (path.dirname(path.resolve(filepath)) !== path.resolve(this.config.mirrorPath)) {
+          throw new Error(`Package ${id}#${version} rejected: its mirror file name is not a plain file name`);
+        }
         fs.writeFileSync(filepath, packageBuffer);
       }
 
@@ -866,60 +927,123 @@ class PackageCrawler {
     }
   }
 
-  async extractNpmPackage(packageBuffer, source) {
-    try {
-      const files = {};
-      if (packageBuffer.length < 2 || packageBuffer[0] !== 0x1f || packageBuffer[1] !== 0x8b) {
-        const start = packageBuffer.subarray(0, 8).toString('hex');
-        throw new Error(`not a gzip file (it starts with ${start || 'nothing'})`);
-      }
-
-      // First decompress the gzip
-      const decompressed = zlib.gunzipSync(packageBuffer);
-
-      // Parse tar manually without any file system operations
-      let offset = 0;
-
-      while (offset < decompressed.length) {
-        // Read tar header (512 bytes)
-        if (offset + 512 > decompressed.length) break;
-
-        const header = decompressed.slice(offset, offset + 512);
-
-        // Check if this is the end (null header)
-        if (header[0] === 0) break;
-
-        // Extract filename (first 100 bytes, null-terminated)
-        let filename = '';
-        for (let i = 0; i < 100; i++) {
-          if (header[i] === 0) break;
-          filename += String.fromCharCode(header[i]);
+  // Reads what the crawler needs from a package as the package is unzipped and untarred:
+  // package.json, .index.json and ig.ini, the file names, and the url of each resource. Only
+  // those files are held, one at a time, so memory is bounded by maxFileMB however large
+  // the package unpacks to (see crawlerLimits)
+  async unpackPackage(packageBuffer) {
+    const limits = crawlerLimits(this.config);
+    const meta = {};
+    const fileNames = [];
+    const resourceUrls = [];
+    const skipped = [];
+
+    await new Promise((resolve, reject) => {
+      let unpacked = 0;
+      let failed = false;
+      const gunzip = zlib.createGunzip();
+      const fail = (e) => {
+        if (!failed) {
+          failed = true;
+          gunzip.destroy();
+          reject(e);
         }
+      };
 
-        // Extract file size (12 bytes starting at offset 124, octal)
-        let sizeStr = '';
-        for (let i = 124; i < 136; i++) {
-          if (header[i] === 0 || header[i] === 32) break; // null or space
-          sizeStr += String.fromCharCode(header[i]);
+      const parser = new TarParser({
+        onReadEntry: (entry) => {
+          if (failed || !['File', 'OldFile', 'ContiguousFile'].includes(entry.type)) {
+            entry.resume();
+            return;
+          }
+          const name = entry.path.replace(/^package\//, '');
+          fileNames.push(name);
+          if (fileNames.length > limits.maxFiles) {
+            fail(new Error(`the package has more than ${limits.maxFiles} files (maxFiles)`));
+            entry.resume();
+            return;
+          }
+          const isMeta = META_FILES.has(name);
+          if (!isMeta && !name.endsWith('.json')) {
+            entry.resume();
+            return;
+          }
+          if (entry.size > limits.maxFileBytes) {
+            if (name === 'package.json') {
+              fail(new Error(`package.json is ${Math.round(entry.size / MB)} MB, more than the limit of ${limits.maxFileBytes / MB} MB (maxFileMB)`));
+            } else {
+              skipped.push(name);
+            }
+            entry.resume();
+            return;
+          }
+          const chunks = [];
+          entry.on('data', (chunk) => chunks.push(chunk));
+          entry.on('end', () => {
+            if (failed) {
+              return;
+            }
+            const text = Buffer.concat(chunks).toString('utf8');
+            if (isMeta) {
+              meta[name] = text;
+            }
+            if (name.endsWith('.json') && name !== 'package.json' && name !== '.index.json') {
+              try {
+                const json = JSON.parse(text);
+                if (json.url && json.resourceType) {
+                  resourceUrls.push(json.url);
+                }
+              } catch (e) {
+                // not a resource
+              }
+            }
+          });
         }
-        const fileSize = parseInt(sizeStr, 8) || 0;
-
-        // Move past header
-        offset += 512;
-
-        // Extract file content if we need this file
-        if (fileSize > 0) {
-          const cleanFilename = filename.replace(/^package\//, ''); // Remove package/ prefix
+      });
+      parser.on('warn', (code, message) => {
+        if (code === 'TAR_BAD_ARCHIVE') {
+          fail(new Error(`not a tar file once unzipped (${message})`));
+        }
+      });
+      parser.on('error', fail);
+      parser.on('end', () => {
+        if (!failed) {
+          resolve();
+        }
+      });
 
-          const fileContent = decompressed.slice(offset, offset + fileSize);
-          files[cleanFilename] = fileContent.toString('utf8');
+      gunzip.on('data', (chunk) => {
+        if (failed) {
+          return;
+        }
+        unpacked += chunk.length;
+        if (unpacked > limits.maxUnpackedBytes) {
+          fail(new Error(`the package unzips to more than the limit of ${limits.maxUnpackedBytes / MB} MB (maxUnpackedMB)`));
+          return;
+        }
+        parser.write(chunk);
+      });
+      gunzip.on('end', () => {
+        if (!failed) {
+          parser.end();
         }
+      });
+      gunzip.on('error', (e) => fail(new Error(`could not unzip the package: ${e.message}`)));
+      gunzip.end(packageBuffer);
+    });
+
+    return { meta, fileNames, resourceUrls, skipped };
+  }
 
-        // Move to next file (files are padded to 512-byte boundaries)
-        const paddedSize = Math.ceil(fileSize / 512) * 512;
-        offset += paddedSize;
+  async extractNpmPackage(packageBuffer, source) {
+    try {
+      if (packageBuffer.length < 2 || packageBuffer[0] !== 0x1f || packageBuffer[1] !== 0x8b) {
+        const start = packageBuffer.subarray(0, 8).toString('hex');
+        throw new Error(`not a gzip file (it starts with ${start || 'nothing'})`);
       }
 
+      const { meta: files, fileNames, resourceUrls, skipped } = await this.unpackPackage(packageBuffer);
+
       // Parse package.json (required)
       if (!files['package.json']) {
         throw new Error('package.json not found in extracted package');
@@ -933,7 +1057,7 @@ class PackageCrawler {
           packageJson.scripts.postinstall
         )
       );
-      const hasJavaScript = Object.keys(files).some(f => f.endsWith('.js') || f.endsWith('.mjs') || f.endsWith('.cjs'));
+      const hasJavaScript = fileNames.some(f => f.endsWith('.js') || f.endsWith('.mjs') || f.endsWith('.cjs'));
 
       // Extract basic NPM fields
       const id = packageJson.name || '';
@@ -1025,6 +1149,10 @@ class PackageCrawler {
         }
       }
 
+      if (skipped.length > 0) {
+        this.log.warn(`Package ${id}#${version}: ${skipped.length} file(s) are more than the limit of ${crawlerLimits(this.config).maxFileBytes / MB} MB (maxFileMB), so were not read for their url: ${skipped.slice(0, 5).join(', ')}${skipped.length > 5 ? ', ...' : ''}`);
+      }
+
       // Default fhirVersion if not found
       if (!fhirVersion) {
         fhirVersion = '4.0.1'; // Default to R4
@@ -1046,7 +1174,8 @@ class PackageCrawler {
         hasInstallScripts,
         hasJavaScript,
         notForPublication,
-        files
+        fileNames,
+        resourceUrls
       };
 
     } catch (error) {
@@ -1103,8 +1232,9 @@ class PackageCrawler {
   }
 
   isValidSemVersion(version) {
-    // Simple semantic version validation
-    return /^\d+\.\d+\.\d+/.test(version);
+    // The whole version must be semver: it goes into the database, URLs and the mirror
+    // file name, so a version like 1.0.0/../../x must not get through
+    return /^\d+\.\d+\.\d+(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$/.test(version);
   }
 
   isAbsoluteUrl(url) {
@@ -1117,31 +1247,8 @@ class PackageCrawler {
   }
 
   processPackageUrls(npmPackage) {
-    const urls = [];
-
-    try {
-
-      for (const filename of Object.keys(npmPackage.files)) {
-        try {
-          const bytes = npmPackage.files[filename];
-          if (filename.endsWith('.json')) {
-            try {
-              const jsonContent = JSON.parse(bytes);
-
-              if (jsonContent.url && jsonContent.resourceType) {
-                urls.push(jsonContent.url);
-              }
-            } catch (fileError) {
-              // this.log.warn(`Error processing package file ${npmPackage.name}#${npmPackage.version}/package/${filename}: ${fileError.message}`);
-            }
-          }
-        } catch (fileError) {
-          this.log.warn(`Error processing package file ${npmPackage.name}#${npmPackage.version}/package/${filename}: ${fileError.message}`);
-        }
-      }
-    } catch (error) {
-      this.log.warn(`Error processing package URLs for ${npmPackage.name}#${npmPackage.version}:`, error.message);
-    }
+    // the resources' urls are read as the package is unpacked (unpackPackage)
+    const urls = [...(npmPackage.resourceUrls || [])];
 
     // Include main package URL
     if (npmPackage.url) {

package/packages/packages.js

@@ -1026,15 +1026,16 @@ class PackagesModule {
     if (this.config.crawler.enabled) {
       pckLog.info('Starting initial package crawler...');
 
-      // Run crawler in background (non-blocking)
-      setImmediate(async () => {
+      // Run crawler in background (non-blocking); errors are logged inside
+      const crawl = async () => {
         try {
           await this.runCrawler();
           pckLog.info('Initial package crawler completed successfully');
         } catch (error) {
           pckLog.error('Initial package crawler failed:', error.message);
         }
-      });
+      };
+      setImmediate(() => void crawl());
     }
   }
 
@@ -1281,11 +1282,15 @@ class PackagesModule {
 
         next();
         pckLog.info(`/download/${id}/${version}`);
-      } finally {
+      } catch (error) {
         this.stats.countRequest('version', Date.now() - start);
+        throw error;
       }
     }, async (req, res) => {
       const start = Date.now();
+      // counted as 'download' (the tarball) or 'version' (its JSON document) - the
+      // daily report counts the downloads as packages served
+      let name = 'version';
       try {
         try {
           const {id, version} = req.params;
@@ -1296,6 +1301,7 @@ class PackagesModule {
             await this.serveVersionDocument(req.secure, id, version, req, res);
             pckLog.info(`/version/${id}/${version}`);
           } else {
+            name = 'download';
             await this.serveDownload(req.secure, id, version, res);
             pckLog.info(`/download/${id}/${version}`);
           }
@@ -1304,7 +1310,7 @@ class PackagesModule {
           res.status(500).json({error: 'Internal server error'});
         }
       } finally {
-        this.stats.countRequest('version', Date.now() - start);
+        this.stats.countRequest(name, Date.now() - start);
       }
     });
 
@@ -1321,8 +1327,9 @@ class PackagesModule {
 
         next();
         pckLog.info(`/page/${page}`);
-      } finally {
+      } catch (error) {
         this.stats.countRequest('page', Date.now() - start);
+        throw error;
       }
     }, async (req, res) => {
       const start = Date.now();
@@ -2804,63 +2811,29 @@ class PackagesModule {
     return va.pre.localeCompare(vb.pre, undefined, { numeric: true });
   }
 
-  generateSearchHtml(req, results, params) {
-    // Simplified HTML generation - you'd want to use a proper template engine
-    const {name, dependson, canonicalPkg, canonicalUrl, fhirVersion} = params;
-    return `
-    <!DOCTYPE html>
-    <html>
-    <head>
-      <title>FHIR Package Search</title>
-      <style>
-        body { font-family: Arial, sans-serif; margin: 20px; }
-        .search-form { background: #f5f5f5; padding: 20px; margin-bottom: 20px; }
-        .search-form input, .search-form select { margin: 5px; padding: 5px; }
-        .results { margin-top: 20px; }
-        .package { border: 1px solid #ddd; margin: 10px 0; padding: 15px; }
-        .package-name { font-weight: bold; font-size: 1.2em; }
-        .package-details { color: #666; margin-top: 5px; }
-      </style>
-    </head>
-    <body>
-      <h1>FHIR Package Search</h1>
-      
-      <form class="search-form" method="GET">
-        <input type="text" name="name" placeholder="Package name" value="${escape(name)}">
-        <input type="text" name="dependson" placeholder="Depends on" value="${escape(dependson)}">
-        <input type="text" name="canonicalPkg" placeholder="Canonical package" value="${escape(canonicalPkg)}">
-        <input type="text" name="canonicalUrl" placeholder="Canonical URL" value="${escape(canonicalUrl)}">
-        <select name="fhirVersion">
-          <option value="">Any FHIR version</option>
-          <option value="R2" ${fhirVersion === 'R2' ? 'selected' : ''}>R2</option>
-          <option value="R3" ${fhirVersion === 'R3' ? 'selected' : ''}>R3</option>
-          <option value="R4" ${fhirVersion === 'R4' ? 'selected' : ''}>R4</option>
-          <option value="R5" ${fhirVersion === 'R5' ? 'selected' : ''}>R5</option>
-        </select>
-        <button type="submit">Search</button>
-      </form>
-      
-      <div class="results">
-        <h2>Results (${results.length} packages found)</h2>
-        ${results.map(pkg => `
-          <div class="package">
-            <div class="package-name">
-              <a href="${pkg.url}">${escape(pkg.name)}</a> v${escape(pkg.version)}
-            </div>
-            <div class="package-details">
-              <strong>FHIR Version:</strong> ${escape(pkg.fhirVersion)}<br>
-              <strong>Type:</strong> ${escape(pkg.kind)}<br>
-              <strong>Canonical:</strong> ${escape(pkg.canonical)}<br>
-              ${pkg.description ? `<strong>Description:</strong> ${escape(pkg.description)}<br>` : ''}
-              ${pkg.date ? `<strong>Published:</strong> ${new Date(pkg.date).toLocaleDateString()}<br>` : ''}
-              ${pkg.count ? `<strong>Downloads:</strong> ${pkg.count}<br>` : ''}
-            </div>
-          </div>
-        `).join('')}
-      </div>
-    </body>
-    </html>
-  `;
+  /**
+   * The paragraph for the server's daily report: packages served (tarball
+   * downloads) and package versions newly indexed by the crawler in the period.
+   */
+  async dailyReport({since, now}) {
+    const {plural} = require('../library/daily-report');
+    const counts = this.stats.countsSince(since, now);
+    const downloads = counts.operations.download || 0;
+    const sentences = [`${plural(downloads, 'package')} ${downloads === 1 ? 'was' : 'were'} served (${plural(counts.total, 'request')} in all).`];
+    if (this.db) {
+      try {
+        const row = await new Promise((resolve, reject) => this.db.get(
+          'SELECT COUNT(*) AS n, COUNT(DISTINCT Id) AS ids FROM PackageVersions WHERE Indexed > ?',
+          [new Date(since).toISOString().replace('T', ' ').substring(0, 19)],
+          (err, row) => err ? reject(err) : resolve(row)));
+        sentences.push(row.n === 0
+          ? 'No new package versions were found.'
+          : `${plural(row.n, 'new package version')} ${row.n === 1 ? 'was' : 'were'} found, of ${plural(row.ids, 'package')}.`);
+      } catch (e) {
+        sentences.push(`The new packages could not be counted: ${e.message}.`);
+      }
+    }
+    return sentences.join(' ');
   }
 
   async shutdown() {

package/packages/utilities/dedupe-versions.js

@@ -9,7 +9,7 @@
  * One-off repair of a package server's database and mirror, for a database built before
  * the crawler stored each id#version only once.
  *
- *   node packages/utilities/dedupe-versions.js -db <packages.db> -mirror <mirror folder> [-apply] [-verbose]
+ *   node packages/utilities/dedupe-versions.js -db <packages.db> -mirror <mirror folder> [-apply] [-verbose] [-keep <GUID>]...
  *
  * Without -apply it reports what it would do and changes nothing. Back up the database
  * first. It is safe to run while the server is up, but best between crawler runs. It can be
@@ -19,7 +19,8 @@
  *    crawler's rule: a direct copy beats a Simplifier copy (a secondary echo); otherwise the
  *    first stored wins. Two direct copies with different content are a conflict - only their
  *    Simplifier rows are removed, and they are listed for manual cleanup. Deleted rows'
- *    download counts are added to the kept row.
+ *    download counts are added to the kept row. To settle a conflict, run it again with
+ *    -keep and the GUID (the URL it lists) of the copy to keep, once per conflict.
  * 2. The unique index on (Id, Version), once there are no duplicates left.
  * 3. The mirror. Every row's mirror file must be the bytes that row's Hash describes; a file
  *    that is missing or different is rewritten from the row's Content. Conflicts are skipped.
@@ -59,8 +60,13 @@ function keepOwnership(file, like) {
 }
 const sha1 = buf => crypto.createHash('sha1').update(buf).digest('hex');
 
-// Which rows of one id#version to keep and delete. rows are in PackageVersionKey order.
-function plan(rows) {
+// Which rows of one id#version to keep and delete. rows are in PackageVersionKey order;
+// keepGuids are the copies chosen (-keep) to settle conflicts.
+function plan(rows, keepGuids = new Set()) {
+  const chosen = rows.filter(r => keepGuids.has(r.GUID));
+  if (chosen.length === 1) {
+    return { keep: chosen, conflict: false };
+  }
   const direct = rows.filter(r => !isSimplifier(r.GUID));
   if (direct.length === 0) {
     return { keep: [rows[0]], conflict: false };
@@ -71,7 +77,8 @@ function plan(rows) {
   return { keep: direct, conflict: true };
 }
 
-function dedupe({ dbPath, mirrorPath, apply = false, verbose = false, log = console.log }) {
+function dedupe({ dbPath, mirrorPath, apply = false, verbose = false, keep = [], log = console.log }) {
+  const keepGuids = new Set(keep);
   if (!fs.existsSync(dbPath)) {
     throw new Error(`No database at ${dbPath}`); // DatabaseSync would create an empty one
   }
@@ -87,7 +94,7 @@ function dedupe({ dbPath, mirrorPath, apply = false, verbose = false, log = cons
     const changes = [];
     for (const g of groups) {
       const rows = rowsOf.all(g.Id, g.Version);
-      const { keep, conflict } = plan(rows);
+      const { keep, conflict } = plan(rows, keepGuids);
       const del = rows.filter(r => !keep.includes(r));
       if (conflict) {
         report.conflicts.push({ id: g.Id, version: g.Version, kept: keep.map(r => r.GUID) });
@@ -100,6 +107,12 @@ function dedupe({ dbPath, mirrorPath, apply = false, verbose = false, log = cons
       }
     }
     report.groups = groups.length;
+    const used = new Set(changes.flatMap(c => [c.keep.GUID]));
+    for (const g of keepGuids) {
+      if (!used.has(g)) {
+        log(`-keep ${g}: not a copy of any duplicated id#version, so ignored`);
+      }
+    }
     const toDelete = new Set(changes.flatMap(c => c.del.map(r => r.PackageVersionKey)));
     const conflicted = new Set(report.conflicts.map(c => `${c.id}#${c.version}`));
 
@@ -219,15 +232,16 @@ function main(argv) {
   const dbPath = arg('-db');
   const mirrorPath = arg('-mirror');
   if (!dbPath) {
-    console.error('usage: node packages/utilities/dedupe-versions.js -db <packages.db> [-mirror <mirror folder>] [-apply] [-verbose]');
+    console.error('usage: node packages/utilities/dedupe-versions.js -db <packages.db> [-mirror <mirror folder>] [-apply] [-verbose] [-keep <GUID>]...');
     return 1;
   }
   const apply = argv.includes('-apply');
+  const keep = argv.flatMap((a, i) => a === '-keep' && argv[i + 1] ? [argv[i + 1]] : []);
   if (!apply) {
     console.log('Dry run: nothing will be changed (use -apply)');
   }
   try {
-    dedupe({ dbPath, mirrorPath, apply, verbose: argv.includes('-verbose') });
+    dedupe({ dbPath, mirrorPath, apply, keep, verbose: argv.includes('-verbose') });
   } catch (e) {
     console.error(e.message);
     return 1;

package/publisher/publisher.js

@@ -2987,6 +2987,35 @@ class PublisherModule {
     });
   }
 
+  /**
+   * The paragraph for the server's daily report: publication runs started in
+   * the period, and how many finished successfully.
+   */
+  async dailyReport({since, now}) {
+    const {plural} = require('../library/daily-report');
+    if (!this.db) {
+      return 'The database is not open.';
+    }
+    const get = (sql, params) => new Promise((resolve, reject) =>
+      this.db.get(sql, params, (err, row) => err ? reject(err) : resolve(row)));
+    // the timestamps are CURRENT_TIMESTAMP: UTC, 'YYYY-MM-DD HH:MM:SS'
+    const utc = ms => new Date(ms).toISOString().replace('T', ' ').substring(0, 19);
+    try {
+      const started = await get('SELECT COUNT(*) AS n FROM tasks WHERE building_at > ? AND building_at <= ?', [utc(since), utc(now)]);
+      const finished = await get('SELECT COUNT(*) AS n FROM tasks WHERE completed_at > ? AND completed_at <= ?', [utc(since), utc(now)]);
+      const failed = await get('SELECT COUNT(*) AS n FROM tasks WHERE failed_at > ? AND failed_at <= ?', [utc(since), utc(now)]);
+      const waiting = await get("SELECT COUNT(*) AS n FROM tasks WHERE status IN ('queued', 'waiting for approval')", []);
+      let text = `${plural(started.n, 'publication run')} ${started.n === 1 ? 'was' : 'were'} started, and ${finished.n} finished successfully`;
+      text += failed.n > 0 ? ` (${failed.n} failed).` : '.';
+      if (waiting.n > 0) {
+        text += ` ${plural(waiting.n, 'task')} ${waiting.n === 1 ? 'is' : 'are'} queued or waiting for approval.`;
+      }
+      return text;
+    } catch (e) {
+      return `The database could not be read: ${e.message}.`;
+    }
+  }
+
   async shutdown() {
     this.logger.info('Shutting down publisher module...');