fhirsmith 0.12.0 → 0.13.1

package/library/languages.js

@@ -62,7 +62,21 @@ class LanguageVariant extends LanguageEntry {}
  * Individual language representation based on BCP 47
  */
 class Language {
-  constructor(code = '', languageDefinitions = null) {
+  /**
+   * @param {string} code - the language tag
+   * @param {LanguageDefinitions} [languageDefinitions] - registry to validate against; without
+   *   one the tag is only broken into parts, not checked
+   * @param {object} [options] - extra checking, off by default so that existing callers
+   *   (Accept-Language handling above all) keep the behaviour they were written against
+   * @param {boolean} [options.caseInsensitive] - fold subtags to their registry case before
+   *   looking them up. BCP 47 s2.1.1 says tags are case-insensitive and the conventional
+   *   casing is a recommendation, so `en-us` is a valid way to write `en-US`
+   * @param {boolean} [options.checkCombinations] - apply the registry's statements about
+   *   which subtags may go together: the Prefix on an extlang or variant, and that a
+   *   variant is registered at all
+   */
+  constructor(code = '', languageDefinitions = null, options = {}) {
+    this.options = options || {};
     this.code = code;
     this.language = '';
     this.extLang = [];
@@ -115,14 +129,26 @@ class Language {
   _parse(languageDefinitions) {
     if (!this.code) return;
 
+    // Grandfathered tags are whole tags, and most do not decompose into valid subtags
+    // (i-klingon has no language 'i'), so they have to be recognised before the tag is
+    // split up. Redundant entries are deliberately NOT here: zh-Hans decomposes, and
+    // matching it whole would rob the caller of the script.
+    if (languageDefinitions && languageDefinitions.wholeTags.has(this.code.toLowerCase())) {
+      const entry = languageDefinitions.wholeTags.get(this.code.toLowerCase());
+      this.language = entry.code;
+      this.wholeTag = entry;
+      return;
+    }
+
+    const fold = this.options.caseInsensitive;
     const parts = this.code.split('-');
     let index = 0;
 
     // Language (required)
     if (index < parts.length) {
-      this.language = parts[index];
+      this.language = fold ? parts[index].toLowerCase() : parts[index];
       if (this.language != '*' && languageDefinitions && !languageDefinitions.languages.has(this.language)) {
-        throw new Error("The language '"+this.language+"' in the code '"+this.code+"' is not valid");
+        throw new Error("The language '"+parts[index]+"' in the code '"+this.code+"' is not valid");
       }
       index++;
     }
@@ -130,10 +156,14 @@ class Language {
     // Extended language (up to 3)
     for (let i = 0; i < 3 && index < parts.length; i++) {
       const part = parts[index];
+      const key = fold ? part.toLowerCase() : part;
       if (part.length === 3 && /^[a-zA-Z]{3}$/.test(part)) {
-        if (languageDefinitions && !languageDefinitions.extLanguages.has(part)) {
+        if (languageDefinitions && !languageDefinitions.extLanguages.has(key)) {
           throw new Error("The extLanguage '"+part+"' in the code '"+this.code+"' is not valid");
         }
+        if (this.options.checkCombinations && languageDefinitions) {
+          this._checkPrefix(languageDefinitions.extLanguages.get(key), 'extLanguage', part);
+        }
         this.extLang.push(part.toLowerCase());
         index++;
       } else {
@@ -145,7 +175,8 @@ class Language {
     if (index < parts.length) {
       const part = parts[index];
       if (part.length === 4 && /^[a-zA-Z]{4}$/.test(part)) {
-        if (languageDefinitions && !languageDefinitions.scripts.has(part)) {
+        const key = fold ? part.charAt(0).toUpperCase() + part.slice(1).toLowerCase() : part;
+        if (languageDefinitions && !languageDefinitions.scripts.has(key)) {
           throw new Error("The script '"+part+"' in the code '"+this.code+"' is not valid");
         }
         this.script = part.charAt(0).toUpperCase() + part.slice(1).toLowerCase();
@@ -158,7 +189,8 @@ class Language {
       const part = parts[index];
       if ((part.length === 2 && /^[a-zA-Z]{2}$/.test(part)) ||
         (part.length === 3 && /^[0-9]{3}$/.test(part))) {
-        if (languageDefinitions && !languageDefinitions.regions.has(part)) {
+        const key = fold ? part.toUpperCase() : part;
+        if (languageDefinitions && !languageDefinitions.regions.has(key)) {
           throw new Error("The region '"+part+"' in the code '"+this.code+"' is not valid");
         }
         this.region = part.toUpperCase();
@@ -171,7 +203,16 @@ class Language {
       const part = parts[index];
       if ((part.length >= 5 && part.length <= 8) ||
         (part.length === 4 && /^[0-9]/.test(part))) {
-        this.variant = part.toLowerCase();
+        const key = part.toLowerCase();
+        if (this.options.checkCombinations && languageDefinitions) {
+          // The variant is the one subtag whose existence was never checked at all -
+          // 'en-abcdef' parsed happily with variant 'abcdef'.
+          if (!languageDefinitions.variants.has(key)) {
+            throw new Error("The variant '"+part+"' in the code '"+this.code+"' is not valid");
+          }
+          this._checkPrefix(languageDefinitions.variants.get(key), 'variant', part);
+        }
+        this.variant = key;
         index++;
       }
     }
@@ -197,6 +238,36 @@ class Language {
     }
   }
 
+  /**
+   * The registry's Prefix field names the language(s) a subtag may follow; it is the only
+   * thing lang.dat says about which subtags go together. An entry with no Prefix may
+   * follow anything.
+   *
+   * The prefix itself may be a multi-subtag tag (variant 'nedis' has Prefix 'sl-rozaj'),
+   * so what has been parsed so far is reassembled and compared from the left.
+   *
+   * @param {object} entry - the registry entry, carrying `prefixes`
+   * @param {string} what - 'extLanguage' or 'variant', for the message
+   * @param {string} part - the subtag as written, for the message
+   * @private
+   */
+  _checkPrefix(entry, what, part) {
+    if (!entry || !entry.prefixes || entry.prefixes.length === 0) {
+      return;
+    }
+    const soFar = [this.language.toLowerCase()]
+      .concat(this.extLang.map(e => e.toLowerCase()))
+      .concat(this.script ? [this.script.toLowerCase()] : [])
+      .concat(this.region ? [this.region.toLowerCase()] : [])
+      .join('-');
+    const ok = entry.prefixes.some(prefix => soFar === prefix || soFar.startsWith(prefix + '-'));
+    if (!ok) {
+      throw new Error("The " + what + " '" + part + "' in the code '" + this.code
+        + "' may only be used with " + entry.prefixes.map(p => "'" + p + "'").join(' or ')
+        + ", not with '" + soFar + "'");
+    }
+  }
+
   /**
    * Check if this language matches another to a given depth
    */
@@ -503,9 +574,63 @@ class LanguageDefinitions {
     this.scripts = new Map();
     this.regions = new Map();
     this.variants = new Map();
+    // Whole tags that predate the current syntax (i-klingon, en-GB-oed) or that the
+    // registry records as ready-made combinations (az-Arab). They are matched as
+    // complete tags, before the tag is broken into subtags, because most of them do
+    // not decompose into valid subtags at all. Keyed lower case: BCP 47 tags are
+    // case-insensitive, and the registry writes these in mixed case.
+    this.wholeTags = new Map();
     this.parsed = new Map(); // Cache for parsed languages
   }
 
+  /**
+   * Expand a registry subtag range into its members.
+   *
+   * The registry writes private-use allocations as ranges - `QM..QZ`, `Qaaa..Qabx`,
+   * `qaa..qtz`, `XA..XZ` - in a single record. Stored literally, the key `QM..QZ` is
+   * one nobody will ever look up, and every code in the range reads as unregistered:
+   * `en-QM` was rejected while `en-AA` (allocated singly) was accepted.
+   *
+   * Only the trailing letters vary, so the shared prefix is kept and the remainder is
+   * counted through base 26, rendered in the case the range itself uses.
+   *
+   * @param {string} subtag - a subtag, with or without `..`
+   * @returns {string[]} every code the entry covers; a single-element array if not a range
+   */
+  static expandSubtagRange(subtag) {
+    const at = subtag.indexOf('..');
+    if (at === -1) {
+      return [subtag];
+    }
+    const from = subtag.substring(0, at);
+    const to = subtag.substring(at + 2);
+    if (from.length !== to.length || !/^[a-zA-Z]+$/.test(from) || !/^[a-zA-Z]+$/.test(to)) {
+      return [subtag];
+    }
+    let shared = 0;
+    while (shared < from.length && from[shared] === to[shared]) {
+      shared++;
+    }
+    const prefix = from.substring(0, shared);
+    const upper = /[A-Z]/.test(from.substring(shared));
+    const A = 'a'.charCodeAt(0);
+    const toIndex = (text) => [...text.toLowerCase()].reduce((n, ch) => n * 26 + (ch.charCodeAt(0) - A), 0);
+    const width = from.length - shared;
+    const render = (n) => {
+      let out = '';
+      for (let i = 0; i < width; i++) {
+        out = String.fromCharCode(A + (n % 26)) + out;
+        n = Math.floor(n / 26);
+      }
+      return upper ? out.toUpperCase() : out;
+    };
+    const result = [];
+    for (let n = toIndex(from.substring(shared)); n <= toIndex(to.substring(shared)); n++) {
+      result.push(prefix + render(n));
+    }
+    return result;
+  }
+
   /**
    * Load definitions from IETF language subtag registry file
    */
@@ -564,8 +689,12 @@ class LanguageDefinitions {
             this._loadVariant(vars);
             break;
           case 'grandfathered':
+            this._loadWholeTag(vars);
+            break;
           case 'redundant':
-            // Skip these for now
+            // Nothing to do: a redundant entry is called that because it decomposes into
+            // subtags that already say the same thing (zh-Hans is zh + Hans), so the
+            // normal path handles it and gives the caller the parts as well.
             break;
           default:
             throw new Error(`Unknown type: ${vars.Type} at line ${i + 1}`);
@@ -576,6 +705,24 @@ class LanguageDefinitions {
     }
   }
 
+  /**
+   * Load a grandfathered entry - a whole tag from before the current syntax, most of
+   * which cannot be decomposed into valid subtags at all ('i-klingon' has no language
+   * 'i'), so they are matched entire.
+   */
+  _loadWholeTag(vars) {
+    const tag = vars.Tag || vars.Subtag;
+    if (!tag) {
+      return;
+    }
+    this.wholeTags.set(tag.toLowerCase(), {
+      code: tag,
+      displays: vars.Description ? vars.Description.split('|') : [],
+      deprecated: !!vars.Deprecated,
+      preferred: vars['Preferred-Value'] || ''
+    });
+  }
+
   /**
    * Read variables from the registry format
    */
@@ -696,11 +843,12 @@ class LanguageDefinitions {
     lang.suppressScript = vars['Suppress-Script'] || '';
     lang.scope = vars.Scope || '';
 
-    if (this.languages.has(lang.code)) {
-      throw new Error(`Duplicate language code: ${lang.code}`);
+    for (const code of LanguageDefinitions.expandSubtagRange(lang.code)) {
+      if (this.languages.has(code)) {
+        throw new Error(`Duplicate language code: ${code}`);
+      }
+      this.languages.set(code, lang.code === code ? lang : Object.assign(Object.create(Object.getPrototypeOf(lang)), lang, { code }));
     }
-
-    this.languages.set(lang.code, lang);
   }
 
   /**
@@ -710,12 +858,15 @@ class LanguageDefinitions {
     const extLang = new LanguageExtLang();
     extLang.code = vars.Subtag;
     extLang.displays = vars.Description ? vars.Description.split('|') : [];
+    // Prefix is the registry saying which language(s) this subtag may follow
+    extLang.prefixes = vars.Prefix ? vars.Prefix.split('|').map(v => v.trim().toLowerCase()) : [];
 
-    if (this.extLanguages.has(extLang.code)) {
-      throw new Error(`Duplicate extlang code: ${extLang.code}`);
+    for (const code of LanguageDefinitions.expandSubtagRange(extLang.code)) {
+      if (this.extLanguages.has(code)) {
+        throw new Error(`Duplicate extlang code: ${code}`);
+      }
+      this.extLanguages.set(code, extLang.code === code ? extLang : Object.assign(Object.create(Object.getPrototypeOf(extLang)), extLang, { code }));
     }
-
-    this.extLanguages.set(extLang.code, extLang);
   }
 
   /**
@@ -726,11 +877,12 @@ class LanguageDefinitions {
     script.code = vars.Subtag;
     script.displays = vars.Description ? vars.Description.split('|') : [];
 
-    if (this.scripts.has(script.code)) {
-      throw new Error(`Duplicate script code: ${script.code}`);
+    for (const code of LanguageDefinitions.expandSubtagRange(script.code)) {
+      if (this.scripts.has(code)) {
+        throw new Error(`Duplicate script code: ${code}`);
+      }
+      this.scripts.set(code, script.code === code ? script : Object.assign(Object.create(Object.getPrototypeOf(script)), script, { code }));
     }
-
-    this.scripts.set(script.code, script);
   }
 
   /**
@@ -741,11 +893,12 @@ class LanguageDefinitions {
     region.code = vars.Subtag;
     region.displays = vars.Description ? vars.Description.split('|') : [];
 
-    if (this.regions.has(region.code)) {
-      throw new Error(`Duplicate region code: ${region.code}`);
+    for (const code of LanguageDefinitions.expandSubtagRange(region.code)) {
+      if (this.regions.has(code)) {
+        throw new Error(`Duplicate region code: ${code}`);
+      }
+      this.regions.set(code, region.code === code ? region : Object.assign(Object.create(Object.getPrototypeOf(region)), region, { code }));
     }
-
-    this.regions.set(region.code, region);
   }
 
   /**
@@ -755,12 +908,14 @@ class LanguageDefinitions {
     const variant = new LanguageVariant();
     variant.code = vars.Subtag;
     variant.displays = vars.Description ? vars.Description.split('|') : [];
+    variant.prefixes = vars.Prefix ? vars.Prefix.split('|').map(v => v.trim().toLowerCase()) : [];
 
-    if (this.variants.has(variant.code)) {
-      throw new Error(`Duplicate variant code: ${variant.code}`);
+    for (const code of LanguageDefinitions.expandSubtagRange(variant.code)) {
+      if (this.variants.has(code)) {
+        throw new Error(`Duplicate variant code: ${code}`);
+      }
+      this.variants.set(code, variant.code === code ? variant : Object.assign(Object.create(Object.getPrototypeOf(variant)), variant, { code }));
     }
-
-    this.variants.set(variant.code, variant);
   }
 
   /**
@@ -768,7 +923,7 @@ class LanguageDefinitions {
    *
    * @return {Language} parsed language (or null)
    */
-  parse(code, msg) {
+  parse(code, msg, options = {}) {
     if (!code) {
       if (msg) {
         msg.message = 'No code provided';
@@ -776,15 +931,17 @@ class LanguageDefinitions {
       return null;
     }
 
-    // Check cache first
-    if (this.parsed.has(code)) {
-      return this.parsed.get(code);
+    // The cache key carries the options: the same tag can be valid under one set of
+    // rules and not another, so a lenient parse must not be served to a strict caller.
+    const key = (options.caseInsensitive ? 'i' : '') + (options.checkCombinations ? 'c' : '') + ':' + code;
+    if (this.parsed.has(key)) {
+      return this.parsed.get(key);
     }
 
     try {
-      const lang = new Language(code, this);
+      const lang = new Language(code, this, options);
       // Cache the result
-      this.parsed.set(code, lang);
+      this.parsed.set(key, lang);
       return lang;
     } catch (e) {
       if (msg) {

package/library/path-safety.js

@@ -0,0 +1,51 @@
+/**
+ * Containment for filesystem paths built from request URLs.
+ *
+ * The rule everywhere a URL path is turned into a filename: resolve first, then check
+ * that the result is still inside the directory you meant to serve. `path.join` does
+ * not do this - it resolves `..` happily and will walk straight out of the directory.
+ *
+ * Note that Express does not URL-decode `req.path`, so `%2e%2e` never arrives here as
+ * `..`. A literal `..` does: nothing stops a client from sending `GET /../../etc/passwd`
+ * on a raw socket, and only the browsers and proxies in between normalise it away.
+ *
+ * @module library/path-safety
+ */
+
+const path = require('path');
+
+/**
+ * Resolve a URL path against a root directory, refusing anything that escapes it.
+ *
+ * The trailing separator in the comparison is load-bearing: without it a sibling
+ * directory whose name merely starts with the same characters ('/srv/site' vs
+ * '/srv/site-backup') would be accepted as inside the root.
+ *
+ * @param {string} rootDir - Directory that the result must stay inside. Resolved first,
+ *   so a relative or non-normalised root is fine.
+ * @param {string} urlPath - Path from the request (e.g. `req.path`), leading slash and all.
+ * @returns {string|null} The absolute path, or null if it would escape rootDir. The root
+ *   directory itself is allowed through; callers that only want files should stat it.
+ */
+function resolveWithin(rootDir, urlPath) {
+  if (typeof rootDir !== 'string' || typeof urlPath !== 'string') {
+    return null;
+  }
+  // A NUL truncates the name inside libc, so a path containing one is never what it
+  // appears to be. Node throws on these anyway; refusing here keeps the answer a null
+  // rather than an exception from whatever the caller does next.
+  if (urlPath.includes('\0')) {
+    return null;
+  }
+  const root = path.resolve(rootDir);
+  // '.' + urlPath keeps the join relative: resolve() would otherwise treat a leading
+  // slash as an absolute path and discard the root entirely.
+  const full = path.resolve(root, '.' + (urlPath.startsWith('/') ? urlPath : '/' + urlPath));
+  const prefix = root.endsWith(path.sep) ? root : root + path.sep;
+  if (full !== root && !full.startsWith(prefix)) {
+    return null;
+  }
+  return full;
+}
+
+module.exports = { resolveWithin };

package/library/request-token.js

@@ -0,0 +1,56 @@
+/**
+ * Shared-secret checks for administrative endpoints.
+ *
+ * These are endpoints that do something expensive or destructive and have no user
+ * behind them - a crawl trigger, a cache flush - so they are gated on a token from the
+ * configuration rather than on a login. Two rules make that safe, and both are easy to
+ * get wrong by hand:
+ *
+ *  - Fail closed. If no token is configured the endpoint is CLOSED, not open. An
+ *    administrative endpoint that quietly defaults to unauthenticated is worse than one
+ *    that is missing, because nothing about the deployment looks wrong.
+ *  - Compare in constant time. A byte-at-a-time comparison leaks the token one
+ *    character per round trip to anyone who can measure it.
+ *
+ * @module library/request-token
+ */
+
+const crypto = require('crypto');
+
+/**
+ * Does `supplied` match the configured token?
+ *
+ * @param {string|undefined|null} configured - The token from configuration. Anything
+ *   empty or non-string means the endpoint is not configured, and the answer is always
+ *   false - callers should say so rather than reporting a bad token.
+ * @param {*} supplied - The token from the request (a header value). Non-strings, and
+ *   the array Node produces for a repeated header, are rejected.
+ * @returns {boolean}
+ */
+function tokenMatches(configured, supplied) {
+  if (typeof configured !== 'string' || configured.length === 0) {
+    return false;
+  }
+  if (typeof supplied !== 'string' || supplied.length === 0) {
+    return false;
+  }
+  const a = Buffer.from(configured, 'utf8');
+  const b = Buffer.from(supplied, 'utf8');
+  // timingSafeEqual throws unless the lengths match, and the length itself is not a
+  // secret worth protecting - but returning early on it would still be a comparison
+  // whose cost depends on the input, so hash both to a fixed width and compare that.
+  const ha = crypto.createHash('sha256').update(a).digest();
+  const hb = crypto.createHash('sha256').update(b).digest();
+  return crypto.timingSafeEqual(ha, hb);
+}
+
+/**
+ * Is this endpoint configured with a token at all?
+ * @param {string|undefined|null} configured
+ * @returns {boolean}
+ */
+function tokenConfigured(configured) {
+  return typeof configured === 'string' && configured.length > 0;
+}
+
+module.exports = { tokenMatches, tokenConfigured };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "fhirsmith",
-  "version": "0.12.0",
-  "txVersion": "1.9.3",
+  "version": "0.13.1",
+  "txVersion": "1.9.4",
   "description": "A Node.js server that provides a collection of tools to serve the FHIR ecosystem",
   "main": "server.js",
   "engines": {

package/packages/packages.js

@@ -16,6 +16,7 @@ const escape = require('escape-html');
 const Logger = require('../library/logger');
 const {validateParameter} = require("../library/utilities");
 const {describeCron} = require("../library/cron-utilities");
+const {tokenMatches, tokenConfigured} = require("../library/request-token");
 const pckLog = Logger.getInstance().child({ module: 'packages' });
 
 class PackagesModule {
@@ -592,75 +593,6 @@ class PackagesModule {
     }
   }
 
-  // Re-fetch a single package tarball and replace whatever is stored for it, bypassing
-  // the crawler's GUID dedup and notForPublication feed gate. Used to push out a
-  // corrected package that was already (mis)published.
-  async forceUpdatePackage(link) {
-    // Only allow fetching over http(s). This endpoint must never be usable to read
-    // local server files (path injection) - tarballs are always published web URLs.
-    if (!link || !/^https?:\/\//i.test(link)) {
-      throw new Error('Invalid package link (must be an http(s) URL): ' + link);
-    }
-    if (!this.crawler) {
-      this.crawler = new PackageCrawler(this.config, this.db, this.stats);
-    }
-
-    const buffer = await this.crawler.fetchUrl(link);
-    const npm = await this.crawler.extractNpmPackage(buffer, link);
-
-    // Refuse to re-store a still-broken package - that would just re-publish the bug.
-    if (npm.notForPublication) {
-      throw new Error('Refusing to store ' + npm.id + '#' + npm.version + ': fetched tarball is still flagged notForPublication');
-    }
-
-    const idver = npm.id + '#' + npm.version;
-
-    // Delete whatever is stored for this package version, matching on the id#version
-    // actually found in the fetched tarball. This used to match on GUID = link, but the
-    // stored GUID is whatever the feed said at crawl time (http vs https, or another
-    // permalink form), so an exact string match could silently delete nothing and leave
-    // the stale copy in place alongside the newly stored one.
-    const old = await this.deleteVersionsByIdVersion(npm.id, npm.version);
-
-    // Keep the old row's GUID when there was one - it is the feed's permalink, and
-    // reusing it stops the next crawl from re-inserting the package as a duplicate.
-    // When we never had the package, the link is the best available GUID (the feeds
-    // use the versioned package.tgz url as the permalink GUID anyway).
-    const guid = old.guids[0] || link;
-
-    const itemLog = { status: '??' };
-    await this.crawler.store(link, link, guid, new Date(), buffer, idver, itemLog);
-
-    pckLog.info('Force-updated ' + idver + ' from ' + link + ' (replaced ' + old.count + ' existing row(s), guid ' + guid + ')');
-    return { status: 'updated', id: npm.id, version: npm.version, replaced: old.count };
-  }
-
-  // Delete a stored package version and all of its child rows, by package id and
-  // version. Resolves to { count, guids }: how many PackageVersions rows went, and
-  // the distinct GUIDs they were stored under (so a caller can reuse the permalink).
-  deleteVersionsByIdVersion(id, version) {
-    return new Promise((resolve, reject) => {
-      this.db.all('SELECT PackageVersionKey, GUID FROM PackageVersions WHERE Id = ? AND Version = ?', [id, version], (err, rows) => {
-        if (err) return reject(err);
-        const keys = (rows || []).map(r => r.PackageVersionKey);
-        const guids = [...new Set((rows || []).map(r => r.GUID))];
-        if (keys.length === 0) return resolve({ count: 0, guids: [] });
-        const ph = keys.map(() => '?').join(',');
-        const stmts = [
-          'DELETE FROM PackageFHIRVersions WHERE PackageVersionKey IN (' + ph + ')',
-          'DELETE FROM PackageDependencies WHERE PackageVersionKey IN (' + ph + ')',
-          'DELETE FROM PackageURLs WHERE PackageVersionKey IN (' + ph + ')',
-          'DELETE FROM PackageVersions WHERE PackageVersionKey IN (' + ph + ')'
-        ];
-        const runNext = (i) => {
-          if (i >= stmts.length) return resolve({ count: keys.length, guids });
-          this.db.run(stmts[i], keys, (e) => e ? reject(e) : runNext(i + 1));
-        };
-        runNext(0);
-      });
-    });
-  }
-
   async initializeDatabase() {
     return new Promise((resolve, reject) => {
       // Use config path if absolute, otherwise resolve relative to data dir
@@ -1269,10 +1201,25 @@ class PackagesModule {
       }
     });
 
-    // Manual crawler trigger (existing)
+    // Manual crawler trigger. A crawl is a long, network-heavy job that anyone could
+    // otherwise start at will, so it is gated on a shared secret from the configuration
+    // (modules.packages.crawlToken), sent in the x-crawl-token header. With no token
+    // configured the endpoint is CLOSED - an administrative trigger that silently
+    // defaults to open is how a deployment ends up exposed without anything looking wrong.
     this.router.post('/crawl', async (req, res) => {
       const start = Date.now();
       try {
+        if (!tokenConfigured(this.config.crawlToken)) {
+          // Say which of the two it is: the operator needs to know the endpoint is off
+          // because it was never configured, not that they typed the token wrongly.
+          res.status(403).json({ error: 'forbidden: no crawlToken is configured, so this endpoint is disabled' });
+          return;
+        }
+        if (!tokenMatches(this.config.crawlToken, req.headers['x-crawl-token'])) {
+          pckLog.warn('Rejected /crawl: missing or invalid x-crawl-token (from ' + req.ip + ')');
+          res.status(403).json({ error: 'forbidden: missing or invalid x-crawl-token' });
+          return;
+        }
         try {
           await this.runCrawler();
           res.json({
@@ -1292,54 +1239,6 @@ class PackagesModule {
       }
     });
 
-    // Force-refresh specific packages, bypassing the feed. The crawler only fetches a
-    // package once (it dedupes on the feed GUID) and skips anything flagged
-    // notForPublication, so there is normally no way to make it re-pick-up a package
-    // that was published incorrectly and later corrected. This endpoint re-fetches the
-    // tarball(s) directly and replaces whatever is stored.
-    //
-    //   POST /update-package   { "links": ["http://hl7.org/fhir/uv/ips/2.0.1/package.tgz", ...] }
-    //
-    // The link is the versioned package.tgz url. The stored copy to replace is found by
-    // the id#version inside the fetched tarball (not by matching the link against the
-    // stored GUID, which can differ in scheme or form from what the feed published), and
-    // the replacement keeps the old row's GUID so a later crawl won't create a duplicate.
-    // If config.updateToken is set, the request must carry it in the x-update-token header.
-    this.router.post('/update-package', async (req, res) => {
-      const start = Date.now();
-      try {
-        if (this.config.updateToken && req.headers['x-update-token'] !== this.config.updateToken) {
-          res.status(403).json({ error: 'forbidden: missing or invalid x-update-token' });
-          return;
-        }
-        let links = req.body && (req.body.links || req.body.packages || (req.body.url ? [req.body.url] : (req.body.link ? [req.body.link] : null)));
-        if (typeof links === 'string') links = [links];
-        if (!Array.isArray(links) || links.length === 0) {
-          res.status(400).json({ error: 'Provide a JSON body like {"links": ["<package.tgz url>", ...]}' });
-          return;
-        }
-        const results = [];
-        for (const link of links) {
-          try {
-            results.push(Object.assign({ link }, await this.forceUpdatePackage(link)));
-          } catch (e) {
-            pckLog.error('Force update failed for ' + link + ': ' + e.message);
-            results.push({ link, status: 'error', error: e.message });
-          }
-        }
-        const failed = results.filter(r => r.status === 'error').length;
-        res.status(failed === results.length ? 500 : 200).json({
-          message: 'Processed ' + results.length + ' package(s), ' + failed + ' failed',
-          results
-        });
-      } catch (error) {
-        pckLog.error('update-package endpoint failed:', error);
-        res.status(500).json({ error: 'update-package failed', message: error.message });
-      } finally {
-        this.stats.countRequest('update-package', Date.now() - start);
-      }
-    });
-
     // Crawler statistics endpoint (existing)
     this.router.get('/stats', async (req, res) => {
       const start = Date.now();