fhirsmith 0.9.6 → 0.10.0

package/publisher/publisher.js

@@ -17,6 +17,7 @@ class PublisherModule {
     this.logger = null;
     this.taskProcessor = null;
     this.isProcessing = false;
+    this.activeTaskIds = new Set();
     this.shutdownRequested = false;
     this.stats = stats;
   }
@@ -199,6 +200,17 @@ class PublisherModule {
         });
       });
     }
+    if (!columnNames.includes('publisher_version')) {
+      await new Promise((resolve, reject) => {
+        this.db.run('ALTER TABLE tasks ADD COLUMN publisher_version TEXT', (err) => {
+          if (err) reject(err);
+          else {
+            this.logger.info('Migration: added publisher_version column to tasks table');
+            resolve();
+          }
+        });
+      });
+    }
     const websiteColumns = await new Promise((resolve, reject) => {
       this.db.all("PRAGMA table_info(websites)", (err, rows) => {
         if (err) reject(err);
@@ -308,13 +320,19 @@ class PublisherModule {
       if (this.shutdownRequested) return;
 
       if (this.isProcessing) {
+        // A publication can legitimately run for longer than an hour (go-publish on a
+        // large IG), so never reset isProcessing while a task is in flight — doing so
+        // lets the poller start a second, concurrent run of the same task, which then
+        // fails on 'ig-registry already exists' and stomps the real run's status.
+        // Recovery from a genuinely hung IG Publisher is handled by the
+        // igPublisherTimeoutMinutes kill in runIgPublisher/runPublisherGoPublish.
         const stuckMs = this.isProcessingStarted ? Date.now() - this.isProcessingStarted : 0;
-        if (stuckMs > 60 * 60 * 1000) {
-          this.logger.warn('Task processor appears stuck (' + Math.round(stuckMs / 60000) + ' min) — resetting');
-          this.isProcessing = false;
-        } else {
-          return;
+        const warnAfterMs = ((this.config.igPublisherTimeoutMinutes || 60) + 30) * 60 * 1000;
+        if (stuckMs > warnAfterMs) {
+          this.logger.warn('Task processor still busy after ' + Math.round(stuckMs / 60000) + ' min (tasks: ' +
+            Array.from(this.activeTaskIds).join(', ') + ') — possible hang, not starting new work');
         }
+        return;
       }
 
       await this.processNextTask();
@@ -325,10 +343,14 @@ class PublisherModule {
     this.isProcessing = true;
     this.isProcessingStarted = Date.now();
 
+    let claimedTaskId = null;
     try {
       // Look for queued tasks first (draft builds)
       let task = await this.getNextQueuedTask();
       if (task) {
+        if (this.activeTaskIds.has(task.id)) return; // already being processed
+        claimedTaskId = task.id;
+        this.activeTaskIds.add(task.id);
         this.stats.task('Publisher', 'Building ' + task.npm_package_id + '#' + task.version);
         await this.processDraftBuild(task);
         this.stats.taskDone('Publisher', 'Built ' + task.npm_package_id + '#' + task.version);
@@ -338,6 +360,9 @@ class PublisherModule {
       // Then look for approved tasks (publishing)
       task = await this.getNextApprovedTask();
       if (task) {
+        if (this.activeTaskIds.has(task.id)) return; // already being processed
+        claimedTaskId = task.id;
+        this.activeTaskIds.add(task.id);
         this.stats.task('Publisher', 'Publishing ' + task.npm_package_id + '#' + task.version);
         await this.processPublication(task);
         this.stats.taskDone('Publisher', 'Published ' + task.npm_package_id + '#' + task.version);
@@ -349,6 +374,9 @@ class PublisherModule {
       this.logger.error('Error in task processor:', error);
       this.stats.taskError('Publisher', 'Error: ' + error.message);
     } finally {
+      if (claimedTaskId !== null) {
+        this.activeTaskIds.delete(claimedTaskId);
+      }
       this.isProcessing = false;
     }
   }
@@ -390,6 +418,9 @@ class PublisherModule {
       fields.push('waiting_approval_at = CURRENT_TIMESTAMP');
     } else if (status === 'complete') {
       fields.push('completed_at = CURRENT_TIMESTAMP');
+      // A successful completion invalidates any earlier failure (e.g. from a
+      // superseded duplicate run) — don't let a stale message outlive success.
+      fields.push('failure_reason = NULL');
     } else if (status === 'failed') {
       fields.push('failed_at = CURRENT_TIMESTAMP');
     }
@@ -414,6 +445,28 @@ class PublisherModule {
     });
   }
 
+  // Update arbitrary task fields WITHOUT touching status. Use this from long-running
+  // work (e.g. saving the announcement mid-publication) — writing back the in-memory
+  // task.status can resurrect a stale status and cause the poller to re-run the task.
+  async updateTaskFields(taskId, fields) {
+    const keys = Object.keys(fields);
+    if (keys.length === 0) return;
+    const assignments = keys.map(key => key + ' = ?');
+    const values = keys.map(key => fields[key]);
+    values.push(taskId);
+
+    return new Promise((resolve, reject) => {
+      this.db.run(
+          'UPDATE tasks SET ' + assignments.join(', ') + ' WHERE id = ?',
+          values,
+          (err) => {
+            if (err) reject(err);
+            else resolve();
+          }
+      );
+    });
+  }
+
   async logTaskMessage(taskId, level, message) {
     return new Promise((resolve) => {
       this.db.run(
@@ -500,6 +553,11 @@ class PublisherModule {
     // Step 3: Clone GitHub repository
     await this.cloneRepository(task, draftDir);
 
+    // Step 3b: Update SUSHI to the latest release. The IG Publisher runs SUSHI with
+    // --require-latest, so a SUSHI that has fallen behind npm's latest makes the build
+    // fail. Refreshing it here keeps draft builds working.
+    await this.ensureLatestSushi(task.id);
+
     // Step 4: Run IG publisher
     await this.runIGPublisher(publisherJar, draftDir, logFile, task.id);
 
@@ -536,6 +594,16 @@ class PublisherModule {
         throw new Error('Could not find publisher.jar in latest release');
       }
 
+      // Record which IG Publisher version this task is using. The same jar is reused
+      // for the publication run, so this is the version that produced the output.
+      const publisherVersion = releaseResponse.data.tag_name || releaseResponse.data.name || 'unknown';
+      try {
+        await this.updateTaskFields(taskId, { publisher_version: publisherVersion });
+      } catch (e) {
+        this.logger.warn('Failed to record publisher_version for task ' + taskId + ': ' + e.message);
+      }
+      await this.logTaskMessage(taskId, 'info', 'Using IG Publisher version ' + publisherVersion);
+
       await this.logTaskMessage(taskId, 'info', 'Downloading from: ' + downloadUrl);
 
       // Download the file
@@ -561,6 +629,62 @@ class PublisherModule {
     }
   }
 
+  // Directory holding FHIRsmith's own copy of SUSHI. Installing here (rather than -g)
+  // needs no root: on the server the global npm prefix (/usr) is root-owned.
+  sushiDir() {
+    return folders.filePath('publisher', 'sushi');
+  }
+
+  // Environment for spawning the IG Publisher so it finds our managed SUSHI on PATH.
+  // The publisher resolves the `sushi` command from PATH, so prepending our bin dir
+  // makes it use the version we just installed.
+  publisherEnv() {
+    const binDir = path.join(this.sushiDir(), 'bin');
+    return Object.assign({}, process.env, {
+      PATH: binDir + path.delimiter + (process.env.PATH || '')
+    });
+  }
+
+  // Update FHIRsmith's managed SUSHI to the latest npm release before a run. The IG
+  // Publisher invokes SUSHI with --require-latest, so a SUSHI that has fallen behind
+  // npm's latest makes the build abort. We install into a FHIRsmith-owned prefix
+  // (sushiDir) to avoid needing root for a global install, and the publisher picks it
+  // up via publisherEnv(). Best-effort: a failure is logged but not fatal - the
+  // corrected runBuild check in the IG Publisher now turns a stale SUSHI into a loud
+  // publication failure rather than a silently-published draft.
+  async ensureLatestSushi(taskId) {
+    const { spawn } = require('child_process');
+    const dir = this.sushiDir();
+    fs.mkdirSync(dir, { recursive: true });
+    await this.logTaskMessage(taskId, 'info', 'Ensuring SUSHI is up to date in ' + dir + ' ...');
+    await new Promise((resolve) => {
+      // -g with --prefix makes npm treat `dir` as the global prefix, so the binary
+      // lands at dir/bin/sushi (a FHIRsmith-owned, writable location).
+      const npm = spawn('npm', ['install', '-g', 'fsh-sushi@latest', '--prefix', dir], { stdio: ['ignore', 'pipe', 'pipe'] });
+      let err = '';
+      npm.stdout.on('data', () => { /* ignore */ });
+      npm.stderr.on('data', (d) => { err += d.toString(); });
+      npm.on('error', async (e) => {
+        await this.logTaskMessage(taskId, 'warn', 'Could not run npm to update SUSHI: ' + e.message);
+        resolve();
+      });
+      npm.on('close', async (code) => {
+        if (code === 0) {
+          let version = '';
+          try {
+            version = require('child_process').execSync('sushi --version', { env: this.publisherEnv(), encoding: 'utf8' }).trim();
+          } catch (e) {
+            version = '(version check failed)';
+          }
+          await this.logTaskMessage(taskId, 'info', 'SUSHI is up to date: ' + version);
+        } else {
+          await this.logTaskMessage(taskId, 'warn', 'SUSHI update exited with code ' + code + (err ? ': ' + err.trim().slice(-400) : ''));
+        }
+        resolve();
+      });
+    });
+  }
+
   async cloneRepository(task, draftDir) {
     const { spawn } = require('child_process');
     const gitUrl = 'https://github.com/' + task.github_org + '/' + task.github_repo + '.git';
@@ -616,20 +740,45 @@ class PublisherModule {
         '.'
       ], {
         cwd: draftDir,
-        stdio: ['pipe', 'pipe', 'pipe']
+        stdio: ['pipe', 'pipe', 'pipe'],
+        env: this.publisherEnv()
       });
 
       // Create log file stream
       const logStream = fs.createWriteStream(logFile);
 
+      const buildStart = Date.now();
+      let lastDataAt = Date.now();
+
       java.stdout.on('data', (data) => {
+        lastDataAt = Date.now();
         logStream.write(data);
       });
 
       java.stderr.on('data', (data) => {
+        lastDataAt = Date.now();
         logStream.write(data);
       });
 
+      // Heartbeat: emit a status line every 60s regardless of stdout activity,
+      // so silent phases of the Publisher (e.g. "Validating Resources") still
+      // surface a signal-of-life in the task log.
+      const heartbeat = setInterval(async () => {
+        const elapsedMs = Date.now() - buildStart;
+        const sinceDataMs = Date.now() - lastDataAt;
+        let logKb = 0;
+        try { logKb = Math.round(fs.statSync(logFile).size / 1024); } catch (_) { /* log file not created yet */ }
+        const elapsedMin = Math.floor(elapsedMs / 60000);
+        const elapsedSec = Math.floor(elapsedMs / 1000) % 60;
+        const idleSec = Math.floor(sinceDataMs / 1000);
+        await this.logTaskMessage(
+          taskId,
+          'info',
+          'IG Publisher heartbeat: elapsed ' + elapsedMin + 'm' + elapsedSec + 's, ' +
+          'log ' + logKb + ' KB, last output ' + idleSec + 's ago'
+        );
+      }, 60 * 1000);
+
       java.on('close', async (code) => {
         logStream.end();
 
@@ -660,6 +809,7 @@ class PublisherModule {
 
       java.on('close', () => {
         clearTimeout(timeout);
+        clearInterval(heartbeat);
       });
     });
   }
@@ -693,6 +843,91 @@ class PublisherModule {
     await this.logTaskMessage(task.id, 'info', 'Build output verified: package-id=' + qaData['package-id'] + ', version=' + qaData['ig-ver']);
   }
 
+  // Read package/package.json out of a .tgz without unpacking the whole archive.
+  inspectPackageTgz(tgzPath) {
+    const { spawn } = require('child_process');
+    return new Promise((resolve, reject) => {
+      const tar = spawn('tar', ['-xzOf', tgzPath, 'package/package.json']);
+      let out = '';
+      let err = '';
+      tar.stdout.on('data', (d) => { out += d.toString(); });
+      tar.stderr.on('data', (d) => { err += d.toString(); });
+      tar.on('error', reject);
+      tar.on('close', (code) => {
+        if (code !== 0) {
+          reject(new Error('Could not read ' + tgzPath + ': ' + (err.trim() || ('tar exit ' + code))));
+        } else {
+          try {
+            resolve(JSON.parse(out));
+          } catch (e) {
+            reject(new Error('Invalid package.json in ' + tgzPath + ': ' + e.message));
+          }
+        }
+      });
+    });
+  }
+
+  // Confirm the package.tgz files that landed in the web tree are publication builds,
+  // not the draft. Throws (failing the task) if a draft package was published.
+  async verifyPublishedPackage(task, website, draftDir) {
+    await this.logTaskMessage(task.id, 'info', 'Verifying published package(s) are publication builds...');
+
+    // The intended publication path comes from the IG's publication-request.json.
+    const prPath = path.join(draftDir, 'publication-request.json');
+    if (!fs.existsSync(prPath)) {
+      await this.logTaskMessage(task.id, 'warn', 'No publication-request.json found at ' + prPath + ' - skipping published-package check');
+      return;
+    }
+    const pr = JSON.parse(fs.readFileSync(prPath, 'utf8'));
+    const pubPath = pr.path;
+
+    // The website base url lets us map a canonical url to a folder in the web tree.
+    const setupPath = path.join(website.local_folder, 'publish-setup.json');
+    if (!pubPath || !fs.existsSync(setupPath)) {
+      await this.logTaskMessage(task.id, 'warn', 'Cannot resolve web path (path or publish-setup.json missing) - skipping published-package check');
+      return;
+    }
+    const setup = JSON.parse(fs.readFileSync(setupPath, 'utf8'));
+    const baseUrl = setup.website && setup.website.url;
+    if (!baseUrl || !pubPath.startsWith(baseUrl)) {
+      await this.logTaskMessage(task.id, 'warn', 'Publication path ' + pubPath + ' is not under website url ' + baseUrl + ' - skipping published-package check');
+      return;
+    }
+
+    const relVer = pubPath.substring(baseUrl.length).replace(/^\/+/, '');
+    const relCur = relVer.replace(/\/[^/]+\/?$/, ''); // strip the version segment for the "current" copy
+    const candidates = [
+      path.join(website.local_folder, relVer, 'package.tgz'),
+      path.join(website.local_folder, relCur, 'package.tgz')
+    ];
+
+    let checked = 0;
+    for (const pkgPath of candidates) {
+      if (!fs.existsSync(pkgPath)) {
+        continue;
+      }
+      checked++;
+      const json = await this.inspectPackageTgz(pkgPath);
+      const problems = [];
+      if (json.notForPublication) {
+        problems.push('notForPublication is set');
+      }
+      if (typeof json.url === 'string' && json.url.startsWith('file:')) {
+        problems.push('url is a local file path (' + json.url + ')');
+      }
+      if (problems.length > 0) {
+        throw new Error('Published package ' + pkgPath + ' is a draft build, not a publication build (' +
+            problems.join('; ') + '). The IG Publisher publication run likely skipped package ' +
+            'regeneration (e.g. a Jekyll/template failure). Not committing.');
+      }
+      await this.logTaskMessage(task.id, 'info', 'Verified publication package: ' + pkgPath);
+    }
+
+    if (checked === 0) {
+      await this.logTaskMessage(task.id, 'warn', 'No published package.tgz found to verify under ' + relVer + ' or ' + relCur);
+    }
+  }
+
   async runPublication(task) {
     const website = await this.getWebsite(task.website_id);
     if (!website) {
@@ -757,6 +992,12 @@ class PublisherModule {
     // Step 3: Pull latest web folder before publishing into it
     await this.runCommand('git', ['pull'], { cwd: website.git_root }, task.id, 'Pulling latest web folder');
 
+    // Step 3b: Update SUSHI to the latest release before the publication run. This is the
+    // step that previously failed silently: the publication build runs SUSHI with
+    // --require-latest, and a draft approved days earlier may now face a newer SUSHI on
+    // npm. Refresh it so the publication build doesn't abort on a version mismatch.
+    await this.ensureLatestSushi(task.id);
+
     // Step 4: Run the IG publisher in go-publish mode
     await this.runPublisherGoPublish(task.id, publisherJar, draftDir, website.local_folder,
         registryDir, historyDir, templatesDir, zipsDir, publishLogFile);
@@ -769,6 +1010,12 @@ class PublisherModule {
     }
     await this.logTaskMessage(task.id, 'info', 'Publication run verified: ' + pubLogName + ' found');
 
+    // Step 5b: Verify the published package is actually a publication build.
+    // If the IG Publisher's publication run skipped package regeneration (e.g. a Jekyll
+    // or template failure), the draft package - flagged notForPublication with a file://
+    // url - can survive into the web tree. Catch that here, before anything is committed.
+    await this.verifyPublishedPackage(task, website, draftDir);
+
     // Step 6: Commit and push the web folder
     await this.logTaskMessage(task.id, 'info', 'Committing changes to web folder...');
     const gitUrl = 'https://github.com/' + task.github_org + '/' + task.github_repo + '.git';
@@ -789,7 +1036,7 @@ class PublisherModule {
     if (fs.existsSync(announcementPath)) {
       try {
         const announcement = fs.readFileSync(announcementPath, 'utf8');
-        await this.updateTaskStatus(task.id, task.status, { announcement: announcement });
+        await this.updateTaskFields(task.id, { announcement: announcement });
         await this.logTaskMessage(task.id, 'info', 'Announcement text saved (' + announcement.length + ' chars)');
       } catch (err) {
         await this.logTaskMessage(task.id, 'warn', 'Failed to read announcement file: ' + err.message);
@@ -825,19 +1072,43 @@ class PublisherModule {
 
     return new Promise((resolve, reject) => {
       const java = spawn('java', args, {
-        stdio: ['pipe', 'pipe', 'pipe']
+        stdio: ['pipe', 'pipe', 'pipe'],
+        env: this.publisherEnv()
       });
 
       const logStream = fs.createWriteStream(logFile);
 
+      const buildStart = Date.now();
+      let lastDataAt = Date.now();
+
       java.stdout.on('data', (data) => {
+        lastDataAt = Date.now();
         logStream.write(data);
       });
 
       java.stderr.on('data', (data) => {
+        lastDataAt = Date.now();
         logStream.write(data);
       });
 
+      // Heartbeat: emit a status line every 60s regardless of stdout activity,
+      // so silent phases of the Publisher still surface a signal-of-life.
+      const heartbeat = setInterval(async () => {
+        const elapsedMs = Date.now() - buildStart;
+        const sinceDataMs = Date.now() - lastDataAt;
+        let logKb = 0;
+        try { logKb = Math.round(fs.statSync(logFile).size / 1024); } catch (_) { /* log file not created yet */ }
+        const elapsedMin = Math.floor(elapsedMs / 60000);
+        const elapsedSec = Math.floor(elapsedMs / 1000) % 60;
+        const idleSec = Math.floor(sinceDataMs / 1000);
+        await this.logTaskMessage(
+          taskId,
+          'info',
+          'IG Publisher go-publish heartbeat: elapsed ' + elapsedMin + 'm' + elapsedSec + 's, ' +
+          'log ' + logKb + ' KB, last output ' + idleSec + 's ago'
+        );
+      }, 60 * 1000);
+
       java.on('close', async (code) => {
         logStream.end();
         if (code === 0) {
@@ -856,16 +1127,18 @@ class PublisherModule {
         reject(error);
       });
 
-      // Timeout after 60 minutes for publication (longer than draft build)
+      // Timeout configurable via publisher.igPublisherTimeoutMinutes (default: 60 minutes)
+      const timeoutMinutes = this.config.igPublisherTimeoutMinutes || 60;
       const timeout = setTimeout(async () => {
         java.kill();
         logStream.end();
-        await this.logTaskMessage(taskId, 'error', 'IG Publisher go-publish timed out after 60 minutes');
+        await this.logTaskMessage(taskId, 'error', 'IG Publisher go-publish timed out after ' + timeoutMinutes + ' minutes');
         reject(new Error('IG Publisher go-publish timed out'));
-      }, 60 * 60 * 1000);
+      }, timeoutMinutes * 60 * 1000);
 
       java.on('close', () => {
         clearTimeout(timeout);
+        clearInterval(heartbeat);
       });
     });
   }
@@ -1125,7 +1398,7 @@ class PublisherModule {
         } else {
           content += '<div class="table-responsive">';
           content += '<table class="table table-striped">';
-          content += '<thead><tr><th>ID</th><th>Package</th><th>Version</th><th>Website</th><th>Status</th><th>Queued</th><th>User</th><th>Actions</th></tr></thead>';
+          content += '<thead><tr><th>ID</th><th>Package</th><th>Version</th><th>Website</th><th>Status</th><th>IG Publisher</th><th>Queued</th><th>User</th><th>Actions</th></tr></thead>';
           content += '<tbody>';
 
           for (const task of tasks) {
@@ -1146,6 +1419,7 @@ class PublisherModule {
             content += '<td>' + task.version + '</td>';
             content += '<td>' + task.website_name + '</td>';
             content += '<td><span class="badge bg-' + this.getStatusColor(task.status) + '">' + task.status + '</span></td>';
+            content += '<td>' + (task.publisher_version ? '<code>' + escape(task.publisher_version) + '</code>' : '<span class="text-muted">—</span>') + '</td>';
             content += '<td>' + new Date(task.queued_at).toLocaleString() + '</td>';
             content += '<td>' + task.user_name + '</td>';
             content += '<td class="task-actions">';
@@ -1450,6 +1724,10 @@ class PublisherModule {
           content += '<p><strong>Status:</strong> <span class="badge bg-' + this.getStatusColor(task.status) + '">' + task.status + '</span></p>';
           content += '<p><strong>GitHub:</strong> ' + task.github_org + '/' + task.github_repo + ' (' + task.git_branch + ')</p>';
 
+          if (task.publisher_version) {
+            content += '<p><strong>IG Publisher:</strong> <code>' + escape(task.publisher_version) + '</code></p>';
+          }
+
           if (task.local_folder) {
             content += '<p><strong>Local Folder:</strong> <code>' + task.local_folder + '</code></p>';
           }

package/registry/crawler.js

@@ -2,15 +2,34 @@
 // Crawler for gathering server information from terminology servers
 
 const axios = require('axios');
-const { 
-  ServerRegistries, 
-  ServerRegistry, 
-  ServerInformation, 
+const dns = require('dns');
+const http = require('http');
+const https = require('https');
+const ipaddr = require('ipaddr.js');
+const {
+  ServerRegistries,
+  ServerRegistry,
+  ServerInformation,
   ServerVersionInformation,
 } = require('./model');
 const {Extensions} = require("../tx/library/extensions");
 const {debugLog} = require("../tx/operation-context");
 
+// True if an IP literal is anything other than a normal public (unicast) address:
+// loopback, private, link-local (incl. 169.254.169.254 cloud metadata), unique-local,
+// CGNAT, multicast, reserved, etc. IPv4-mapped IPv6 addresses are unwrapped first.
+function isNonPublicAddress(ip) {
+  try {
+    let addr = ipaddr.parse(ip);
+    if (addr.kind() === 'ipv6' && addr.isIPv4MappedAddress()) {
+      addr = addr.toIPv4Address();
+    }
+    return addr.range() !== 'unicast';
+  } catch (e) {
+    return true; // unparseable - treat as unsafe
+  }
+}
+
 const MASTER_URL = 'https://fhir.github.io/ig-registry/tx-servers.json';
 
 class RegistryCrawler {
@@ -22,7 +41,8 @@ class RegistryCrawler {
       masterUrl: config.masterUrl || MASTER_URL,
       userAgent: config.userAgent || 'HealthIntersections/FhirServer',
       crawlInterval: config.crawlInterval || 5 * 60 * 1000, // 5 minutes default
-      apiKeys: config.apiKeys || {} // Map of server URL or code to API key
+      apiKeys: config.apiKeys || {}, // Map of server URL or code to API key
+      allowPrivateAddresses: config.allowPrivateAddresses || false // SSRF opt-out for local test servers
     };
     this.stats = stats;
 
@@ -39,6 +59,63 @@ class RegistryCrawler {
     this.log = logv;
   }
 
+  // A DNS lookup wrapper that rejects any host resolving to a non-public address.
+  // Enforced at connection time (so it also covers redirect targets and defeats
+  // DNS-rebinding), this is the SSRF guard for outbound fetches. The registry is
+  // crawled from server-supplied "next" links, so the target host is not trusted.
+  // Set config.allowPrivateAddresses = true to disable (e.g. for local test servers).
+  ssrfLookup() {
+    const allowPrivate = !!(this.config && this.config.allowPrivateAddresses);
+    return (hostname, options, callback) => {
+      if (typeof options === 'function') {
+        callback = options;
+        options = {};
+      }
+      const wantAll = !!(options && options.all);
+      dns.lookup(hostname, Object.assign({}, options, { all: true }), (err, addresses) => {
+        if (err) {
+          callback(err);
+          return;
+        }
+        if (!allowPrivate) {
+          for (const a of addresses) {
+            if (isNonPublicAddress(a.address)) {
+              callback(new Error('Blocked request to non-public address ' + a.address + ' (host ' + hostname + ')'));
+              return;
+            }
+          }
+        }
+        if (wantAll) {
+          callback(null, addresses);
+        } else {
+          callback(null, addresses[0].address, addresses[0].family);
+        }
+      });
+    };
+  }
+
+  // http/https agents that route every connection through the SSRF lookup.
+  guardedAgents() {
+    if (!this._guardedAgents) {
+      const lookup = this.ssrfLookup();
+      class GuardedHttpAgent extends http.Agent {
+        createConnection(options, cb) {
+          return super.createConnection(Object.assign({}, options, { lookup }), cb);
+        }
+      }
+      class GuardedHttpsAgent extends https.Agent {
+        createConnection(options, cb) {
+          return super.createConnection(Object.assign({}, options, { lookup }), cb);
+        }
+      }
+      this._guardedAgents = {
+        httpAgent: new GuardedHttpAgent({ keepAlive: true }),
+        httpsAgent: new GuardedHttpsAgent({ keepAlive: true })
+      };
+    }
+    return this._guardedAgents;
+  }
+
 
   /**
    * Main entry point - crawl the registry starting from the master URL
@@ -455,7 +532,9 @@ class RegistryCrawler {
         timeout: this.config.timeout,
         headers: headers,
         signal: this.abortController?.signal,
-        validateStatus: (status) => status < 500 // Don't throw on 4xx
+        validateStatus: (status) => status < 500, // Don't throw on 4xx
+        httpAgent: this.guardedAgents().httpAgent,
+        httpsAgent: this.guardedAgents().httpsAgent
       });
       
       if (response.status >= 400) {