crawlforge-mcp-server 5.2.9 → 5.3.0

package/src/utils/complianceAudit.js

@@ -0,0 +1,72 @@
+/**
+ * complianceAudit — durable record of the compliance decisions a customer made.
+ *
+ * Ground rule G5 allows `respect_robots: false`, but only as *the customer's
+ * documented decision*. That means the override has to leave a trace: which key
+ * asked, for which URL, from which tool, when. Without the row the override is
+ * a silent product default again, which is the thing G5 exists to prevent.
+ *
+ * Rows go to `logs/compliance-audit.log` as JSONL (one row per line, appended)
+ * and to a small in-memory ring the tools and tests can read back. Writing is
+ * best-effort: an audit sink that throws must never fail a customer's fetch.
+ *
+ * The API key is never stored. `apiKeyId` is a truncated SHA-256 of it — stable
+ * enough to group a key's overrides, useless as a credential.
+ */
+
+import { createHash } from 'crypto';
+import { appendFile, mkdir } from 'fs/promises';
+import { dirname, join } from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const DEFAULT_LOG_PATH = join(__dirname, '../../logs/compliance-audit.log');
+
+const RING_SIZE = 200;
+const ring = [];
+
+let sink = null; // null → the default JSONL file sink
+
+/** Truncated, non-reversible identifier for an API key. */
+export function apiKeyId(apiKey) {
+  if (typeof apiKey !== 'string' || !apiKey) return 'anonymous';
+  return createHash('sha256').update(apiKey).digest('hex').slice(0, 16);
+}
+
+async function defaultSink(row) {
+  await mkdir(dirname(DEFAULT_LOG_PATH), { recursive: true });
+  await appendFile(DEFAULT_LOG_PATH, `${JSON.stringify(row)}\n`, 'utf8');
+}
+
+/**
+ * Record a compliance event. Never throws and never rejects.
+ * @param {{ event: string, url?: string, tool?: string, apiKeyId?: string, [k: string]: unknown }} event
+ * @returns {{ event: string, timestamp: string }} the row as written
+ */
+export function recordComplianceEvent(event = {}) {
+  const row = { timestamp: new Date().toISOString(), ...event };
+  ring.push(row);
+  if (ring.length > RING_SIZE) ring.shift();
+
+  Promise.resolve()
+    .then(() => (sink || defaultSink)(row))
+    .catch(() => { /* an audit sink must never break a fetch */ });
+
+  return row;
+}
+
+/** Most recent audit rows, newest last. Test/diagnostic hook. */
+export function getComplianceAuditRows() {
+  return ring.slice();
+}
+
+/** Replace the persistence sink (tests, or a hosted deployment). */
+export function setComplianceAuditSink(fn) {
+  sink = typeof fn === 'function' ? fn : null;
+}
+
+/** Test hook. */
+export function _resetComplianceAudit() {
+  ring.length = 0;
+  sink = null;
+}

package/src/utils/contentUtils.js

@@ -312,7 +312,18 @@ export class ContentQualityAssessor {
   }
 
   /**
-   * Calculate simple readability metrics
+   * Calculate Flesch Reading-Ease metrics.
+   *
+   * This is the single Flesch implementation behind process_document and
+   * extract_content: both the `readabilityScore` field and
+   * `qualityAssessment.metrics.readability` are derived from here, so the two
+   * can never report different scores for the same text.
+   *
+   * The score is deliberately NOT clamped to [0, 100] — Flesch is unbounded
+   * (very simple text exceeds 100, dense text goes negative), and
+   * assessContentQuality() treats `score > 100` as a quality signal, which a
+   * clamp would make unreachable.
+   *
    * @param {string} text - Text to analyze
    * @returns {Object} - Readability metrics
    */

package/src/utils/domainFilter.js

@@ -209,24 +209,36 @@ export class DomainFilter {
   /**
    * Check if URL is allowed based on all filtering rules
    * @param {string} url - URL to check
+   * @param {Object} [options] - Evaluation options
+   * @param {boolean} [options.isSeed=false] - Treat url as a crawl's start URL, which is
+   *   exempt from the include-pattern gate (the caller asked for it explicitly). Blacklist,
+   *   exclude patterns and whitelist checks are unchanged.
    * @returns {Object} Decision object with allowed status and metadata
    */
-  isAllowed(url) {
+  isAllowed(url, options = {}) {
+    const { isSeed = false } = options;
     try {
       const normalizedUrl = normalizeUrl(url);
-      
+      // Patterns are tested against the raw URL as well, so a pattern written with a
+      // trailing slash ('/docs/') still matches a URL normalizeUrl has stripped it from.
+      // The two forms can therefore reach different decisions and must not share a cache
+      // entry; seed checks skip the cache entirely since they use a different gate.
+      const cacheKey = normalizedUrl === url ? normalizedUrl : `${normalizedUrl}|${url}`;
+
       // Check cache first
-      if (this.cache.has(normalizedUrl)) {
+      if (!isSeed && this.cache.has(cacheKey)) {
         this.cacheHits++;
-        return this.cache.get(normalizedUrl);
+        return this.cache.get(cacheKey);
       }
 
       this.cacheMisses++;
-      const decision = this.evaluateUrl(normalizedUrl);
-      
+      const decision = this.evaluateUrl(normalizedUrl, url, isSeed);
+
       // Cache the decision
-      this.addToCache(normalizedUrl, decision);
-      
+      if (!isSeed) {
+        this.addToCache(cacheKey, decision);
+      }
+
       return decision;
     } catch (error) {
       return {
@@ -241,9 +253,11 @@ export class DomainFilter {
   /**
    * Internal URL evaluation logic
    * @param {string} url - Normalized URL to evaluate
+   * @param {string} [rawUrl=url] - URL as the caller supplied it, before normalization
+   * @param {boolean} [isSeed=false] - Skip the include-pattern gate for a crawl's start URL
    * @returns {Object} Decision object
    */
-  evaluateUrl(url) {
+  evaluateUrl(url, rawUrl = url, isSeed = false) {
     const urlObj = new URL(url);
     const domain = urlObj.hostname;
     const path = urlObj.pathname;
@@ -255,7 +269,7 @@ export class DomainFilter {
     }
 
     // 2. Check exclude patterns
-    const excludePatternResult = this.checkExcludePatterns(url);
+    const excludePatternResult = this.checkExcludePatterns(url, rawUrl);
     if (!excludePatternResult.allowed) {
       return excludePatternResult;
     }
@@ -267,13 +281,16 @@ export class DomainFilter {
     }
 
     // 4. Check include patterns
-    const includePatternResult = this.checkIncludePatterns(url);
+    const includePatternResult = this.checkIncludePatterns(url, rawUrl);
     if (includePatternResult.allowed) {
       return includePatternResult;
     }
 
-    // 5. Default behavior - if no whitelist exists, allow; if whitelist exists, deny
-    const hasWhitelist = this.whitelist.size > 0 || this.patterns.include.length > 0;
+    // 5. Default behavior - if no whitelist exists, allow; if whitelist exists, deny.
+    // A seed URL is not subject to the include-pattern gate: include patterns scope where a
+    // crawl may go, not whether the URL the caller named may be fetched at all.
+    const hasWhitelist = this.whitelist.size > 0 ||
+      (!isSeed && this.patterns.include.length > 0);
     
     return {
       allowed: !hasWhitelist,
@@ -366,12 +383,13 @@ export class DomainFilter {
 
   /**
    * Check exclude patterns
-   * @param {string} url - URL to check
+   * @param {string} url - Normalized URL to check
+   * @param {string} [rawUrl=url] - URL before normalization; tested as well
    * @returns {Object} Decision object
    */
-  checkExcludePatterns(url) {
+  checkExcludePatterns(url, rawUrl = url) {
     for (const patternConfig of this.patterns.exclude) {
-      if (patternConfig.pattern.test(url)) {
+      if (patternConfig.pattern.test(url) || patternConfig.pattern.test(rawUrl)) {
         return {
           allowed: false,
           reason: `Matches exclude pattern: ${patternConfig.rawPattern}`,
@@ -390,12 +408,13 @@ export class DomainFilter {
 
   /**
    * Check include patterns
-   * @param {string} url - URL to check
+   * @param {string} url - Normalized URL to check
+   * @param {string} [rawUrl=url] - URL before normalization; tested as well
    * @returns {Object} Decision object
    */
-  checkIncludePatterns(url) {
+  checkIncludePatterns(url, rawUrl = url) {
     for (const patternConfig of this.patterns.include) {
-      if (patternConfig.pattern.test(url)) {
+      if (patternConfig.pattern.test(url) || patternConfig.pattern.test(rawUrl)) {
         return {
           allowed: true,
           reason: `Matches include pattern: ${patternConfig.rawPattern}`,

package/src/utils/fetchIdentity.js

@@ -0,0 +1,62 @@
+/**
+ * fetchIdentity — the single outbound identity for every CrawlForge fetch.
+ *
+ * Nine tools used to hardcode nine different User-Agents, so nine tools saw nine
+ * different versions of the same page (a Zillow listing served 41 `address`
+ * elements to one tool and 9 to another). That is a correctness bug, not a
+ * disguise problem, and the fix is one honest identity everywhere:
+ *
+ *     CrawlForge/<version> (+https://crawlforge.dev)
+ *
+ * Ground rule G4: identify honestly by default — real product name, real
+ * contact URL — so a site that wants to block us can. Callers that have their
+ * own agreement with a target can pass a per-request `userAgent` override
+ * (G4's escape hatch); the override wins, the canonical UA applies otherwise.
+ *
+ * The literal 'User-Agent' header name lives here and nowhere else outside the
+ * browser paths, so `identityHeaders()` is the only way to spell it.
+ */
+
+import { createRequire } from 'module';
+
+const _require = createRequire(import.meta.url);
+const _pkg = _require('../../package.json');
+
+/** The canonical, honest identity every page fetch sends. */
+export const CRAWLFORGE_USER_AGENT = `CrawlForge/${_pkg.version} (+https://crawlforge.dev)`;
+
+/**
+ * Identity for non-page traffic (webhook delivery, health checks, alerting).
+ * Same product name and contact URL, with the role appended so a receiving
+ * server can tell a webhook POST from a crawl. Falls back to the canonical UA.
+ * @param {string} [role]
+ * @returns {string}
+ */
+export function serviceUserAgent(role) {
+  const trimmed = typeof role === 'string' ? role.trim() : '';
+  return trimmed
+    ? `CrawlForge/${_pkg.version} (+https://crawlforge.dev; ${trimmed})`
+    : CRAWLFORGE_USER_AGENT;
+}
+
+/**
+ * Resolve the User-Agent for a request: a non-empty override wins, otherwise
+ * the canonical identity (optionally role-suffixed).
+ * @param {string} [override]
+ * @param {string} [role]
+ * @returns {string}
+ */
+export function resolveUserAgent(override, role) {
+  const trimmed = typeof override === 'string' ? override.trim() : '';
+  return trimmed || serviceUserAgent(role);
+}
+
+/**
+ * The outbound identity headers for a fetch. Spread this into a headers object
+ * rather than writing the header name at the call site.
+ * @param {{ userAgent?: string, role?: string }} [options]
+ * @returns {{ 'User-Agent': string }}
+ */
+export function identityHeaders(options = {}) {
+  return { 'User-Agent': resolveUserAgent(options.userAgent, options.role) };
+}

package/src/utils/hostBlocklist.js

@@ -0,0 +1,81 @@
+/**
+ * hostBlocklist — the platform's permanent refusals (ground rule G7).
+ *
+ * When a site owner opts out or sends a takedown, the block has to hold at the
+ * platform layer. A blocklist any customer can switch off is not a blocklist,
+ * so nothing here reads a per-request flag: `respect_robots: false` does not
+ * reach it, and neither does a `userAgent` override.
+ *
+ * To add an entry, append to BLOCKED_HOSTS with a dated one-line reason. Blocks
+ * cover the host and all of its subdomains. `CRAWLFORGE_BLOCKED_HOSTS` (comma
+ * separated) adds more at runtime; it can only ever extend the list.
+ */
+
+/** @type {string[]} host → blocked, with the date and reason it was added. */
+const BLOCKED_HOSTS = [
+  // e.g. 'example.com', // 2026-08-27 owner opt-out, ref #123
+];
+
+export class BlockedHostError extends Error {
+  constructor(host) {
+    super(
+      `${host} is on CrawlForge's permanent blocklist (site-owner opt-out or takedown) ` +
+      `and cannot be fetched. This block is not overridable. ` +
+      `If you believe it is in error, contact support@crawlforge.dev.`
+    );
+    this.name = 'BlockedHostError';
+    this.code = 'HOST_BLOCKED';
+    this.host = host;
+  }
+}
+
+let overrideList = null; // tests only
+
+function blockedSet() {
+  if (overrideList) return overrideList;
+  const fromEnv = (process.env.CRAWLFORGE_BLOCKED_HOSTS || '')
+    .split(',')
+    .map((h) => h.trim().toLowerCase())
+    .filter(Boolean);
+  return new Set([...BLOCKED_HOSTS.map((h) => h.toLowerCase()), ...fromEnv]);
+}
+
+/** The hostname of a URL, lowercased, or null if it will not parse. */
+function hostOf(url) {
+  try {
+    return new URL(url).hostname.toLowerCase().replace(/\.$/, '');
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * True when the URL's host, or any parent domain of it, is blocked.
+ * @param {string} url
+ */
+export function isBlockedHost(url) {
+  const host = hostOf(url);
+  if (!host) return false;
+  const blocked = blockedSet();
+  if (blocked.size === 0) return false;
+
+  const labels = host.split('.');
+  for (let i = 0; i < labels.length - 1; i++) {
+    if (blocked.has(labels.slice(i).join('.'))) return true;
+  }
+  return blocked.has(host);
+}
+
+/**
+ * Throw BlockedHostError if the URL's host is blocked. Call before any network
+ * work — the point is that a blocked host never gets a request.
+ * @param {string} url
+ */
+export function assertHostAllowed(url) {
+  if (isBlockedHost(url)) throw new BlockedHostError(hostOf(url));
+}
+
+/** Test hook: replace the effective list. Pass null to restore. */
+export function _setBlockedHostsForTests(hosts) {
+  overrideList = hosts ? new Set(hosts.map((h) => h.toLowerCase())) : null;
+}

package/src/utils/hostRateLimiter.js

@@ -10,6 +10,14 @@
  * effective behaviour), enabled by RATE_LIMIT_PER_DOMAIN (default true). Setting
  * RATE_LIMIT_PER_DOMAIN=false disables the throttle entirely — there is no global
  * cross-host cap, so broad multi-host crawls are never slowed by this.
+ *
+ * Two politeness signals the host itself sends are honoured on top of that
+ * (ground rule G6 — load we impose is load someone pays for):
+ *   - robots.txt `Crawl-delay`, passed in per request by the robots gate;
+ *   - `Retry-After` on a 429/503, recorded by the fetch helpers so the *next*
+ *     request to that host waits instead of retrying straight into the wall.
+ * Both are host-scoped and survive RATE_LIMIT_PER_DOMAIN=false: an operator
+ * turning off our own throttle is not a licence to ignore the site's.
  */
 import { RateLimiter } from './rateLimiter.js';
 import { config } from '../constants/config.js';
@@ -26,13 +34,103 @@ function limiter() {
   return _limiter;
 }
 
+/** host → { lastRequestAt, notBefore } */
+const hostState = new Map();
+
+function hostOf(url) {
+  try {
+    return new URL(url).hostname.toLowerCase();
+  } catch {
+    return null;
+  }
+}
+
+function stateFor(host) {
+  let state = hostState.get(host);
+  if (!state) {
+    state = { lastRequestAt: 0, notBefore: 0 };
+    hostState.set(host, state);
+  }
+  return state;
+}
+
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+/**
+ * Parse a `Retry-After` header into milliseconds.
+ * Accepts delta-seconds ("2") and an HTTP-date ("Wed, 21 Oct 2026 07:28:00 GMT").
+ * Returns 0 for anything unparseable, negative, or absent.
+ * @param {string|null|undefined} value
+ * @param {number} [now] epoch ms, for deterministic tests
+ * @returns {number}
+ */
+export function parseRetryAfter(value, now = Date.now()) {
+  if (value === null || value === undefined) return 0;
+  const raw = String(value).trim();
+  if (!raw) return 0;
+
+  if (/^\d+$/.test(raw)) return parseInt(raw, 10) * 1000;
+
+  const asDate = Date.parse(raw);
+  if (!Number.isNaN(asDate)) return Math.max(0, asDate - now);
+
+  return 0;
+}
+
+/**
+ * Record a `Retry-After` the host asked for. Subsequent requests to that host
+ * wait it out. Capped at 5 minutes so a hostile or mistaken header cannot pin a
+ * worker indefinitely.
+ * @param {string} url
+ * @param {string|null|undefined} retryAfterHeader
+ * @returns {number} the backoff applied, in ms (0 if none)
+ */
+export function noteRetryAfter(url, retryAfterHeader) {
+  const host = hostOf(url);
+  if (!host) return 0;
+
+  const delayMs = Math.min(parseRetryAfter(retryAfterHeader), 5 * 60 * 1000);
+  if (delayMs <= 0) return 0;
+
+  const state = stateFor(host);
+  state.notBefore = Math.max(state.notBefore, Date.now() + delayMs);
+  return delayMs;
+}
+
+/** Remaining backoff for a host in ms (0 if none). Diagnostic/test hook. */
+export function getHostBackoffMs(url) {
+  const host = hostOf(url);
+  if (!host) return 0;
+  const state = hostState.get(host);
+  return state ? Math.max(0, state.notBefore - Date.now()) : 0;
+}
+
 /**
  * Wait (if necessary) until another request to this URL's host is allowed.
  * Never throws — a limiter failure must not block a legitimate fetch.
  * @param {string} url
+ * @param {{ crawlDelayMs?: number }} [options] `crawlDelayMs` from robots.txt
  */
-export async function throttleHost(url) {
-  if (config.rateLimit.perDomain === false) return; // feature disabled
+export async function throttleHost(url, options = {}) {
+  const host = hostOf(url);
+  const crawlDelayMs = Number(options.crawlDelayMs) > 0 ? Number(options.crawlDelayMs) : 0;
+
+  if (host) {
+    const state = stateFor(host);
+
+    // The host's own signals, honoured whether or not our throttle is enabled.
+    const waits = [];
+    if (state.notBefore > Date.now()) waits.push(state.notBefore - Date.now());
+    if (crawlDelayMs > 0 && state.lastRequestAt > 0) {
+      waits.push(state.lastRequestAt + crawlDelayMs - Date.now());
+    }
+    const wait = Math.max(0, ...waits);
+    if (wait > 0) await sleep(wait);
+
+    state.lastRequestAt = Date.now();
+  }
+
+  if (config.rateLimit.perDomain === false) return; // our own throttle disabled
   try {
     await limiter().checkLimit(url);
   } catch {
@@ -43,4 +141,5 @@ export async function throttleHost(url) {
 /** Test/diagnostic hook. */
 export function _resetHostRateLimiter() {
   _limiter = null;
+  hostState.clear();
 }

package/src/utils/robotsChecker.js

@@ -1,68 +1,123 @@
 import robotsParser from 'robots-parser';
 import { safeFetch } from './ssrfGuard.js';
+import { identityHeaders, CRAWLFORGE_USER_AGENT } from './fetchIdentity.js';
+
+/**
+ * The token this crawler used to identify as, honoured as a source of disallow
+ * only so that robots.txt rules written against the old name keep working.
+ */
+const LEGACY_PRODUCT_TOKEN = 'CrawlForge-Bot';
+
+/** How long a parsed robots.txt stays good for. */
+const DEFAULT_TTL_MS = parseInt(process.env.ROBOTS_CACHE_TTL_MS || '3600000', 10); // 1h
 
 export class RobotsChecker {
-  constructor(userAgent = 'CrawlForge/1.0') {
+  /**
+   * @param {string} [userAgent] identity the robots rules are evaluated against
+   * @param {{ ttlMs?: number }} [options]
+   */
+  constructor(userAgent = CRAWLFORGE_USER_AGENT, options = {}) {
     this.userAgent = userAgent;
+    this.ttlMs = options.ttlMs ?? DEFAULT_TTL_MS;
+    /** @type {Map<string, { robots: unknown, fetchedAt: number }>} */
     this.robotsCache = new Map();
+    /** In-flight fetches, so N concurrent requests to one host fetch robots once. */
+    this.inflight = new Map();
+    /** Diagnostic: how many robots.txt requests this checker has actually made. */
+    this.fetchCount = 0;
+  }
+
+  static robotsUrlFor(url) {
+    const urlObj = new URL(url);
+    return `${urlObj.protocol}//${urlObj.host}/robots.txt`;
+  }
+
+  /**
+   * Parsed robots.txt for a URL's host, served from cache while it is fresh.
+   * Concurrent callers share one in-flight fetch rather than each starting one.
+   */
+  async getRobots(url) {
+    const robotsUrl = RobotsChecker.robotsUrlFor(url);
+
+    const cached = this.robotsCache.get(robotsUrl);
+    if (cached && Date.now() - cached.fetchedAt < this.ttlMs) return cached.robots;
+
+    const pending = this.inflight.get(robotsUrl);
+    if (pending) return pending;
+
+    const promise = (async () => {
+      const robotsTxt = await this.fetchRobotsTxt(robotsUrl);
+      const robots = robotsParser(robotsUrl, robotsTxt);
+      this.robotsCache.set(robotsUrl, { robots, fetchedAt: Date.now() });
+      return robots;
+    })().finally(() => this.inflight.delete(robotsUrl));
+
+    this.inflight.set(robotsUrl, promise);
+    return promise;
   }
 
   async canFetch(url) {
     try {
-      const urlObj = new URL(url);
-      const robotsUrl = `${urlObj.protocol}//${urlObj.host}/robots.txt`;
-      
-      let robots = this.robotsCache.get(robotsUrl);
-      
-      if (!robots) {
-        const robotsTxt = await this.fetchRobotsTxt(robotsUrl);
-        robots = robotsParser(robotsUrl, robotsTxt);
-        this.robotsCache.set(robotsUrl, robots);
-      }
-      
-      return robots.isAllowed(url, this.userAgent);
+      const robots = await this.getRobots(url);
+      // robots-parser returns undefined when it has no opinion — that is "allowed".
+      // The legacy token is consulted as a source of disallow only: unifying on
+      // CrawlForge would otherwise silently un-block every site owner who had
+      // already written `User-agent: CrawlForge-Bot`, discarding a decision they
+      // made about us (G7). Where a file names neither token both fall through to
+      // the same `*` group, so this is a no-op.
+      const allowedFor = (ua) => robots.isAllowed(url, ua) !== false;
+      return allowedFor(this.userAgent) && allowedFor(LEGACY_PRODUCT_TOKEN);
     } catch (error) {
-      // If we can't fetch robots.txt, assume we can crawl
+      // A robots.txt we cannot read is not a disallow. Standard practice, and
+      // the alternative (fail closed on a network blip) blocks legitimate work.
       console.warn(`Failed to check robots.txt for ${url}:`, error.message);
       return true;
     }
   }
 
   async fetchRobotsTxt(robotsUrl) {
+    this.fetchCount++;
+    const controller = new AbortController();
+    // The timeout must stay armed for the body read, not just until headers
+    // arrive: a host that sends headers then trickles robots.txt forever would
+    // otherwise pin every tool behind the gate. Now that the gate runs before
+    // every fetching tool rather than only crawl_deep, one slow host would
+    // hang all of them. clearTimeout moves to the finally accordingly.
+    const timeoutId = setTimeout(() => controller.abort(), 5000);
+
     try {
-      const controller = new AbortController();
-      const timeoutId = setTimeout(() => controller.abort(), 5000);
-      
       const response = await safeFetch(robotsUrl, {
         signal: controller.signal,
-        headers: {
-          'User-Agent': this.userAgent
-        }
+        headers: identityHeaders({ userAgent: this.userAgent })
       });
-      
-      clearTimeout(timeoutId);
-      
+
       if (!response.ok) {
         return ''; // Empty robots.txt means everything is allowed
       }
-      
+
       return await response.text();
     } catch (error) {
       return ''; // If we can't fetch, assume no restrictions
+    } finally {
+      clearTimeout(timeoutId);
     }
   }
 
+  /** Crawl-delay in seconds from an already-cached robots.txt (0 if unknown). */
   getCrawlDelay(url) {
     try {
-      const urlObj = new URL(url);
-      const robotsUrl = `${urlObj.protocol}//${urlObj.host}/robots.txt`;
-      const robots = this.robotsCache.get(robotsUrl);
-      
-      if (robots) {
-        return robots.getCrawlDelay(this.userAgent) || 0;
-      }
-      
+      const cached = this.robotsCache.get(RobotsChecker.robotsUrlFor(url));
+      return cached ? cached.robots.getCrawlDelay(this.userAgent) || 0 : 0;
+    } catch {
       return 0;
+    }
+  }
+
+  /** Crawl-delay in seconds, fetching robots.txt if it is not cached yet. */
+  async fetchCrawlDelay(url) {
+    try {
+      const robots = await this.getRobots(url);
+      return robots.getCrawlDelay(this.userAgent) || 0;
     } catch {
       return 0;
     }
@@ -70,15 +125,8 @@ export class RobotsChecker {
 
   getSitemaps(url) {
     try {
-      const urlObj = new URL(url);
-      const robotsUrl = `${urlObj.protocol}//${urlObj.host}/robots.txt`;
-      const robots = this.robotsCache.get(robotsUrl);
-      
-      if (robots) {
-        return robots.getSitemaps() || [];
-      }
-      
-      return [];
+      const cached = this.robotsCache.get(RobotsChecker.robotsUrlFor(url));
+      return cached ? cached.robots.getSitemaps() || [] : [];
     } catch {
       return [];
     }
@@ -86,7 +134,6 @@ export class RobotsChecker {
 
   clearCache() {
     this.robotsCache.clear();
+    this.inflight.clear();
   }
 }
-
-export default RobotsChecker;