crawlforge-mcp-server 6.4.0 → 6.6.0

package/src/tools/extract/extractStructured.js

@@ -258,6 +258,9 @@ export class ExtractStructuredTool {
           if (result?.method === 'llm') {
             extractionResult = result;
             extractionMethod = 'llm';
+            // A salvaged cut-off response: the rows are real, the caller must
+            // know the list is not the whole table (R21, 2026-09-09).
+            if (result.warning) warnings.push(result.warning);
           } else {
             llmErrorMessage = result?.error || 'LLM did not return usable JSON';
           }
@@ -433,6 +436,7 @@ export class ExtractStructuredTool {
         },
         extractionNotes,
         provenance,
+        ...(extractionResult.partial ? { partial: true } : {}),
         ...(warnings?.length ? { warnings } : {})
       };
 

package/src/tools/extract/processDocument.js

@@ -118,6 +118,57 @@ const ProcessDocumentResult = z.object({
   error: z.string().optional()
 });
 
+/**
+ * What a fetched body is, from its Content-Type, magic bytes and URL:
+ * 'pdf', 'docx', 'binary' (something this tool cannot read) or 'html'
+ * (HTML, XML, JSON and plain text all go through the page pipeline).
+ * @param {string} contentType
+ * @param {Buffer} buffer
+ * @param {string} [url]
+ * @returns {'pdf'|'docx'|'binary'|'html'}
+ */
+export function sniffDocumentKind(contentType = '', buffer, url = '') {
+  const type = String(contentType).split(';')[0].trim().toLowerCase();
+  const head = buffer.subarray(0, 8).toString('latin1');
+  let path = String(url).toLowerCase();
+  try { path = new URL(url).pathname.toLowerCase(); } catch { /* keep the raw string */ }
+
+  if (type === 'application/pdf' || head.startsWith('%PDF')) return 'pdf';
+
+  const zip = head.startsWith('PK\u0003\u0004');
+  if (
+    type === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' ||
+    (zip && (path.endsWith('.docx') || buffer.includes('word/document.xml')))
+  ) return 'docx';
+
+  if (zip) return 'binary';
+  if (/^(image|audio|video|font)\//.test(type)) return 'binary';
+  if (/^application\/(x-)?(zip|gzip|tar|7z|rar|msword|vnd\.)/.test(type)) return 'binary';
+  const textual = type.startsWith('text/') || /html|xml|json|javascript/.test(type);
+  if (!textual && buffer.subarray(0, 512).includes(0)) return 'binary';
+  return 'html';
+}
+
+/**
+ * Decode a text body by its Content-Type charset, else the <meta charset>
+ * it declares, else UTF-8.
+ * @param {Buffer} buffer
+ * @param {string} contentType
+ * @returns {string}
+ */
+export function decodeTextBody(buffer, contentType = '') {
+  let charset = (/charset=["']?([\w-]+)/i.exec(contentType) || [])[1];
+  if (!charset) {
+    const head = buffer.subarray(0, 4096).toString('latin1');
+    charset = (/<meta[^>]+charset=["']?([\w-]+)/i.exec(head) || [])[1];
+  }
+  try {
+    return new TextDecoder(charset || 'utf-8').decode(buffer);
+  } catch {
+    return buffer.toString('utf8');
+  }
+}
+
 export class ProcessDocumentTool {
   constructor() {
     this.pdfProcessor = new PDFProcessor();
@@ -340,7 +391,30 @@ export class ProcessDocumentTool {
         throw new Error(`HTTP ${response.status}: ${response.statusText}`);
       }
 
-      html = await response.text();
+      // What came back decides how it is read, not the sourceType the caller
+      // guessed: a .docx fetched as 'url' was run through the HTML pipeline
+      // and returned its ZIP bytes as page text, success:true (R21,
+      // 2026-09-09). PDFs reach their parser the same way.
+      const contentType = response.headers.get('content-type') || '';
+      const buffer = Buffer.from(await response.arrayBuffer());
+      const kind = sniffDocumentKind(contentType, buffer, source);
+      if (kind === 'pdf') {
+        result.documentType = 'pdf';
+        await this.processPDFDocument(result, buffer, 'pdf_buffer', options, identity);
+        return;
+      }
+      if (kind === 'docx') {
+        result.documentType = 'docx';
+        await this.processDocxBuffer(result, buffer, options);
+        return;
+      }
+      if (kind === 'binary') {
+        throw new Error(
+          `${contentType.split(';')[0] || 'This'} content is not a document this tool reads: ` +
+            'process_document reads PDF, DOCX, HTML and plain text.'
+        );
+      }
+      html = decodeTextBody(buffer, contentType);
       pageTitle = this.extractTitleFromHTML(html);
     }
 
@@ -349,6 +423,25 @@ export class ProcessDocumentTool {
     await this.processFetchedHtml(result, html, source, options);
   }
 
+  /**
+   * Read a Word document: its text (and markdown when asked) via mammoth.
+   * @param {Object} result - Result object to populate
+   * @param {Buffer} buffer - the .docx bytes
+   * @param {Object} options - Processing options
+   */
+  async processDocxBuffer(result, buffer, options) {
+    const mammoth = (await import('mammoth')).default;
+    const raw = await mammoth.extractRawText({ buffer });
+    result.content = { text: (raw.value || '').trim() };
+    if (options.outputFormat === 'markdown') {
+      const md = await mammoth.convertToMarkdown({ buffer });
+      result.content.markdown = md.value || '';
+    }
+    result.title = null;
+    const notes = (raw.messages || []).map((m) => m.message).filter(Boolean);
+    if (notes.length > 0) result.warnings = [...(result.warnings || []), ...notes.slice(0, 5)];
+  }
+
   /**
    * Process a local non-PDF file (sourceType 'file'): read it from disk and
    * run it through the same content-processing pipeline used for web pages.

package/src/tools/scrape/_brandingExtractor.js

@@ -15,6 +15,7 @@
  */
 
 import { safeFetch } from '../../utils/ssrfGuard.js';
+import { mediaAppliesToScreen } from '../../utils/hiddenContent.js';
 
 const GENERIC_FAMILIES = new Set([
   'serif', 'sans-serif', 'monospace', 'cursive', 'fantasy', 'system-ui',
@@ -123,19 +124,36 @@ export async function collectCssSources($, pageUrl, opts) {
   let inlineStyleEls = 0;
   let cssText = '';
 
+  // Print (and other non-screen) stylesheets never paint a screen render, so
+  // neither branding nor the hidden-content pass may read them. `media:
+  // 'unconditional-screen'` also drops sheets gated on a media query.
+  const unconditional = opts.media === 'unconditional-screen';
+  const applies = (el) => mediaAppliesToScreen($(el).attr('media'), { unconditional });
+
   $('style').each((_, el) => {
+    if (!applies(el)) return;
     const t = $(el).html();
     if (t) { cssText += '\n' + t; styleBlocks++; }
   });
 
-  $('[style]').each((_, el) => {
-    const t = $(el).attr('style');
-    if (t) { cssText += '\n*{' + t + '}'; inlineStyleEls++; }
-  });
+  // Inline style="" attributes are folded in as universal rules so branding
+  // can mine their colours and fonts. They are NOT visibility rules: one
+  // element carrying style="display:none" became `*{display:none}`, and the
+  // hidden-content pass then deleted every element small enough to pass its
+  // bulk guard — irs.gov's tax-bracket page came back as its header (R21,
+  // 2026-09-09). That pass reads inline styles per element itself, so it
+  // asks for `inlineStyles: false`.
+  if (opts.inlineStyles !== false) {
+    $('[style]').each((_, el) => {
+      const t = $(el).attr('style');
+      if (t) { cssText += '\n*{' + t + '}'; inlineStyleEls++; }
+    });
+  }
 
   if (opts.fetchLinkedCss) {
     const hrefs = [];
     $('link[rel~="stylesheet"][href]').each((_, el) => {
+      if (!applies(el)) return;
       const href = $(el).attr('href');
       if (href) hrefs.push(resolveUrl(href, pageUrl));
     });
@@ -404,7 +422,7 @@ function extractTokens(cssText, cssVariables) {
  * Extract the full branding object from a loaded cheerio $.
  * @param {import('cheerio').CheerioAPI} $
  * @param {string} pageUrl
- * @param {{ fetchLinkedCss?: boolean, maxStylesheets?: number, perFileTimeoutMs?: number, timeoutMs?: number, overallTimeoutMs?: number, stylesheetConcurrency?: number }} [opts]
+ * @param {{ fetchLinkedCss?: boolean, maxStylesheets?: number, perFileTimeoutMs?: number, timeoutMs?: number, overallTimeoutMs?: number, stylesheetConcurrency?: number, media?: 'screen'|'unconditional-screen', inlineStyles?: boolean }} [opts]
  * @returns {Promise<object>}
  */
 export async function extractBranding($, pageUrl, opts = {}) {

package/src/tools/scrape/unifiedScrape.js

@@ -542,7 +542,14 @@ export class UnifiedScrapeTool {
             // Themes split visibility rules across many component sheets — the
             // rule hiding Shopify's sold-out badge sits at index 12 of 38 on a
             // stock Dawn storefront, so a cap of 10 silently misses it.
-            maxStylesheets: 20
+            maxStylesheets: 20,
+            // Only sheets a screen render applies unconditionally: a
+            // media="print" sheet hid every screen element of irs.gov's
+            // tax-bracket page and left the print logo (R21, 2026-09-09).
+            media: 'unconditional-screen',
+            // stripHiddenFromDom reads style="" per element; folded into
+            // universal rules they hid the whole page (R21).
+            inlineStyles: false
           });
           css = collected.cssText || '';
         }

package/src/tools/search/redditSearch.js

@@ -15,11 +15,14 @@
  *   cross-subreddit full-text search, but has known post-2023 archive gaps
  *   and recurring outages.
  *
- * Routing: scoped searches go to Arctic Shift. An unscoped keyword search
- * finds posts through a site-restricted web search and then reads them — or
- * searches their comments — in the archive by id. PullPush stopped serving
- * automated clients in August 2026 and is used only on explicit request.
- * Both archives are free and need no credentials.
+ * Routing: Arctic Shift first, PullPush second. A scoped search queries
+ * Arctic Shift and, if that fails, PullPush. An unscoped keyword search finds
+ * posts through a site-restricted web search and reads them — or searches
+ * their comments — in Arctic Shift by id, then falls back to PullPush's own
+ * full-text search. Thread mode is Arctic Shift only. PullPush has refused
+ * automated clients since August 2026, so the fallback usually reports that
+ * refusal; it stays second for whenever it answers again. Both archives are
+ * free and need no credentials.
  *
  * Optional official-API path: if the user sets REDDIT_CLIENT_ID and
  * REDDIT_CLIENT_SECRET (their own Reddit app), posts/thread requests can read
@@ -101,9 +104,8 @@ export class RedditSearchTool {
     // Lazily constructed on first official-path use; overridable for tests.
     this._officialAdapter = options.officialAdapter || null;
 
-    // Web discovery serves the one shape no archive can: a keyword search
-    // across all of Reddit. Arctic Shift requires a subreddit or author scope,
-    // and PullPush stopped serving automated clients in August 2026.
+    // Web discovery serves the one shape Arctic Shift cannot: a keyword search
+    // across all of Reddit, which it requires a subreddit or author scope for.
     this.searchAdapter = options.searchAdapter || null;
     this.searchApiKey = options.searchApiKey || null;
     this.searchApiBaseUrl = options.searchApiBaseUrl || null;
@@ -179,13 +181,16 @@ export class RedditSearchTool {
       order = ['pullpush'];
     } else {
       // auto: prefer the user's own official API (live, authoritative) when it
-      // can serve this request, then fall back to the community archives.
-      // PullPush is no longer tried automatically: every request now returns
-      // 429 "This website does not provide free scraping resources for agents",
-      // or a Cloudflare 403 challenge. It stays available on explicit request.
+      // can serve this request, then the community archives — Arctic Shift
+      // first, PullPush second (owner's call, 2026-09-10). PullPush has refused
+      // automated clients since August 2026 — 429 "This website does not
+      // provide free scraping resources for agents", or a Cloudflare 403
+      // challenge — so the fallback is one un-retried request that usually
+      // reports that refusal after the real Arctic Shift error, and answers
+      // when PullPush is serving again. Thread mode has no second source.
       const archives = v.mode === 'thread' ? ['arctic_shift']
-        : arcticPossible ? ['arctic_shift']
-        : discoveryPossible ? ['web_discovery']
+        : arcticPossible ? ['arctic_shift', 'pullpush']
+        : discoveryPossible ? ['web_discovery', 'pullpush']
         : [];
       order = officialPossible ? ['reddit_api', ...archives] : archives;
     }
@@ -252,7 +257,7 @@ export class RedditSearchTool {
         ? 'Reddit-wide comment search: posts were found with a site-restricted web search, then each post\'s comments were searched for the keywords in the Arctic Shift archive, in post relevance order.'
         : 'Reddit-wide keyword search: posts were found with a site-restricted web search, then read from the Arctic Shift archive by ID.',
       'Results are ordered by web-search relevance, not by score or date.',
-      'Arctic Shift cannot keyword-search across all of Reddit, and PullPush no longer serves automated clients — scope the search to a subreddit or author to query the archive directly.',
+      'Arctic Shift cannot keyword-search across all of Reddit — scope the search to a subreddit or author to query the archive directly.',
     ];
     if (v.after || v.before) {
       // Silently dropping a date filter would return results the caller
@@ -489,12 +494,14 @@ export class RedditSearchTool {
     if (response.status === 429) {
       const reset = response.headers?.get?.('x-ratelimit-reset');
       // PullPush's 429 body states its actual policy ("does not provide free
-      // scraping resources for agents...") — pass that through verbatim.
+      // scraping resources for agents...") — pass that through verbatim. That
+      // policy refusal is not a transient throttle: retrying it only spends the
+      // delay, which matters now that PullPush is the automatic second source.
       let detail = '';
       try { detail = (await response.json())?.error ?? ''; } catch { /* no body */ }
       throw Object.assign(
         new Error(`rate limited (429)${reset ? `, retry in ${reset}s` : ''}${detail ? ` — ${detail}` : ''}`),
-        { retryable: true },
+        { retryable: !/does not provide free scraping/i.test(detail) },
       );
     }
     if (!response.ok) {

package/src/utils/hiddenContent.js

@@ -83,6 +83,38 @@ function stripConditionalBlocks(css) {
   return out;
 }
 
+/**
+ * Split a selector list on its top-level commas only. A naive split cut
+ * irs.gov's `.callout:has(> .a, .b, p, ul, h2, h3, table)` into bare `p`,
+ * `ul`, `h2`, `h3` and `table` rules carrying its `display:none !important`,
+ * and the stripper then deleted the article body (R21, 2026-09-09).
+ * @param {string} list
+ * @returns {string[]}
+ */
+export function splitSelectorList(list) {
+  const out = [];
+  let depth = 0;
+  let quote = null;
+  let start = 0;
+  for (let i = 0; i < list.length; i++) {
+    const ch = list[i];
+    if (quote) {
+      if (ch === '\\') i++;
+      else if (ch === quote) quote = null;
+      continue;
+    }
+    if (ch === '"' || ch === "'") quote = ch;
+    else if (ch === '(' || ch === '[') depth++;
+    else if (ch === ')' || ch === ']') depth = Math.max(0, depth - 1);
+    else if (ch === ',' && depth === 0) {
+      out.push(list.slice(start, i));
+      start = i + 1;
+    }
+  }
+  out.push(list.slice(start));
+  return out;
+}
+
 /** True when cheerio cannot meaningfully evaluate the selector. */
 function isUnsupportedSelector(selector) {
   return (
@@ -91,7 +123,7 @@ function isUnsupportedSelector(selector) {
     // case and these rules would hide content every real visitor sees.
     /(^|[\s.#\[])no-js(\b|[.\[])/.test(selector) ||
     selector.includes('::') ||
-    /:(hover|focus|focus-within|focus-visible|active|target|checked|disabled|placeholder|before|after|root|host|where|is|not\()/i.test(selector) ||
+    /:(hover|focus|focus-within|focus-visible|active|target|checked|disabled|placeholder|before|after|root|host|where|is|not\(|has\()/i.test(selector) ||
     selector.includes('@') ||
     selector.length === 0
   );
@@ -199,7 +231,7 @@ export function collectVisibilitySelectors(css) {
 
     const important = /!\s*important/i.test(body);
 
-    for (const raw of selectorList.split(',')) {
+    for (const raw of splitSelectorList(selectorList)) {
       const selector = raw.trim();
       if (isUnsupportedSelector(selector) || NEVER_REMOVE.has(selector)) continue;
       const entry = { selector, spec: specificity(selector), order, important };
@@ -249,12 +281,45 @@ function renderedSize($, el) {
 export function inlineStyleText($) {
   const parts = [];
   $('style').each((_, el) => {
+    // A <style media="print"> block is only consulted when printing.
+    if (!mediaAppliesToScreen($(el).attr('media'), { unconditional: true })) return;
     const text = $(el).html();
     if (text) parts.push(text);
   });
   return parts.join('\n');
 }
 
+/**
+ * Whether a stylesheet's `media` attribute applies to a screen render.
+ *
+ * `<link rel="stylesheet" media="print">` is consulted only when printing.
+ * irs.gov's tax-bracket page ships one that hides every screen element and
+ * shows a print-only logo; read as screen CSS it emptied the page to that logo
+ * (R21, 2026-09-09). Absent or empty, `all`, `screen` and `not print` apply.
+ * `print`, `speech` and the other non-screen types, and `not screen`/`not all`,
+ * do not. With `unconditional: true` a media QUERY (`screen and (max-width:
+ * 600px)`, a bare `(prefers-color-scheme: dark)`) does not apply either — the
+ * hidden-content pass ignores @media blocks for the same reason.
+ *
+ * @param {string|undefined} media - the attribute value
+ * @param {{ unconditional?: boolean }} [options]
+ * @returns {boolean}
+ */
+export function mediaAppliesToScreen(media, { unconditional = false } = {}) {
+  const value = (media || '').trim().toLowerCase();
+  if (!value) return true;
+  return value.split(',').some((part) => {
+    const query = part.trim();
+    if (!query) return false;
+    if (unconditional && query.includes('(')) return false;
+    if (query.startsWith('(')) return true; // a bare feature query: type "all"
+    const m = query.match(/^(?:(not|only)\s+)?([a-z-]+)/);
+    if (!m) return true;
+    const screenType = m[2] === 'all' || m[2] === 'screen';
+    return m[1] === 'not' ? !screenType : screenType;
+  });
+}
+
 /**
  * Remove browser-invisible content from a cheerio document, in place.
  *

package/src/utils/redditHosts.js

@@ -0,0 +1,123 @@
+/**
+ * redditHosts — reddit.com is never fetched.
+ *
+ * reddit.com refuses every non-browser client (403, and the stealth browsers
+ * too — the block is IP/TLS-reputation based), so a scrape or fetch_url
+ * against it always fails and leaves the caller guessing. Reddit is served by
+ * reddit_search, which reads the same posts and comments from the Arctic
+ * Shift community archive (PullPush second). A reddit.com target is therefore
+ * refused before any network work, with the reddit_search call that gets the
+ * same data spelled out — derived from the URL where the URL says enough.
+ *
+ * Runs inside the pre-fetch gate (robotsGate.js), so every fetching tool and
+ * both browser paths get it without knowing. Not overridable: respect_robots
+ * is about robots.txt, and fetching reddit.com fails whatever the caller sends.
+ *
+ * Mirrors the website's `src/lib/tools/reddit-hosts.ts` — same rule, same
+ * message — so a reddit.com URL is answered identically on both surfaces.
+ */
+
+export class UseRedditSearchError extends Error {
+  constructor(url) {
+    const call = redditSearchCallFor(url);
+    const nextStep = Object.keys(call).length
+      ? `reddit_search(${JSON.stringify(call)})`
+      : 'reddit_search with a query, subreddit or author — or mode "thread" with a post\'s link_id';
+    super(
+      `${hostOf(url)} is not fetched: reddit.com refuses every non-browser client, stealth browsers included, ` +
+      `so this call would fail. Reddit is served by reddit_search (5 credits), which reads the same posts ` +
+      `and comments from the Arctic Shift community archive. Next step: ${nextStep}`
+    );
+    this.name = 'UseRedditSearchError';
+    this.code = 'USE_REDDIT_SEARCH';
+    this.url = url;
+    this.redditSearchCall = call;
+  }
+}
+
+/** The hostname of a URL, lowercased, or null if it will not parse. */
+function hostOf(url) {
+  try {
+    return new URL(url).hostname.toLowerCase().replace(/\.$/, '');
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * True for reddit.com and its subdomains (www, old, new, np, sh …) and for the
+ * bare redd.it short-link host. The media hosts (i.redd.it, v.redd.it,
+ * preview.redd.it) serve files to any client and are left alone.
+ * @param {string} url
+ */
+export function isRedditUrl(url) {
+  const host = hostOf(url);
+  if (!host) return false;
+  return host === 'reddit.com' || host.endsWith('.reddit.com') || host === 'redd.it';
+}
+
+/**
+ * The reddit_search call that answers a reddit.com URL, or {} when the URL
+ * names nothing reddit_search can be pointed at (the front page, a wiki, a
+ * settings page).
+ *
+ *   /r/{sub}/comments/{id}/…, /comments/{id}, /gallery/{id}, redd.it/{id}
+ *                                        → mode "thread", link_id
+ *   /r/{sub}/search?q=…                  → query scoped to the subreddit
+ *   /search?q=…                          → an unscoped query
+ *   /r/{sub}[/new|/top|…]                → the subreddit's posts
+ *   /user/{name} or /u/{name}[/comments] → the author's posts (or comments)
+ *   ?type=comment                        → mode "comments"
+ * @param {string} url
+ * @returns {{ mode?: string, query?: string, subreddit?: string, author?: string, link_id?: string }}
+ */
+export function redditSearchCallFor(url) {
+  let parsed;
+  try {
+    parsed = new URL(url);
+  } catch {
+    return {};
+  }
+  const segments = parsed.pathname
+    .split('/')
+    .filter(Boolean)
+    .map((segment) => {
+      try {
+        return decodeURIComponent(segment);
+      } catch {
+        return segment;
+      }
+    });
+
+  if (parsed.hostname.toLowerCase() === 'redd.it') {
+    return segments[0] ? { mode: 'thread', link_id: segments[0] } : {};
+  }
+
+  // A post's id follows "comments" or "gallery" wherever it sits in the path;
+  // a permalink to one comment still reads as its thread.
+  const idAt = segments.findIndex((segment) => segment === 'comments' || segment === 'gallery');
+  if (idAt !== -1 && segments[idAt + 1]) {
+    return { mode: 'thread', link_id: segments[idAt + 1] };
+  }
+
+  const call = {};
+  if (segments[0] === 'r' && segments[1]) call.subreddit = segments[1];
+  if ((segments[0] === 'user' || segments[0] === 'u') && segments[1]) {
+    call.author = segments[1];
+    if (segments[2] === 'comments') call.mode = 'comments';
+  }
+  const query = parsed.searchParams.get('q')?.trim();
+  if (query) call.query = query;
+  if (parsed.searchParams.get('type') === 'comment') call.mode = 'comments';
+  return call;
+}
+
+/**
+ * Throw UseRedditSearchError for a reddit.com target. Call before any network
+ * work — the point is that reddit.com never gets a request, not even for its
+ * robots.txt.
+ * @param {string} url
+ */
+export function assertNotRedditUrl(url) {
+  if (isRedditUrl(url)) throw new UseRedditSearchError(url);
+}

package/src/utils/robotsGate.js

@@ -19,6 +19,7 @@
 
 import { RobotsChecker } from './robotsChecker.js';
 import { assertHostAllowed } from './hostBlocklist.js';
+import { assertNotRedditUrl } from './redditHosts.js';
 import { identityHeaders, resolveUserAgent } from './fetchIdentity.js';
 import { throttleHost } from './hostRateLimiter.js';
 import { recordComplianceEvent, apiKeyId } from './complianceAudit.js';
@@ -75,8 +76,13 @@ export async function robotsPreflight(url, options = {}) {
   // costs the caller nothing: we refused, we fetched nothing.
   try {
     assertHostAllowed(url);
+    // reddit.com refuses every non-browser client, so it is never fetched and
+    // the caller is pointed at reddit_search instead. Also not overridable.
+    assertNotRedditUrl(url);
   } catch (error) {
-    if (error?.code === 'HOST_BLOCKED') markPreflightRefusal('HOST_BLOCKED');
+    if (error?.code === 'HOST_BLOCKED' || error?.code === 'USE_REDDIT_SEARCH') {
+      markPreflightRefusal(error.code);
+    }
     throw error;
   }
 
@@ -178,11 +184,29 @@ export async function preflightFetch(url, options = {}) {
   return {
     headers: outboundHeaders(decision.userAgent, signature),
     userAgent: decision.userAgent,
-    warnings: decision.warnings,
+    warnings: [...decision.warnings, ...crawlDelayWarning(url, decision.crawlDelayMs)],
     overridden: decision.overridden
   };
 }
 
+/**
+ * Name a long Crawl-delay, so a slow multi-page call reads as compliance
+ * rather than a stall: eff.org asks every agent for 30 s, and a 10-page
+ * llms.txt run took 13 minutes with nothing in the response saying why (R21,
+ * 2026-09-09).
+ * @param {string} url
+ * @param {number} crawlDelayMs
+ * @returns {string[]}
+ */
+function crawlDelayWarning(url, crawlDelayMs) {
+  if (!(crawlDelayMs >= 5000)) return [];
+  let host = url;
+  try { host = new URL(url).host; } catch { /* keep the raw url */ }
+  return [
+    `robots.txt on ${host} asks for a ${Math.round(crawlDelayMs / 1000)} s crawl delay; requests to it are spaced by that much, so a multi-page call takes about that long per page.`
+  ];
+}
+
 /**
  * The gate for browser paths. Same decision as {@link preflightFetch}, minus
  * the identity and signature headers — those belong on an HTTP fetch, not on a
@@ -213,7 +237,7 @@ export async function browserPreflight(url, options = {}) {
   }
 
   await throttleHost(url, { crawlDelayMs: decision.crawlDelayMs });
-  return decision.warnings;
+  return [...decision.warnings, ...crawlDelayWarning(url, decision.crawlDelayMs)];
 }
 
 /** Test/diagnostic hook: drop every cached robots.txt. */