crawlforge-mcp-server 6.3.1 → 6.5.0

package/src/tools/crawl/mapSite.js

@@ -10,6 +10,44 @@ import { CRAWLFORGE_USER_AGENT } from '../../utils/fetchIdentity.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
 import { pageTitle } from '../../utils/pageTitle.js';
 
+/**
+ * The path prefix a seed URL asks for: nps.gov/yell/ means the Yellowstone
+ * subtree, not the first 200 URLs of the park service's site-wide sitemap
+ * (R21, 2026-09-09: map_site returned Abraham Lincoln Birthplace pages for it).
+ * A page URL scopes to its directory; the site root scopes to nothing.
+ * @param {string} url
+ * @returns {string|null} e.g. "/yell/", or null for the root
+ */
+export function scopePathOf(url) {
+  try {
+    const { pathname } = new URL(url);
+    const dir = pathname.endsWith('/') ? pathname : pathname.slice(0, pathname.lastIndexOf('/') + 1);
+    return dir && dir !== '/' ? dir : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * How many of a search's terms a URL's own path and query contain — the
+ * relevance signal the generic ranker lacks when every candidate is a bare
+ * URL (it scored 0.19 for all 200 nps.gov URLs of a "fees" search, none of
+ * which mentioned fees).
+ * @param {string} url
+ * @param {string} search
+ * @returns {number}
+ */
+export function searchScore(url, search) {
+  const terms = [...new Set(String(search || '').toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((t) => t.length >= 2))];
+  if (terms.length === 0) return 0;
+  let haystack = url.toLowerCase();
+  try {
+    const { pathname, search: query } = new URL(url);
+    haystack = decodeURIComponent(pathname + query).toLowerCase();
+  } catch { /* keep the raw url */ }
+  return terms.filter((t) => haystack.includes(t)).length;
+}
+
 // Lazy singleton — avoids creating a CacheManager timer per request
 let _ranker = null;
 function getRanker() {
@@ -107,22 +145,47 @@ export class MapSiteTool {
         }
       }
 
+      // A seed with a path asks for that subtree, and a search needs a pool
+      // wider than max_urls to rank — otherwise the cut falls before the
+      // relevant URLs ever enter (the sitemap head is alphabetical).
+      const scopePath = scopePathOf(validated.url);
+      const widen = scopePath || validated.search;
+      const poolLimit = widen ? Math.min(10000, Math.max(validated.max_urls * 10, 2000)) : validated.max_urls;
+      const warnings = [];
+
       // Try to fetch sitemap first
       if (validated.include_sitemap) {
-        const sitemapUrls = await this.fetchSitemapUrls(baseUrl, domainFilter, validated.max_urls);
+        const sitemapUrls = await this.fetchSitemapUrls(baseUrl, domainFilter, poolLimit, scopePath);
         sitemapUrls.forEach(url => urls.add(normalizeUrl(url)));
       }
 
       // Fetch and parse the main page for additional URLs
       const pageUrls = await this.fetchPageUrls(validated.url, domainFilter, identity);
       pageUrls.forEach(url => {
-        if (urls.size < validated.max_urls) {
+        if (urls.size < poolLimit) {
           urls.add(normalizeUrl(url));
         }
       });
 
+      let pool = Array.from(urls);
+      if (scopePath) {
+        const inScope = pool.filter((u) => { try { return new URL(u).pathname.startsWith(scopePath); } catch { return false; } });
+        if (inScope.length > 0) {
+          pool = inScope;
+        } else {
+          warnings.push(`No URL under ${scopePath} was found in the sitemap or on the page; the whole site is listed instead.`);
+        }
+      }
+      if (validated.search) {
+        // Stable: relevance first, discovery order among equals.
+        pool = pool
+          .map((url, i) => ({ url, i, score: searchScore(url, validated.search) }))
+          .sort((a, b) => b.score - a.score || a.i - b.i)
+          .map((x) => x.url);
+      }
+
       // Convert to array and limit
-      const urlArray = Array.from(urls).slice(0, validated.max_urls);
+      const urlArray = pool.slice(0, validated.max_urls);
 
       // Fetch metadata if requested
       if (validated.include_metadata) {
@@ -142,7 +205,9 @@ export class MapSiteTool {
         site_map: this.generateSiteMap(urlArray),
         statistics: this.generateStatistics(urlArray),
         domain_filter_config: domainFilter ? domainFilter.exportConfig() : null,
-        filter_stats: domainFilter ? domainFilter.getStats() : null
+        filter_stats: domainFilter ? domainFilter.getStats() : null,
+        ...(scopePath ? { scope: scopePath } : {}),
+        ...(warnings.length ? { warnings } : {})
       };
 
       // Optional: rank URLs by relevance to a search string
@@ -157,7 +222,11 @@ export class MapSiteTool {
             return { link: url, title, snippet: '' };
           });
           const ranked = await getRanker().rankResults(rankerInput, validated.search);
-          result.ranked_urls = ranked.map(r => ({ url: r.link, score: r.finalScore ?? 0 }));
+          // The ranker's score is flat across bare URLs; the path-term count
+          // is what separates /yell/planyourvisit/fees.htm from the rest.
+          result.ranked_urls = ranked
+            .map(r => ({ url: r.link, score: Number(((r.finalScore ?? 0) + searchScore(r.link, validated.search)).toFixed(3)) }))
+            .sort((a, b) => b.score - a.score);
         } catch {
           // ranking is best-effort; don't fail the whole call
           result.ranked_urls = urlArray.map(u => ({ url: u, score: 0 }));
@@ -193,7 +262,11 @@ export class MapSiteTool {
     });
   }
 
-  async fetchSitemapUrls(baseUrl, domainFilter = null, maxUrls = Infinity) {
+  async fetchSitemapUrls(baseUrl, domainFilter = null, maxUrls = Infinity, scopePath = null) {
+    const inScope = (url) => {
+      if (!scopePath) return true;
+      try { return new URL(url).pathname.startsWith(scopePath); } catch { return false; }
+    };
     // Discover sitemaps via robots.txt and common paths, then parse with full
     // SitemapParser support (sitemap-index recursion, gzip, CDATA/entities).
     const discovered = await this.sitemapParser.discoverSitemaps(baseUrl, {
@@ -212,6 +285,7 @@ export class MapSiteTool {
         if (parsed.success) {
           for (const entry of parsed.urls) {
             const url = entry.loc || entry;
+            if (!inScope(url)) continue;
             if (!domainFilter || domainFilter.isAllowed(url).allowed) {
               urls.add(url);
             }
@@ -267,7 +341,7 @@ export class MapSiteTool {
       // A gate refusal is the answer to the request, not a page we failed to
       // read: surface it instead of returning an emptier map than the caller
       // would notice.
-      if (error.code === 'ROBOTS_DISALLOWED' || error.code === 'HOST_BLOCKED') throw error;
+      if (error.code === 'ROBOTS_DISALLOWED' || error.code === 'HOST_BLOCKED' || error.code === 'USE_REDDIT_SEARCH') throw error;
       return [];
     }
   }

package/src/tools/extract/extractEmbeddedState.js

@@ -41,7 +41,23 @@ export async function extractEmbeddedStateHandler({ url, path, user_agent, respe
       );
     }
 
-    const data = path ? selectJsonPath(state.data, path) : state.data;
+    // A path is naturally written against the payload ("props.pageProps"),
+    // not against this tool's envelope ("next_data.props.pageProps"). When
+    // the page carries exactly one payload and the path's root is not one of
+    // the envelope keys, read it inside that payload and say so (R21,
+    // 2026-09-09: four Next.js pages in a row failed on the bare path).
+    let effectivePath = path;
+    if (path && state.found.length === 1) {
+      const root = path.split(/[.[]/)[0];
+      if (root && !(root in state.data)) {
+        effectivePath = `${state.found[0].name}.${path}`;
+        warnings.push(
+          `path "${path}" was read as "${effectivePath}": "${state.found[0].name}" is the only payload on this page, so the path is resolved inside it.`
+        );
+      }
+    }
+
+    const data = effectivePath ? selectJsonPath(state.data, effectivePath) : state.data;
     const bytes = Buffer.byteLength(JSON.stringify(data) ?? '');
 
     if (!path && bytes > LARGE_RESULT_BYTES) {
@@ -57,7 +73,7 @@ export async function extractEmbeddedStateHandler({ url, path, user_agent, respe
         text: JSON.stringify({
           url: finalUrl,
           found: state.found,
-          path: path || null,
+          path: effectivePath || null,
           bytes,
           data,
           warnings

package/src/tools/extract/extractStructured.js

@@ -258,6 +258,9 @@ export class ExtractStructuredTool {
           if (result?.method === 'llm') {
             extractionResult = result;
             extractionMethod = 'llm';
+            // A salvaged cut-off response: the rows are real, the caller must
+            // know the list is not the whole table (R21, 2026-09-09).
+            if (result.warning) warnings.push(result.warning);
           } else {
             llmErrorMessage = result?.error || 'LLM did not return usable JSON';
           }
@@ -433,6 +436,7 @@ export class ExtractStructuredTool {
         },
         extractionNotes,
         provenance,
+        ...(extractionResult.partial ? { partial: true } : {}),
         ...(warnings?.length ? { warnings } : {})
       };
 

package/src/tools/extract/processDocument.js

@@ -118,6 +118,57 @@ const ProcessDocumentResult = z.object({
   error: z.string().optional()
 });
 
+/**
+ * What a fetched body is, from its Content-Type, magic bytes and URL:
+ * 'pdf', 'docx', 'binary' (something this tool cannot read) or 'html'
+ * (HTML, XML, JSON and plain text all go through the page pipeline).
+ * @param {string} contentType
+ * @param {Buffer} buffer
+ * @param {string} [url]
+ * @returns {'pdf'|'docx'|'binary'|'html'}
+ */
+export function sniffDocumentKind(contentType = '', buffer, url = '') {
+  const type = String(contentType).split(';')[0].trim().toLowerCase();
+  const head = buffer.subarray(0, 8).toString('latin1');
+  let path = String(url).toLowerCase();
+  try { path = new URL(url).pathname.toLowerCase(); } catch { /* keep the raw string */ }
+
+  if (type === 'application/pdf' || head.startsWith('%PDF')) return 'pdf';
+
+  const zip = head.startsWith('PK\u0003\u0004');
+  if (
+    type === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' ||
+    (zip && (path.endsWith('.docx') || buffer.includes('word/document.xml')))
+  ) return 'docx';
+
+  if (zip) return 'binary';
+  if (/^(image|audio|video|font)\//.test(type)) return 'binary';
+  if (/^application\/(x-)?(zip|gzip|tar|7z|rar|msword|vnd\.)/.test(type)) return 'binary';
+  const textual = type.startsWith('text/') || /html|xml|json|javascript/.test(type);
+  if (!textual && buffer.subarray(0, 512).includes(0)) return 'binary';
+  return 'html';
+}
+
+/**
+ * Decode a text body by its Content-Type charset, else the <meta charset>
+ * it declares, else UTF-8.
+ * @param {Buffer} buffer
+ * @param {string} contentType
+ * @returns {string}
+ */
+export function decodeTextBody(buffer, contentType = '') {
+  let charset = (/charset=["']?([\w-]+)/i.exec(contentType) || [])[1];
+  if (!charset) {
+    const head = buffer.subarray(0, 4096).toString('latin1');
+    charset = (/<meta[^>]+charset=["']?([\w-]+)/i.exec(head) || [])[1];
+  }
+  try {
+    return new TextDecoder(charset || 'utf-8').decode(buffer);
+  } catch {
+    return buffer.toString('utf8');
+  }
+}
+
 export class ProcessDocumentTool {
   constructor() {
     this.pdfProcessor = new PDFProcessor();
@@ -340,7 +391,30 @@ export class ProcessDocumentTool {
         throw new Error(`HTTP ${response.status}: ${response.statusText}`);
       }
 
-      html = await response.text();
+      // What came back decides how it is read, not the sourceType the caller
+      // guessed: a .docx fetched as 'url' was run through the HTML pipeline
+      // and returned its ZIP bytes as page text, success:true (R21,
+      // 2026-09-09). PDFs reach their parser the same way.
+      const contentType = response.headers.get('content-type') || '';
+      const buffer = Buffer.from(await response.arrayBuffer());
+      const kind = sniffDocumentKind(contentType, buffer, source);
+      if (kind === 'pdf') {
+        result.documentType = 'pdf';
+        await this.processPDFDocument(result, buffer, 'pdf_buffer', options, identity);
+        return;
+      }
+      if (kind === 'docx') {
+        result.documentType = 'docx';
+        await this.processDocxBuffer(result, buffer, options);
+        return;
+      }
+      if (kind === 'binary') {
+        throw new Error(
+          `${contentType.split(';')[0] || 'This'} content is not a document this tool reads: ` +
+            'process_document reads PDF, DOCX, HTML and plain text.'
+        );
+      }
+      html = decodeTextBody(buffer, contentType);
       pageTitle = this.extractTitleFromHTML(html);
     }
 
@@ -349,6 +423,25 @@ export class ProcessDocumentTool {
     await this.processFetchedHtml(result, html, source, options);
   }
 
+  /**
+   * Read a Word document: its text (and markdown when asked) via mammoth.
+   * @param {Object} result - Result object to populate
+   * @param {Buffer} buffer - the .docx bytes
+   * @param {Object} options - Processing options
+   */
+  async processDocxBuffer(result, buffer, options) {
+    const mammoth = (await import('mammoth')).default;
+    const raw = await mammoth.extractRawText({ buffer });
+    result.content = { text: (raw.value || '').trim() };
+    if (options.outputFormat === 'markdown') {
+      const md = await mammoth.convertToMarkdown({ buffer });
+      result.content.markdown = md.value || '';
+    }
+    result.title = null;
+    const notes = (raw.messages || []).map((m) => m.message).filter(Boolean);
+    if (notes.length > 0) result.warnings = [...(result.warnings || []), ...notes.slice(0, 5)];
+  }
+
   /**
    * Process a local non-PDF file (sourceType 'file'): read it from disk and
    * run it through the same content-processing pipeline used for web pages.

package/src/tools/scrape/_brandingExtractor.js

@@ -15,6 +15,7 @@
  */
 
 import { safeFetch } from '../../utils/ssrfGuard.js';
+import { mediaAppliesToScreen } from '../../utils/hiddenContent.js';
 
 const GENERIC_FAMILIES = new Set([
   'serif', 'sans-serif', 'monospace', 'cursive', 'fantasy', 'system-ui',
@@ -123,19 +124,36 @@ export async function collectCssSources($, pageUrl, opts) {
   let inlineStyleEls = 0;
   let cssText = '';
 
+  // Print (and other non-screen) stylesheets never paint a screen render, so
+  // neither branding nor the hidden-content pass may read them. `media:
+  // 'unconditional-screen'` also drops sheets gated on a media query.
+  const unconditional = opts.media === 'unconditional-screen';
+  const applies = (el) => mediaAppliesToScreen($(el).attr('media'), { unconditional });
+
   $('style').each((_, el) => {
+    if (!applies(el)) return;
     const t = $(el).html();
     if (t) { cssText += '\n' + t; styleBlocks++; }
   });
 
-  $('[style]').each((_, el) => {
-    const t = $(el).attr('style');
-    if (t) { cssText += '\n*{' + t + '}'; inlineStyleEls++; }
-  });
+  // Inline style="" attributes are folded in as universal rules so branding
+  // can mine their colours and fonts. They are NOT visibility rules: one
+  // element carrying style="display:none" became `*{display:none}`, and the
+  // hidden-content pass then deleted every element small enough to pass its
+  // bulk guard — irs.gov's tax-bracket page came back as its header (R21,
+  // 2026-09-09). That pass reads inline styles per element itself, so it
+  // asks for `inlineStyles: false`.
+  if (opts.inlineStyles !== false) {
+    $('[style]').each((_, el) => {
+      const t = $(el).attr('style');
+      if (t) { cssText += '\n*{' + t + '}'; inlineStyleEls++; }
+    });
+  }
 
   if (opts.fetchLinkedCss) {
     const hrefs = [];
     $('link[rel~="stylesheet"][href]').each((_, el) => {
+      if (!applies(el)) return;
       const href = $(el).attr('href');
       if (href) hrefs.push(resolveUrl(href, pageUrl));
     });
@@ -404,7 +422,7 @@ function extractTokens(cssText, cssVariables) {
  * Extract the full branding object from a loaded cheerio $.
  * @param {import('cheerio').CheerioAPI} $
  * @param {string} pageUrl
- * @param {{ fetchLinkedCss?: boolean, maxStylesheets?: number, perFileTimeoutMs?: number, timeoutMs?: number, overallTimeoutMs?: number, stylesheetConcurrency?: number }} [opts]
+ * @param {{ fetchLinkedCss?: boolean, maxStylesheets?: number, perFileTimeoutMs?: number, timeoutMs?: number, overallTimeoutMs?: number, stylesheetConcurrency?: number, media?: 'screen'|'unconditional-screen', inlineStyles?: boolean }} [opts]
  * @returns {Promise<object>}
  */
 export async function extractBranding($, pageUrl, opts = {}) {

package/src/tools/scrape/_mainContent.js

@@ -172,6 +172,22 @@ function isDataTable(table) {
   return rows >= 10 || columns > 4;
 }
 
+/**
+ * A table whose author marked header cells carries data whatever its size —
+ * layout tables do not use <th>. The size test above misses every small fee
+ * or spec table: WestJet's checked-bag fees are 6 rows × 3 columns inside a
+ * `com-tabs` component (Readability's `negative` regex matches `com-`), and
+ * the page came back reading "fees are as follows:" with nothing following
+ * (R20, 2026-09-07).
+ * @param {HTMLTableElement} table
+ * @returns {boolean}
+ */
+function isHeadedTable(table) {
+  if (table.rows.length < 2) return false;
+  if (!table.querySelector('th')) return false;
+  return Array.from(table.rows).some((row) => row.cells.length >= 2);
+}
+
 /**
  * A data table's text with its structure kept: one line per row, cells joined
  * by " | ". A bare `textContent` runs every cell together
@@ -216,7 +232,7 @@ export function recoverDroppedTables(html, url, keptText = '') {
       // A nested table travels with its parent; re-attaching it separately
       // would duplicate it.
       .filter((table) => !table.parentElement?.closest('table'))
-      .filter(isDataTable)
+      .filter((table) => isDataTable(table) || isHeadedTable(table))
       .filter((table) => {
         const signature = normalizeWhitespace(table.textContent || '').slice(0, SIGNATURE_LENGTH);
         return signature.length > 0 && !kept.includes(signature);

package/src/tools/scrape/unifiedScrape.js

@@ -542,7 +542,14 @@ export class UnifiedScrapeTool {
             // Themes split visibility rules across many component sheets — the
             // rule hiding Shopify's sold-out badge sits at index 12 of 38 on a
             // stock Dawn storefront, so a cap of 10 silently misses it.
-            maxStylesheets: 20
+            maxStylesheets: 20,
+            // Only sheets a screen render applies unconditionally: a
+            // media="print" sheet hid every screen element of irs.gov's
+            // tax-bracket page and left the print logo (R21, 2026-09-09).
+            media: 'unconditional-screen',
+            // stripHiddenFromDom reads style="" per element; folded into
+            // universal rules they hid the whole page (R21).
+            inlineStyles: false
           });
           css = collected.cssText || '';
         }

package/src/tools/search/redditSearch.js

@@ -15,11 +15,14 @@
  *   cross-subreddit full-text search, but has known post-2023 archive gaps
  *   and recurring outages.
  *
- * Routing: scoped searches go to Arctic Shift. An unscoped keyword search
- * finds posts through a site-restricted web search and then reads them — or
- * searches their comments — in the archive by id. PullPush stopped serving
- * automated clients in August 2026 and is used only on explicit request.
- * Both archives are free and need no credentials.
+ * Routing: Arctic Shift first, PullPush second. A scoped search queries
+ * Arctic Shift and, if that fails, PullPush. An unscoped keyword search finds
+ * posts through a site-restricted web search and reads them — or searches
+ * their comments — in Arctic Shift by id, then falls back to PullPush's own
+ * full-text search. Thread mode is Arctic Shift only. PullPush has refused
+ * automated clients since August 2026, so the fallback usually reports that
+ * refusal; it stays second for whenever it answers again. Both archives are
+ * free and need no credentials.
  *
  * Optional official-API path: if the user sets REDDIT_CLIENT_ID and
  * REDDIT_CLIENT_SECRET (their own Reddit app), posts/thread requests can read
@@ -101,9 +104,8 @@ export class RedditSearchTool {
     // Lazily constructed on first official-path use; overridable for tests.
     this._officialAdapter = options.officialAdapter || null;
 
-    // Web discovery serves the one shape no archive can: a keyword search
-    // across all of Reddit. Arctic Shift requires a subreddit or author scope,
-    // and PullPush stopped serving automated clients in August 2026.
+    // Web discovery serves the one shape Arctic Shift cannot: a keyword search
+    // across all of Reddit, which it requires a subreddit or author scope for.
     this.searchAdapter = options.searchAdapter || null;
     this.searchApiKey = options.searchApiKey || null;
     this.searchApiBaseUrl = options.searchApiBaseUrl || null;
@@ -179,13 +181,16 @@ export class RedditSearchTool {
       order = ['pullpush'];
     } else {
       // auto: prefer the user's own official API (live, authoritative) when it
-      // can serve this request, then fall back to the community archives.
-      // PullPush is no longer tried automatically: every request now returns
-      // 429 "This website does not provide free scraping resources for agents",
-      // or a Cloudflare 403 challenge. It stays available on explicit request.
+      // can serve this request, then the community archives — Arctic Shift
+      // first, PullPush second (owner's call, 2026-09-10). PullPush has refused
+      // automated clients since August 2026 — 429 "This website does not
+      // provide free scraping resources for agents", or a Cloudflare 403
+      // challenge — so the fallback is one un-retried request that usually
+      // reports that refusal after the real Arctic Shift error, and answers
+      // when PullPush is serving again. Thread mode has no second source.
       const archives = v.mode === 'thread' ? ['arctic_shift']
-        : arcticPossible ? ['arctic_shift']
-        : discoveryPossible ? ['web_discovery']
+        : arcticPossible ? ['arctic_shift', 'pullpush']
+        : discoveryPossible ? ['web_discovery', 'pullpush']
         : [];
       order = officialPossible ? ['reddit_api', ...archives] : archives;
     }
@@ -252,7 +257,7 @@ export class RedditSearchTool {
         ? 'Reddit-wide comment search: posts were found with a site-restricted web search, then each post\'s comments were searched for the keywords in the Arctic Shift archive, in post relevance order.'
         : 'Reddit-wide keyword search: posts were found with a site-restricted web search, then read from the Arctic Shift archive by ID.',
       'Results are ordered by web-search relevance, not by score or date.',
-      'Arctic Shift cannot keyword-search across all of Reddit, and PullPush no longer serves automated clients — scope the search to a subreddit or author to query the archive directly.',
+      'Arctic Shift cannot keyword-search across all of Reddit — scope the search to a subreddit or author to query the archive directly.',
     ];
     if (v.after || v.before) {
       // Silently dropping a date filter would return results the caller
@@ -373,7 +378,18 @@ export class RedditSearchTool {
       // caller the 422. Thread mode returned above, so both remaining modes
       // (posts, comments) narrow.
       const narrowable = Boolean(v.query) && !v.after;
-      if (!error.retryable || !narrowable) throw error;
+      if (!error.retryable || !narrowable) {
+        // The caller's window is respected, so the 422 is theirs to act on —
+        // but the generic hint ("add subreddit or author") is useless on a
+        // search that is already scoped (R20: r/aviation "737 MAX" after=30d).
+        if (error.retryable && v.query && v.after) {
+          throw new Error(
+            `${error.message} — Arctic Shift timed out searching your after=${v.after} window. ` +
+            'Pass a narrower after (7d, 3d, 1d), drop the query to list the newest posts in the scope, or read one post with mode:"thread" and link_id.'
+          );
+        }
+        throw error;
+      }
       let lastError = error;
       for (const window of ['7d', '3d', '1d']) {
         await new Promise((resolve) => setTimeout(resolve, this.retryDelayMs));
@@ -478,12 +494,14 @@ export class RedditSearchTool {
     if (response.status === 429) {
       const reset = response.headers?.get?.('x-ratelimit-reset');
       // PullPush's 429 body states its actual policy ("does not provide free
-      // scraping resources for agents...") — pass that through verbatim.
+      // scraping resources for agents...") — pass that through verbatim. That
+      // policy refusal is not a transient throttle: retrying it only spends the
+      // delay, which matters now that PullPush is the automatic second source.
       let detail = '';
       try { detail = (await response.json())?.error ?? ''; } catch { /* no body */ }
       throw Object.assign(
         new Error(`rate limited (429)${reset ? `, retry in ${reset}s` : ''}${detail ? ` — ${detail}` : ''}`),
-        { retryable: true },
+        { retryable: !/does not provide free scraping/i.test(detail) },
       );
     }
     if (!response.ok) {

package/src/utils/hiddenContent.js

@@ -83,6 +83,38 @@ function stripConditionalBlocks(css) {
   return out;
 }
 
+/**
+ * Split a selector list on its top-level commas only. A naive split cut
+ * irs.gov's `.callout:has(> .a, .b, p, ul, h2, h3, table)` into bare `p`,
+ * `ul`, `h2`, `h3` and `table` rules carrying its `display:none !important`,
+ * and the stripper then deleted the article body (R21, 2026-09-09).
+ * @param {string} list
+ * @returns {string[]}
+ */
+export function splitSelectorList(list) {
+  const out = [];
+  let depth = 0;
+  let quote = null;
+  let start = 0;
+  for (let i = 0; i < list.length; i++) {
+    const ch = list[i];
+    if (quote) {
+      if (ch === '\\') i++;
+      else if (ch === quote) quote = null;
+      continue;
+    }
+    if (ch === '"' || ch === "'") quote = ch;
+    else if (ch === '(' || ch === '[') depth++;
+    else if (ch === ')' || ch === ']') depth = Math.max(0, depth - 1);
+    else if (ch === ',' && depth === 0) {
+      out.push(list.slice(start, i));
+      start = i + 1;
+    }
+  }
+  out.push(list.slice(start));
+  return out;
+}
+
 /** True when cheerio cannot meaningfully evaluate the selector. */
 function isUnsupportedSelector(selector) {
   return (
@@ -91,7 +123,7 @@ function isUnsupportedSelector(selector) {
     // case and these rules would hide content every real visitor sees.
     /(^|[\s.#\[])no-js(\b|[.\[])/.test(selector) ||
     selector.includes('::') ||
-    /:(hover|focus|focus-within|focus-visible|active|target|checked|disabled|placeholder|before|after|root|host|where|is|not\()/i.test(selector) ||
+    /:(hover|focus|focus-within|focus-visible|active|target|checked|disabled|placeholder|before|after|root|host|where|is|not\(|has\()/i.test(selector) ||
     selector.includes('@') ||
     selector.length === 0
   );
@@ -199,7 +231,7 @@ export function collectVisibilitySelectors(css) {
 
     const important = /!\s*important/i.test(body);
 
-    for (const raw of selectorList.split(',')) {
+    for (const raw of splitSelectorList(selectorList)) {
       const selector = raw.trim();
       if (isUnsupportedSelector(selector) || NEVER_REMOVE.has(selector)) continue;
       const entry = { selector, spec: specificity(selector), order, important };
@@ -249,12 +281,45 @@ function renderedSize($, el) {
 export function inlineStyleText($) {
   const parts = [];
   $('style').each((_, el) => {
+    // A <style media="print"> block is only consulted when printing.
+    if (!mediaAppliesToScreen($(el).attr('media'), { unconditional: true })) return;
     const text = $(el).html();
     if (text) parts.push(text);
   });
   return parts.join('\n');
 }
 
+/**
+ * Whether a stylesheet's `media` attribute applies to a screen render.
+ *
+ * `<link rel="stylesheet" media="print">` is consulted only when printing.
+ * irs.gov's tax-bracket page ships one that hides every screen element and
+ * shows a print-only logo; read as screen CSS it emptied the page to that logo
+ * (R21, 2026-09-09). Absent or empty, `all`, `screen` and `not print` apply.
+ * `print`, `speech` and the other non-screen types, and `not screen`/`not all`,
+ * do not. With `unconditional: true` a media QUERY (`screen and (max-width:
+ * 600px)`, a bare `(prefers-color-scheme: dark)`) does not apply either — the
+ * hidden-content pass ignores @media blocks for the same reason.
+ *
+ * @param {string|undefined} media - the attribute value
+ * @param {{ unconditional?: boolean }} [options]
+ * @returns {boolean}
+ */
+export function mediaAppliesToScreen(media, { unconditional = false } = {}) {
+  const value = (media || '').trim().toLowerCase();
+  if (!value) return true;
+  return value.split(',').some((part) => {
+    const query = part.trim();
+    if (!query) return false;
+    if (unconditional && query.includes('(')) return false;
+    if (query.startsWith('(')) return true; // a bare feature query: type "all"
+    const m = query.match(/^(?:(not|only)\s+)?([a-z-]+)/);
+    if (!m) return true;
+    const screenType = m[2] === 'all' || m[2] === 'screen';
+    return m[1] === 'not' ? !screenType : screenType;
+  });
+}
+
 /**
  * Remove browser-invisible content from a cheerio document, in place.
  *