crawlforge-mcp-server 6.4.0 → 6.5.0

package/README.md

@@ -48,7 +48,7 @@
 | | **CrawlForge MCP** | Firecrawl | Raw scraping API |
 |---|:---:|:---:|:---:|
 | Native MCP server | ✅ 30 tools | ✅ | ❌ |
-| Free tier | ✅ 1,000 credits, rollover | Limited | Varies |
+| Free tier | ✅ 1,000 credits, one-time, never expire | Limited | Varies |
 | Self-hosted / local LLM extraction (Ollama) | ✅ default, $0/token | ❌ | ❌ |
 | Autonomous agent (no URLs needed) | ✅ `agent` | ✅ | ❌ |
 | Deep research with source verification | ✅ `deep_research` | Partial | ❌ |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.4.0",
+  "version": "6.5.0",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 30 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -115,6 +115,7 @@
     "@modelcontextprotocol/node": "^2.0.0",
     "@modelcontextprotocol/server": "^2.0.0",
     "@mozilla/readability": "^0.6.0",
+    "mammoth": "^1.12.2",
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",

package/server.js

@@ -107,7 +107,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.4.0",
+  version: "6.5.0",
   description: "Production-ready MCP server with 30 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -541,7 +541,7 @@ registerToolIfEnabled("serp_rank", {
 
 // Tool: reddit_search — search Reddit posts/comments or read a full thread (via community archives)
 registerToolIfEnabled("reddit_search", {
-  description: "Use this to search Reddit posts or comments, or read a full comment thread - reddit.com blocks direct scraping, so this reads the Arctic Shift community archive instead (free, no Reddit credentials). Modes: 'posts' (default) and 'comments' search; 'thread' returns a post plus its nested comment tree by link_id. A subreddit/author-scoped search queries the archive directly. A keyword search across ALL of Reddit finds posts with a site-restricted web search and then reads those posts from the archive, because Arctic Shift can only keyword-search within a scope; results come back as real archive rows, ordered by search relevance. An unscoped COMMENT search discovers posts the same way and then searches each post's comments for the keywords. A scoped comment search Arctic Shift times out on is retried over narrower windows (7d, 3d, 1d) and reports window_applied. Not for reddit.com URLs via scrape or fetch_url (blocked) - use mode:'thread' with the post's link_id. Cost: 5 credits. Example: reddit_search({query: \"best mechanical keyboard\", subreddit: \"MechanicalKeyboards\", limit: 10})",
+  description: "Use this to search Reddit posts or comments, or read a full comment thread - reddit.com blocks direct scraping, so this reads the Arctic Shift community archive instead (free, no Reddit credentials). Modes: 'posts' (default) and 'comments' search; 'thread' returns a post plus its nested comment tree by link_id. A subreddit/author-scoped search queries the archive directly. A keyword search across ALL of Reddit finds posts with a site-restricted web search and then reads those posts from the archive, because Arctic Shift can only keyword-search within a scope; results come back as real archive rows, ordered by search relevance. An unscoped COMMENT search discovers posts the same way and then searches each post's comments for the keywords. A scoped comment search Arctic Shift times out on is retried over narrower windows (7d, 3d, 1d) and reports window_applied. Arctic Shift is tried first and the PullPush archive second for posts/comments searches (fallback_used says so; PullPush has refused automated clients since August 2026). Not for reddit.com URLs via scrape or fetch_url (blocked) - use mode:'thread' with the post's link_id. Cost: 5 credits. Example: reddit_search({query: \"best mechanical keyboard\", subreddit: \"MechanicalKeyboards\", limit: 10})",
   annotations: { title: "Reddit Search", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     query: z.string().optional().describe("Keyword search. Posts: matches title+selftext; comments: matches body. Supports \"quoted phrases\", OR, -exclusion"),
@@ -680,7 +680,7 @@ registerToolIfEnabled("extract_content", {
 
 // Tool: process_document
 registerToolIfEnabled("process_document", {
-  description: "Use this to extract text from a PDF URL or file - research papers, contracts, reports. Returns structured sections, metadata, and word count. Not for ordinary web pages (scrape), though an HTML URL is accepted. Cost: 2 credits. Example: process_document({source: \"https://example.com/report.pdf\", sourceType: \"pdf_url\"})",
+  description: "Use this to extract text from a PDF or DOCX URL or file - research papers, contracts, reports. The body decides how it is read: a PDF or Word document served under sourceType \"url\" still reaches its parser, and a body this tool cannot read (an image, an archive) is refused by name. Returns structured sections, metadata, and word count. Not for ordinary web pages (scrape), though an HTML URL is accepted. Cost: 2 credits. Example: process_document({source: \"https://example.com/report.pdf\", sourceType: \"pdf_url\"})",
   annotations: { title: "Process Document", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     source: z.string().describe("Document source - URL or file path"),
@@ -991,11 +991,15 @@ registerToolIfEnabled("scrape_with_actions", {
 
     // Publish captured screenshots as crawlforge://screenshot/{actionId}
     // resources (the documented contract) and annotate each with its URI.
+    // The base64 `data` is dropped once stored, as the stealth_mode block
+    // below already does: a failed chain's error screenshot came back inline
+    // as 1.7 MB of base64 beside a 40-char markdown (R21, 2026-09-09).
     if (Array.isArray(result.screenshots)) {
       result.screenshots = result.screenshots.map((shot) => {
         if (shot?.actionId && shot?.data) {
           resourceRegistry.storeScreenshot(shot.actionId, shot.data);
-          return { ...shot, resourceUri: `crawlforge://screenshot/${shot.actionId}` };
+          const { data, ...rest } = shot;
+          return { ...rest, resourceUri: `crawlforge://screenshot/${shot.actionId}` };
         }
         return shot;
       });

package/src/core/ActionExecutor.js

@@ -314,7 +314,15 @@ export class ActionExecutor extends EventEmitter {
               fullPage: true,
               description: 'Error screenshot'
             });
-            executionContext.screenshots.push(errorScreenshot);
+            // An actionId is what lets the server publish the shot as a
+            // crawlforge://screenshot/{actionId} resource and drop the base64
+            // from the result; without one a failed chain shipped 1.7 MB of
+            // PNG inline (R21, 2026-09-09).
+            executionContext.screenshots.push({
+              ...errorScreenshot,
+              actionId: this.generateActionId(),
+              error: true
+            });
           } catch (screenshotError) {
             this.log('warn', 'Failed to capture error screenshot: ' + screenshotError.message);
           }

package/src/core/ChangeTracker.js

@@ -720,14 +720,16 @@ export class ChangeTracker extends EventEmitter {
     significanceScore += Math.min(totalElements * 0.05, 1) * 
       (weights.additions + weights.removals + weights.modifications);
     
-    // Text changes impact
-    if (changeAnalysis.textChanges.length > 0) {
-      const textChangeRatio = changeAnalysis.textChanges.reduce(
-        (sum, change) => sum + (change.added?.length || 0) + (change.removed?.length || 0),
-        0
-      ) / 1000; // Normalize by character count
-      
-      significanceScore += Math.min(textChangeRatio, 1) * weights.textChanges;
+    // Text changes impact. textChanges holds diff GROUPS — {type:'word_diff',
+    // changes:[{added, removed, value}]} — not flat parts, so reading
+    // `change.added.length` off a group was always 0 and this term never
+    // fired: a feed that grew by a whole record scored on similarity alone.
+    // The USGS all-hour earthquake feed gained an event (622 words, 84%
+    // similar) and compare reported hasChanges:false, "No significant
+    // changes detected" (R21, 2026-09-09).
+    const changedChars = this.changedTextChars(changeAnalysis.textChanges);
+    if (changedChars > 0) {
+      significanceScore += Math.min(changedChars / 1000, 1) * weights.textChanges;
     }
     
     // Determine significance level
@@ -1317,6 +1319,21 @@ export class ChangeTracker extends EventEmitter {
     );
   }
 
+  /**
+   * Characters added or removed at the text level, for significance scoring.
+   * Same word-diff-then-line-diff choice as countTextChanges, and for the
+   * same reason: the two describe one edit.
+   */
+  changedTextChars(textChanges = []) {
+    const diff = textChanges.find(c => c.type === 'word_diff')
+      || textChanges.find(c => c.type === 'line_diff');
+    if (!diff) return 0;
+    return diff.changes.reduce(
+      (chars, part) => chars + ((part.added || part.removed) && typeof part.value === 'string' ? part.value.length : 0),
+      0
+    );
+  }
+
   generateChangeSummary(changeAnalysis, significance) {
     const { addedElements, removedElements, modifiedElements, similarity } = changeAnalysis;
 

package/src/core/llm/LLMManager.js

@@ -27,6 +27,70 @@ function endsMidJson(text) {
   return !trimmed.endsWith('}') && !trimmed.endsWith(']');
 }
 
+/**
+ * Recover the complete rows of a JSON response the model stopped writing.
+ *
+ * Walks the text tracking strings and the bracket stack, remembers the end of
+ * every complete element written directly inside an array, cuts there and
+ * closes what is still open. Returns null when no element completed, so a
+ * caller can still fall back. A schema asking for a table of rows routinely
+ * overruns a small model's output budget — the ECB key-rates table came back
+ * cut off at 1,800 and again at 3,600 tokens and the whole extraction failed,
+ * although dozens of rows were complete (R21, 2026-09-09).
+ *
+ * @param {string} text
+ * @returns {{ data: unknown, rows: number }|null}
+ */
+export function salvageTruncatedJson(text) {
+  const src = text.trim();
+  if (!src.startsWith('{') && !src.startsWith('[')) return null;
+  const stack = [];
+  let inString = false;
+  let cut = -1;
+  for (let i = 0; i < src.length; i++) {
+    const ch = src[i];
+    if (inString) {
+      if (ch === '\\') i++;
+      else if (ch === '"') {
+        inString = false;
+        // A string written directly into an array is a complete element.
+        if (stack[stack.length - 1] === '[') cut = i + 1;
+      }
+      continue;
+    }
+    if (ch === '"') inString = true;
+    else if (ch === '{' || ch === '[') stack.push(ch);
+    else if (ch === '}' || ch === ']') {
+      stack.pop();
+      if (stack[stack.length - 1] === '[') cut = i + 1;
+    }
+  }
+  if (cut < 0) return null;
+  // Close everything still open at the cut, innermost first.
+  const open = [];
+  inString = false;
+  for (let i = 0; i < cut; i++) {
+    const ch = src[i];
+    if (inString) { if (ch === '\\') i++; else if (ch === '"') inString = false; continue; }
+    if (ch === '"') inString = true;
+    else if (ch === '{' || ch === '[') open.push(ch);
+    else if (ch === '}' || ch === ']') open.pop();
+  }
+  const closers = open.reverse().map((c) => (c === '{' ? '}' : ']')).join('');
+  let data;
+  try {
+    data = JSON.parse(src.slice(0, cut) + closers);
+  } catch {
+    return null;
+  }
+  let rows = 0;
+  (function count(v) {
+    if (Array.isArray(v)) { rows += v.length; v.forEach(count); }
+    else if (v && typeof v === 'object') Object.values(v).forEach(count);
+  })(data);
+  return rows > 0 ? { data, rows } : null;
+}
+
 /**
  * LLM Manager
  * Manages multiple LLM providers and provides unified interface
@@ -987,12 +1051,25 @@ Extract the data and return valid JSON:`;
         try {
           parsed = JSON.parse(cleaned);
         } catch (parseError) {
+          if (!endsMidJson(cleaned)) throw parseError;
           // Name the cause when the JSON simply stops: "unexpected end of
           // input at position 2608" tells a caller nothing they can act on,
           // whereas "cut off at the 1000-token limit" points at the schema.
-          throw endsMidJson(cleaned)
-            ? new Error(`model response was cut off at the ${budget}-token output limit (${cleaned.length} chars) — the schema asks for more rows than fit`)
-            : parseError;
+          const cutOff = `model response was cut off at the ${budget}-token output limit (${cleaned.length} chars) — the schema asks for more rows than fit`;
+          // The doubled budget was the retry; when that is cut off too, the
+          // rows it did complete are worth more than a fallback that has none.
+          const salvaged = attempt > 0 ? salvageTruncatedJson(cleaned) : null;
+          if (!salvaged) throw new Error(cutOff);
+          const validation = this.validateAgainstSchema(salvaged.data, schema);
+          this.logger.warn('LLM structured extraction salvaged a cut-off response', { budget, rows: salvaged.rows });
+          return {
+            data: salvaged.data,
+            method: 'llm',
+            valid: validation.valid,
+            validationErrors: validation.errors,
+            partial: true,
+            warning: `${cutOff}; kept the ${salvaged.rows} complete row(s) it had written. Ask for fewer rows (a prompt naming the rows you need, or a narrower schema) to get the rest.`
+          };
         }
 
         const validation = this.validateAgainstSchema(parsed, schema);
@@ -1016,7 +1093,7 @@ Extract the data and return valid JSON:`;
     // Report which path produced the data. Callers previously labelled this
     // result "llm", so a failed LLM call was returned as a high-confidence
     // LLM extraction.
-    return { ...this.fallbackStructuredExtraction(content, schema), error: lastError.message };
+    return { ...this.fallbackStructuredExtraction(content, schema, lastError.message), error: lastError.message };
   }
 
   /**
@@ -1036,7 +1113,7 @@ Extract the data and return valid JSON:`;
   /**
    * Fallback structured extraction without LLM — keyword/regex matching for primitives
    */
-  fallbackStructuredExtraction(content, schema) {
+  fallbackStructuredExtraction(content, schema, reason = 'no LLM provider available') {
     const extracted = {};
     const properties = schema.properties || {};
 
@@ -1065,7 +1142,10 @@ Extract the data and return valid JSON:`;
       data: extracted,
       method: 'keyword_fallback',
       valid: false,
-      validationErrors: ['Used fallback extraction — no LLM provider available']
+      // Name the real reason: this fallback also runs after an LLM attempt
+      // that failed, and "no LLM provider available" beside a working Ollama
+      // sent a caller looking at the wrong thing (R21, 2026-09-09).
+      validationErrors: [`Used fallback extraction — ${reason}`]
     };
   }
 

package/src/core/processing/PDFProcessor.js

@@ -12,7 +12,9 @@ import { config } from '../../constants/config.js';
 import { identityHeaders } from '../../utils/fetchIdentity.js';
 
 const PDFProcessorSchema = z.object({
-  source: z.string().min(1),
+  // A Buffer is the 'buffer' source: process_document hands over a body it
+  // has already fetched and sniffed as a PDF (R21, 2026-09-09).
+  source: z.union([z.string().min(1), z.instanceof(Buffer)]),
   sourceType: z.enum(['url', 'file', 'buffer']).default('url'),
   options: z.object({
     extractMetadata: z.boolean().default(true),

package/src/server/fallbackHints.js

@@ -80,6 +80,9 @@ export function appendFallbackHint(toolName, result) {
     // plain text
   }
 
+  // An error that already names its next step — a reddit.com refusal points
+  // at reddit_search — keeps it: the tool's generic hint would contradict it.
+  if (parsed?.next_step || /\bNext step:/.test(first.text)) return result;
   const hint = hintFor(toolName, parsed);
   if (!hint || first.text.includes(hint)) return result;
   if (parsed) {

package/src/server/inlineThreshold.js

@@ -72,6 +72,27 @@ export function resultTextView(resultObject, textPaths = []) {
   return { view: 'json', view_path: null, text: JSON.stringify(resultObject, null, 2) };
 }
 
+/**
+ * The non-text fields of `content` small enough to stay inline: everything
+ * except the text views (markdown, text, html, …) up to a quarter of the
+ * inline budget each. Null when there is nothing to keep.
+ */
+export function keepSmallContentFields(content, textPaths = [], maxInline = DEFAULT_MAX_INLINE_CHARS) {
+  if (!content || typeof content !== 'object' || Array.isArray(content)) return null;
+  const textLeaves = new Set(
+    textPaths.filter((p) => p.startsWith('content.')).map((p) => p.slice('content.'.length))
+  );
+  const cap = Math.max(1000, Math.floor(maxInline / 4));
+  const kept = {};
+  for (const [key, value] of Object.entries(content)) {
+    if (textLeaves.has(key) || value === undefined) continue;
+    if (typeof value === 'string' && value.length > cap) continue;
+    if (typeof value === 'object' && value !== null && JSON.stringify(value).length > cap) continue;
+    kept[key] = value;
+  }
+  return Object.keys(kept).length > 0 ? kept : null;
+}
+
 function warningsOf(resultObject) {
   return Array.isArray(resultObject.warnings) ? resultObject.warnings.filter((w) => typeof w === 'string') : [];
 }
@@ -129,7 +150,9 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
   }
 
   const preview = text.slice(0, maxInline);
-  const hint = `Result is ${json.length} chars as JSON, over the inline limit of ${maxInline}; preview holds the first ${preview.length} chars of ${viewDesc} (${text.length} chars in total) and the full result is kept for 1 hour under result_handle ${handle}: ${readWith}.`;
+  const keptFields = keepSmallContentFields(resultObject.content, config.textPaths, maxInline);
+  const keptDesc = keptFields ? `; content.${Object.keys(keptFields).join(', content.')} kept inline` : '';
+  const hint = `Result is ${json.length} chars as JSON, over the inline limit of ${maxInline}; preview holds the first ${preview.length} chars of ${viewDesc} (${text.length} chars in total)${keptDesc} and the full result is kept for 1 hour under result_handle ${handle}: ${readWith}.`;
 
   const shaped = {};
   for (const [key, value] of Object.entries(resultObject)) {
@@ -143,6 +166,13 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
   if (resultObject.redaction && typeof resultObject.redaction === 'object') {
     shaped.redaction = resultObject.redaction;
   }
+  // The query-scoped formats live beside the page text under `content`
+  // (highlights, answer, json, metadata, links). They are the small, exact
+  // answer the caller paid for, and truncating the markdown must not drop
+  // them: an nhs.uk scrape with highlights and a question came back as a
+  // markdown preview and nothing else (R21, 2026-09-09). Keep every
+  // non-text `content` field that fits a quarter of the inline budget.
+  if (keptFields) shaped.content = keptFields;
   Object.assign(shaped, {
     preview,
     result_handle: handle,

package/src/server/requestContext.js

@@ -36,7 +36,7 @@ export function isInternalRequest() {
  * reaches withAuth. It also survives both routes a refusal can take — thrown,
  * or swallowed into an isError result.
  *
- * @param {string} code 'ROBOTS_DISALLOWED' | 'HOST_BLOCKED'
+ * @param {string} code 'ROBOTS_DISALLOWED' | 'HOST_BLOCKED' | 'USE_REDDIT_SEARCH'
  */
 export function markPreflightRefusal(code) {
   const store = requestContext.getStore();

package/src/tools/crawl/mapSite.js

@@ -10,6 +10,44 @@ import { CRAWLFORGE_USER_AGENT } from '../../utils/fetchIdentity.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
 import { pageTitle } from '../../utils/pageTitle.js';
 
+/**
+ * The path prefix a seed URL asks for: nps.gov/yell/ means the Yellowstone
+ * subtree, not the first 200 URLs of the park service's site-wide sitemap
+ * (R21, 2026-09-09: map_site returned Abraham Lincoln Birthplace pages for it).
+ * A page URL scopes to its directory; the site root scopes to nothing.
+ * @param {string} url
+ * @returns {string|null} e.g. "/yell/", or null for the root
+ */
+export function scopePathOf(url) {
+  try {
+    const { pathname } = new URL(url);
+    const dir = pathname.endsWith('/') ? pathname : pathname.slice(0, pathname.lastIndexOf('/') + 1);
+    return dir && dir !== '/' ? dir : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * How many of a search's terms a URL's own path and query contain — the
+ * relevance signal the generic ranker lacks when every candidate is a bare
+ * URL (it scored 0.19 for all 200 nps.gov URLs of a "fees" search, none of
+ * which mentioned fees).
+ * @param {string} url
+ * @param {string} search
+ * @returns {number}
+ */
+export function searchScore(url, search) {
+  const terms = [...new Set(String(search || '').toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((t) => t.length >= 2))];
+  if (terms.length === 0) return 0;
+  let haystack = url.toLowerCase();
+  try {
+    const { pathname, search: query } = new URL(url);
+    haystack = decodeURIComponent(pathname + query).toLowerCase();
+  } catch { /* keep the raw url */ }
+  return terms.filter((t) => haystack.includes(t)).length;
+}
+
 // Lazy singleton — avoids creating a CacheManager timer per request
 let _ranker = null;
 function getRanker() {
@@ -107,22 +145,47 @@ export class MapSiteTool {
         }
       }
 
+      // A seed with a path asks for that subtree, and a search needs a pool
+      // wider than max_urls to rank — otherwise the cut falls before the
+      // relevant URLs ever enter (the sitemap head is alphabetical).
+      const scopePath = scopePathOf(validated.url);
+      const widen = scopePath || validated.search;
+      const poolLimit = widen ? Math.min(10000, Math.max(validated.max_urls * 10, 2000)) : validated.max_urls;
+      const warnings = [];
+
       // Try to fetch sitemap first
       if (validated.include_sitemap) {
-        const sitemapUrls = await this.fetchSitemapUrls(baseUrl, domainFilter, validated.max_urls);
+        const sitemapUrls = await this.fetchSitemapUrls(baseUrl, domainFilter, poolLimit, scopePath);
         sitemapUrls.forEach(url => urls.add(normalizeUrl(url)));
       }
 
       // Fetch and parse the main page for additional URLs
       const pageUrls = await this.fetchPageUrls(validated.url, domainFilter, identity);
       pageUrls.forEach(url => {
-        if (urls.size < validated.max_urls) {
+        if (urls.size < poolLimit) {
           urls.add(normalizeUrl(url));
         }
       });
 
+      let pool = Array.from(urls);
+      if (scopePath) {
+        const inScope = pool.filter((u) => { try { return new URL(u).pathname.startsWith(scopePath); } catch { return false; } });
+        if (inScope.length > 0) {
+          pool = inScope;
+        } else {
+          warnings.push(`No URL under ${scopePath} was found in the sitemap or on the page; the whole site is listed instead.`);
+        }
+      }
+      if (validated.search) {
+        // Stable: relevance first, discovery order among equals.
+        pool = pool
+          .map((url, i) => ({ url, i, score: searchScore(url, validated.search) }))
+          .sort((a, b) => b.score - a.score || a.i - b.i)
+          .map((x) => x.url);
+      }
+
       // Convert to array and limit
-      const urlArray = Array.from(urls).slice(0, validated.max_urls);
+      const urlArray = pool.slice(0, validated.max_urls);
 
       // Fetch metadata if requested
       if (validated.include_metadata) {
@@ -142,7 +205,9 @@ export class MapSiteTool {
         site_map: this.generateSiteMap(urlArray),
         statistics: this.generateStatistics(urlArray),
         domain_filter_config: domainFilter ? domainFilter.exportConfig() : null,
-        filter_stats: domainFilter ? domainFilter.getStats() : null
+        filter_stats: domainFilter ? domainFilter.getStats() : null,
+        ...(scopePath ? { scope: scopePath } : {}),
+        ...(warnings.length ? { warnings } : {})
       };
 
       // Optional: rank URLs by relevance to a search string
@@ -157,7 +222,11 @@ export class MapSiteTool {
             return { link: url, title, snippet: '' };
           });
           const ranked = await getRanker().rankResults(rankerInput, validated.search);
-          result.ranked_urls = ranked.map(r => ({ url: r.link, score: r.finalScore ?? 0 }));
+          // The ranker's score is flat across bare URLs; the path-term count
+          // is what separates /yell/planyourvisit/fees.htm from the rest.
+          result.ranked_urls = ranked
+            .map(r => ({ url: r.link, score: Number(((r.finalScore ?? 0) + searchScore(r.link, validated.search)).toFixed(3)) }))
+            .sort((a, b) => b.score - a.score);
         } catch {
           // ranking is best-effort; don't fail the whole call
           result.ranked_urls = urlArray.map(u => ({ url: u, score: 0 }));
@@ -193,7 +262,11 @@ export class MapSiteTool {
     });
   }
 
-  async fetchSitemapUrls(baseUrl, domainFilter = null, maxUrls = Infinity) {
+  async fetchSitemapUrls(baseUrl, domainFilter = null, maxUrls = Infinity, scopePath = null) {
+    const inScope = (url) => {
+      if (!scopePath) return true;
+      try { return new URL(url).pathname.startsWith(scopePath); } catch { return false; }
+    };
     // Discover sitemaps via robots.txt and common paths, then parse with full
     // SitemapParser support (sitemap-index recursion, gzip, CDATA/entities).
     const discovered = await this.sitemapParser.discoverSitemaps(baseUrl, {
@@ -212,6 +285,7 @@ export class MapSiteTool {
         if (parsed.success) {
           for (const entry of parsed.urls) {
             const url = entry.loc || entry;
+            if (!inScope(url)) continue;
             if (!domainFilter || domainFilter.isAllowed(url).allowed) {
               urls.add(url);
             }
@@ -267,7 +341,7 @@ export class MapSiteTool {
       // A gate refusal is the answer to the request, not a page we failed to
       // read: surface it instead of returning an emptier map than the caller
       // would notice.
-      if (error.code === 'ROBOTS_DISALLOWED' || error.code === 'HOST_BLOCKED') throw error;
+      if (error.code === 'ROBOTS_DISALLOWED' || error.code === 'HOST_BLOCKED' || error.code === 'USE_REDDIT_SEARCH') throw error;
       return [];
     }
   }

package/src/tools/extract/extractEmbeddedState.js

@@ -41,7 +41,23 @@ export async function extractEmbeddedStateHandler({ url, path, user_agent, respe
       );
     }
 
-    const data = path ? selectJsonPath(state.data, path) : state.data;
+    // A path is naturally written against the payload ("props.pageProps"),
+    // not against this tool's envelope ("next_data.props.pageProps"). When
+    // the page carries exactly one payload and the path's root is not one of
+    // the envelope keys, read it inside that payload and say so (R21,
+    // 2026-09-09: four Next.js pages in a row failed on the bare path).
+    let effectivePath = path;
+    if (path && state.found.length === 1) {
+      const root = path.split(/[.[]/)[0];
+      if (root && !(root in state.data)) {
+        effectivePath = `${state.found[0].name}.${path}`;
+        warnings.push(
+          `path "${path}" was read as "${effectivePath}": "${state.found[0].name}" is the only payload on this page, so the path is resolved inside it.`
+        );
+      }
+    }
+
+    const data = effectivePath ? selectJsonPath(state.data, effectivePath) : state.data;
     const bytes = Buffer.byteLength(JSON.stringify(data) ?? '');
 
     if (!path && bytes > LARGE_RESULT_BYTES) {
@@ -57,7 +73,7 @@ export async function extractEmbeddedStateHandler({ url, path, user_agent, respe
         text: JSON.stringify({
           url: finalUrl,
           found: state.found,
-          path: path || null,
+          path: effectivePath || null,
           bytes,
           data,
           warnings

package/src/tools/extract/extractStructured.js

@@ -258,6 +258,9 @@ export class ExtractStructuredTool {
           if (result?.method === 'llm') {
             extractionResult = result;
             extractionMethod = 'llm';
+            // A salvaged cut-off response: the rows are real, the caller must
+            // know the list is not the whole table (R21, 2026-09-09).
+            if (result.warning) warnings.push(result.warning);
           } else {
             llmErrorMessage = result?.error || 'LLM did not return usable JSON';
           }
@@ -433,6 +436,7 @@ export class ExtractStructuredTool {
         },
         extractionNotes,
         provenance,
+        ...(extractionResult.partial ? { partial: true } : {}),
         ...(warnings?.length ? { warnings } : {})
       };
 

package/src/tools/extract/processDocument.js

@@ -118,6 +118,57 @@ const ProcessDocumentResult = z.object({
   error: z.string().optional()
 });
 
+/**
+ * What a fetched body is, from its Content-Type, magic bytes and URL:
+ * 'pdf', 'docx', 'binary' (something this tool cannot read) or 'html'
+ * (HTML, XML, JSON and plain text all go through the page pipeline).
+ * @param {string} contentType
+ * @param {Buffer} buffer
+ * @param {string} [url]
+ * @returns {'pdf'|'docx'|'binary'|'html'}
+ */
+export function sniffDocumentKind(contentType = '', buffer, url = '') {
+  const type = String(contentType).split(';')[0].trim().toLowerCase();
+  const head = buffer.subarray(0, 8).toString('latin1');
+  let path = String(url).toLowerCase();
+  try { path = new URL(url).pathname.toLowerCase(); } catch { /* keep the raw string */ }
+
+  if (type === 'application/pdf' || head.startsWith('%PDF')) return 'pdf';
+
+  const zip = head.startsWith('PK\u0003\u0004');
+  if (
+    type === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' ||
+    (zip && (path.endsWith('.docx') || buffer.includes('word/document.xml')))
+  ) return 'docx';
+
+  if (zip) return 'binary';
+  if (/^(image|audio|video|font)\//.test(type)) return 'binary';
+  if (/^application\/(x-)?(zip|gzip|tar|7z|rar|msword|vnd\.)/.test(type)) return 'binary';
+  const textual = type.startsWith('text/') || /html|xml|json|javascript/.test(type);
+  if (!textual && buffer.subarray(0, 512).includes(0)) return 'binary';
+  return 'html';
+}
+
+/**
+ * Decode a text body by its Content-Type charset, else the <meta charset>
+ * it declares, else UTF-8.
+ * @param {Buffer} buffer
+ * @param {string} contentType
+ * @returns {string}
+ */
+export function decodeTextBody(buffer, contentType = '') {
+  let charset = (/charset=["']?([\w-]+)/i.exec(contentType) || [])[1];
+  if (!charset) {
+    const head = buffer.subarray(0, 4096).toString('latin1');
+    charset = (/<meta[^>]+charset=["']?([\w-]+)/i.exec(head) || [])[1];
+  }
+  try {
+    return new TextDecoder(charset || 'utf-8').decode(buffer);
+  } catch {
+    return buffer.toString('utf8');
+  }
+}
+
 export class ProcessDocumentTool {
   constructor() {
     this.pdfProcessor = new PDFProcessor();
@@ -340,7 +391,30 @@ export class ProcessDocumentTool {
         throw new Error(`HTTP ${response.status}: ${response.statusText}`);
       }
 
-      html = await response.text();
+      // What came back decides how it is read, not the sourceType the caller
+      // guessed: a .docx fetched as 'url' was run through the HTML pipeline
+      // and returned its ZIP bytes as page text, success:true (R21,
+      // 2026-09-09). PDFs reach their parser the same way.
+      const contentType = response.headers.get('content-type') || '';
+      const buffer = Buffer.from(await response.arrayBuffer());
+      const kind = sniffDocumentKind(contentType, buffer, source);
+      if (kind === 'pdf') {
+        result.documentType = 'pdf';
+        await this.processPDFDocument(result, buffer, 'pdf_buffer', options, identity);
+        return;
+      }
+      if (kind === 'docx') {
+        result.documentType = 'docx';
+        await this.processDocxBuffer(result, buffer, options);
+        return;
+      }
+      if (kind === 'binary') {
+        throw new Error(
+          `${contentType.split(';')[0] || 'This'} content is not a document this tool reads: ` +
+            'process_document reads PDF, DOCX, HTML and plain text.'
+        );
+      }
+      html = decodeTextBody(buffer, contentType);
       pageTitle = this.extractTitleFromHTML(html);
     }
 
@@ -349,6 +423,25 @@ export class ProcessDocumentTool {
     await this.processFetchedHtml(result, html, source, options);
   }
 
+  /**
+   * Read a Word document: its text (and markdown when asked) via mammoth.
+   * @param {Object} result - Result object to populate
+   * @param {Buffer} buffer - the .docx bytes
+   * @param {Object} options - Processing options
+   */
+  async processDocxBuffer(result, buffer, options) {
+    const mammoth = (await import('mammoth')).default;
+    const raw = await mammoth.extractRawText({ buffer });
+    result.content = { text: (raw.value || '').trim() };
+    if (options.outputFormat === 'markdown') {
+      const md = await mammoth.convertToMarkdown({ buffer });
+      result.content.markdown = md.value || '';
+    }
+    result.title = null;
+    const notes = (raw.messages || []).map((m) => m.message).filter(Boolean);
+    if (notes.length > 0) result.warnings = [...(result.warnings || []), ...notes.slice(0, 5)];
+  }
+
   /**
    * Process a local non-PDF file (sourceType 'file'): read it from disk and
    * run it through the same content-processing pipeline used for web pages.

package/src/tools/scrape/_brandingExtractor.js

@@ -15,6 +15,7 @@
  */
 
 import { safeFetch } from '../../utils/ssrfGuard.js';
+import { mediaAppliesToScreen } from '../../utils/hiddenContent.js';
 
 const GENERIC_FAMILIES = new Set([
   'serif', 'sans-serif', 'monospace', 'cursive', 'fantasy', 'system-ui',
@@ -123,19 +124,36 @@ export async function collectCssSources($, pageUrl, opts) {
   let inlineStyleEls = 0;
   let cssText = '';
 
+  // Print (and other non-screen) stylesheets never paint a screen render, so
+  // neither branding nor the hidden-content pass may read them. `media:
+  // 'unconditional-screen'` also drops sheets gated on a media query.
+  const unconditional = opts.media === 'unconditional-screen';
+  const applies = (el) => mediaAppliesToScreen($(el).attr('media'), { unconditional });
+
   $('style').each((_, el) => {
+    if (!applies(el)) return;
     const t = $(el).html();
     if (t) { cssText += '\n' + t; styleBlocks++; }
   });
 
-  $('[style]').each((_, el) => {
-    const t = $(el).attr('style');
-    if (t) { cssText += '\n*{' + t + '}'; inlineStyleEls++; }
-  });
+  // Inline style="" attributes are folded in as universal rules so branding
+  // can mine their colours and fonts. They are NOT visibility rules: one
+  // element carrying style="display:none" became `*{display:none}`, and the
+  // hidden-content pass then deleted every element small enough to pass its
+  // bulk guard — irs.gov's tax-bracket page came back as its header (R21,
+  // 2026-09-09). That pass reads inline styles per element itself, so it
+  // asks for `inlineStyles: false`.
+  if (opts.inlineStyles !== false) {
+    $('[style]').each((_, el) => {
+      const t = $(el).attr('style');
+      if (t) { cssText += '\n*{' + t + '}'; inlineStyleEls++; }
+    });
+  }
 
   if (opts.fetchLinkedCss) {
     const hrefs = [];
     $('link[rel~="stylesheet"][href]').each((_, el) => {
+      if (!applies(el)) return;
       const href = $(el).attr('href');
       if (href) hrefs.push(resolveUrl(href, pageUrl));
     });
@@ -404,7 +422,7 @@ function extractTokens(cssText, cssVariables) {
  * Extract the full branding object from a loaded cheerio $.
  * @param {import('cheerio').CheerioAPI} $
  * @param {string} pageUrl
- * @param {{ fetchLinkedCss?: boolean, maxStylesheets?: number, perFileTimeoutMs?: number, timeoutMs?: number, overallTimeoutMs?: number, stylesheetConcurrency?: number }} [opts]
+ * @param {{ fetchLinkedCss?: boolean, maxStylesheets?: number, perFileTimeoutMs?: number, timeoutMs?: number, overallTimeoutMs?: number, stylesheetConcurrency?: number, media?: 'screen'|'unconditional-screen', inlineStyles?: boolean }} [opts]
  * @returns {Promise<object>}
  */
 export async function extractBranding($, pageUrl, opts = {}) {

package/src/tools/scrape/unifiedScrape.js

@@ -542,7 +542,14 @@ export class UnifiedScrapeTool {
             // Themes split visibility rules across many component sheets — the
             // rule hiding Shopify's sold-out badge sits at index 12 of 38 on a
             // stock Dawn storefront, so a cap of 10 silently misses it.
-            maxStylesheets: 20
+            maxStylesheets: 20,
+            // Only sheets a screen render applies unconditionally: a
+            // media="print" sheet hid every screen element of irs.gov's
+            // tax-bracket page and left the print logo (R21, 2026-09-09).
+            media: 'unconditional-screen',
+            // stripHiddenFromDom reads style="" per element; folded into
+            // universal rules they hid the whole page (R21).
+            inlineStyles: false
           });
           css = collected.cssText || '';
         }

package/src/tools/search/redditSearch.js

@@ -15,11 +15,14 @@
  *   cross-subreddit full-text search, but has known post-2023 archive gaps
  *   and recurring outages.
  *
- * Routing: scoped searches go to Arctic Shift. An unscoped keyword search
- * finds posts through a site-restricted web search and then reads them — or
- * searches their comments — in the archive by id. PullPush stopped serving
- * automated clients in August 2026 and is used only on explicit request.
- * Both archives are free and need no credentials.
+ * Routing: Arctic Shift first, PullPush second. A scoped search queries
+ * Arctic Shift and, if that fails, PullPush. An unscoped keyword search finds
+ * posts through a site-restricted web search and reads them — or searches
+ * their comments — in Arctic Shift by id, then falls back to PullPush's own
+ * full-text search. Thread mode is Arctic Shift only. PullPush has refused
+ * automated clients since August 2026, so the fallback usually reports that
+ * refusal; it stays second for whenever it answers again. Both archives are
+ * free and need no credentials.
  *
  * Optional official-API path: if the user sets REDDIT_CLIENT_ID and
  * REDDIT_CLIENT_SECRET (their own Reddit app), posts/thread requests can read
@@ -101,9 +104,8 @@ export class RedditSearchTool {
     // Lazily constructed on first official-path use; overridable for tests.
     this._officialAdapter = options.officialAdapter || null;
 
-    // Web discovery serves the one shape no archive can: a keyword search
-    // across all of Reddit. Arctic Shift requires a subreddit or author scope,
-    // and PullPush stopped serving automated clients in August 2026.
+    // Web discovery serves the one shape Arctic Shift cannot: a keyword search
+    // across all of Reddit, which it requires a subreddit or author scope for.
     this.searchAdapter = options.searchAdapter || null;
     this.searchApiKey = options.searchApiKey || null;
     this.searchApiBaseUrl = options.searchApiBaseUrl || null;
@@ -179,13 +181,16 @@ export class RedditSearchTool {
       order = ['pullpush'];
     } else {
       // auto: prefer the user's own official API (live, authoritative) when it
-      // can serve this request, then fall back to the community archives.
-      // PullPush is no longer tried automatically: every request now returns
-      // 429 "This website does not provide free scraping resources for agents",
-      // or a Cloudflare 403 challenge. It stays available on explicit request.
+      // can serve this request, then the community archives — Arctic Shift
+      // first, PullPush second (owner's call, 2026-09-10). PullPush has refused
+      // automated clients since August 2026 — 429 "This website does not
+      // provide free scraping resources for agents", or a Cloudflare 403
+      // challenge — so the fallback is one un-retried request that usually
+      // reports that refusal after the real Arctic Shift error, and answers
+      // when PullPush is serving again. Thread mode has no second source.
       const archives = v.mode === 'thread' ? ['arctic_shift']
-        : arcticPossible ? ['arctic_shift']
-        : discoveryPossible ? ['web_discovery']
+        : arcticPossible ? ['arctic_shift', 'pullpush']
+        : discoveryPossible ? ['web_discovery', 'pullpush']
         : [];
       order = officialPossible ? ['reddit_api', ...archives] : archives;
     }
@@ -252,7 +257,7 @@ export class RedditSearchTool {
         ? 'Reddit-wide comment search: posts were found with a site-restricted web search, then each post\'s comments were searched for the keywords in the Arctic Shift archive, in post relevance order.'
         : 'Reddit-wide keyword search: posts were found with a site-restricted web search, then read from the Arctic Shift archive by ID.',
       'Results are ordered by web-search relevance, not by score or date.',
-      'Arctic Shift cannot keyword-search across all of Reddit, and PullPush no longer serves automated clients — scope the search to a subreddit or author to query the archive directly.',
+      'Arctic Shift cannot keyword-search across all of Reddit — scope the search to a subreddit or author to query the archive directly.',
     ];
     if (v.after || v.before) {
       // Silently dropping a date filter would return results the caller
@@ -489,12 +494,14 @@ export class RedditSearchTool {
     if (response.status === 429) {
       const reset = response.headers?.get?.('x-ratelimit-reset');
       // PullPush's 429 body states its actual policy ("does not provide free
-      // scraping resources for agents...") — pass that through verbatim.
+      // scraping resources for agents...") — pass that through verbatim. That
+      // policy refusal is not a transient throttle: retrying it only spends the
+      // delay, which matters now that PullPush is the automatic second source.
       let detail = '';
       try { detail = (await response.json())?.error ?? ''; } catch { /* no body */ }
       throw Object.assign(
         new Error(`rate limited (429)${reset ? `, retry in ${reset}s` : ''}${detail ? ` — ${detail}` : ''}`),
-        { retryable: true },
+        { retryable: !/does not provide free scraping/i.test(detail) },
       );
     }
     if (!response.ok) {

package/src/utils/hiddenContent.js

@@ -83,6 +83,38 @@ function stripConditionalBlocks(css) {
   return out;
 }
 
+/**
+ * Split a selector list on its top-level commas only. A naive split cut
+ * irs.gov's `.callout:has(> .a, .b, p, ul, h2, h3, table)` into bare `p`,
+ * `ul`, `h2`, `h3` and `table` rules carrying its `display:none !important`,
+ * and the stripper then deleted the article body (R21, 2026-09-09).
+ * @param {string} list
+ * @returns {string[]}
+ */
+export function splitSelectorList(list) {
+  const out = [];
+  let depth = 0;
+  let quote = null;
+  let start = 0;
+  for (let i = 0; i < list.length; i++) {
+    const ch = list[i];
+    if (quote) {
+      if (ch === '\\') i++;
+      else if (ch === quote) quote = null;
+      continue;
+    }
+    if (ch === '"' || ch === "'") quote = ch;
+    else if (ch === '(' || ch === '[') depth++;
+    else if (ch === ')' || ch === ']') depth = Math.max(0, depth - 1);
+    else if (ch === ',' && depth === 0) {
+      out.push(list.slice(start, i));
+      start = i + 1;
+    }
+  }
+  out.push(list.slice(start));
+  return out;
+}
+
 /** True when cheerio cannot meaningfully evaluate the selector. */
 function isUnsupportedSelector(selector) {
   return (
@@ -91,7 +123,7 @@ function isUnsupportedSelector(selector) {
     // case and these rules would hide content every real visitor sees.
     /(^|[\s.#\[])no-js(\b|[.\[])/.test(selector) ||
     selector.includes('::') ||
-    /:(hover|focus|focus-within|focus-visible|active|target|checked|disabled|placeholder|before|after|root|host|where|is|not\()/i.test(selector) ||
+    /:(hover|focus|focus-within|focus-visible|active|target|checked|disabled|placeholder|before|after|root|host|where|is|not\(|has\()/i.test(selector) ||
     selector.includes('@') ||
     selector.length === 0
   );
@@ -199,7 +231,7 @@ export function collectVisibilitySelectors(css) {
 
     const important = /!\s*important/i.test(body);
 
-    for (const raw of selectorList.split(',')) {
+    for (const raw of splitSelectorList(selectorList)) {
       const selector = raw.trim();
       if (isUnsupportedSelector(selector) || NEVER_REMOVE.has(selector)) continue;
       const entry = { selector, spec: specificity(selector), order, important };
@@ -249,12 +281,45 @@ function renderedSize($, el) {
 export function inlineStyleText($) {
   const parts = [];
   $('style').each((_, el) => {
+    // A <style media="print"> block is only consulted when printing.
+    if (!mediaAppliesToScreen($(el).attr('media'), { unconditional: true })) return;
     const text = $(el).html();
     if (text) parts.push(text);
   });
   return parts.join('\n');
 }
 
+/**
+ * Whether a stylesheet's `media` attribute applies to a screen render.
+ *
+ * `<link rel="stylesheet" media="print">` is consulted only when printing.
+ * irs.gov's tax-bracket page ships one that hides every screen element and
+ * shows a print-only logo; read as screen CSS it emptied the page to that logo
+ * (R21, 2026-09-09). Absent or empty, `all`, `screen` and `not print` apply.
+ * `print`, `speech` and the other non-screen types, and `not screen`/`not all`,
+ * do not. With `unconditional: true` a media QUERY (`screen and (max-width:
+ * 600px)`, a bare `(prefers-color-scheme: dark)`) does not apply either — the
+ * hidden-content pass ignores @media blocks for the same reason.
+ *
+ * @param {string|undefined} media - the attribute value
+ * @param {{ unconditional?: boolean }} [options]
+ * @returns {boolean}
+ */
+export function mediaAppliesToScreen(media, { unconditional = false } = {}) {
+  const value = (media || '').trim().toLowerCase();
+  if (!value) return true;
+  return value.split(',').some((part) => {
+    const query = part.trim();
+    if (!query) return false;
+    if (unconditional && query.includes('(')) return false;
+    if (query.startsWith('(')) return true; // a bare feature query: type "all"
+    const m = query.match(/^(?:(not|only)\s+)?([a-z-]+)/);
+    if (!m) return true;
+    const screenType = m[2] === 'all' || m[2] === 'screen';
+    return m[1] === 'not' ? !screenType : screenType;
+  });
+}
+
 /**
  * Remove browser-invisible content from a cheerio document, in place.
  *

package/src/utils/redditHosts.js

@@ -0,0 +1,123 @@
+/**
+ * redditHosts — reddit.com is never fetched.
+ *
+ * reddit.com refuses every non-browser client (403, and the stealth browsers
+ * too — the block is IP/TLS-reputation based), so a scrape or fetch_url
+ * against it always fails and leaves the caller guessing. Reddit is served by
+ * reddit_search, which reads the same posts and comments from the Arctic
+ * Shift community archive (PullPush second). A reddit.com target is therefore
+ * refused before any network work, with the reddit_search call that gets the
+ * same data spelled out — derived from the URL where the URL says enough.
+ *
+ * Runs inside the pre-fetch gate (robotsGate.js), so every fetching tool and
+ * both browser paths get it without knowing. Not overridable: respect_robots
+ * is about robots.txt, and fetching reddit.com fails whatever the caller sends.
+ *
+ * Mirrors the website's `src/lib/tools/reddit-hosts.ts` — same rule, same
+ * message — so a reddit.com URL is answered identically on both surfaces.
+ */
+
+export class UseRedditSearchError extends Error {
+  constructor(url) {
+    const call = redditSearchCallFor(url);
+    const nextStep = Object.keys(call).length
+      ? `reddit_search(${JSON.stringify(call)})`
+      : 'reddit_search with a query, subreddit or author — or mode "thread" with a post\'s link_id';
+    super(
+      `${hostOf(url)} is not fetched: reddit.com refuses every non-browser client, stealth browsers included, ` +
+      `so this call would fail. Reddit is served by reddit_search (5 credits), which reads the same posts ` +
+      `and comments from the Arctic Shift community archive. Next step: ${nextStep}`
+    );
+    this.name = 'UseRedditSearchError';
+    this.code = 'USE_REDDIT_SEARCH';
+    this.url = url;
+    this.redditSearchCall = call;
+  }
+}
+
+/** The hostname of a URL, lowercased, or null if it will not parse. */
+function hostOf(url) {
+  try {
+    return new URL(url).hostname.toLowerCase().replace(/\.$/, '');
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * True for reddit.com and its subdomains (www, old, new, np, sh …) and for the
+ * bare redd.it short-link host. The media hosts (i.redd.it, v.redd.it,
+ * preview.redd.it) serve files to any client and are left alone.
+ * @param {string} url
+ */
+export function isRedditUrl(url) {
+  const host = hostOf(url);
+  if (!host) return false;
+  return host === 'reddit.com' || host.endsWith('.reddit.com') || host === 'redd.it';
+}
+
+/**
+ * The reddit_search call that answers a reddit.com URL, or {} when the URL
+ * names nothing reddit_search can be pointed at (the front page, a wiki, a
+ * settings page).
+ *
+ *   /r/{sub}/comments/{id}/…, /comments/{id}, /gallery/{id}, redd.it/{id}
+ *                                        → mode "thread", link_id
+ *   /r/{sub}/search?q=…                  → query scoped to the subreddit
+ *   /search?q=…                          → an unscoped query
+ *   /r/{sub}[/new|/top|…]                → the subreddit's posts
+ *   /user/{name} or /u/{name}[/comments] → the author's posts (or comments)
+ *   ?type=comment                        → mode "comments"
+ * @param {string} url
+ * @returns {{ mode?: string, query?: string, subreddit?: string, author?: string, link_id?: string }}
+ */
+export function redditSearchCallFor(url) {
+  let parsed;
+  try {
+    parsed = new URL(url);
+  } catch {
+    return {};
+  }
+  const segments = parsed.pathname
+    .split('/')
+    .filter(Boolean)
+    .map((segment) => {
+      try {
+        return decodeURIComponent(segment);
+      } catch {
+        return segment;
+      }
+    });
+
+  if (parsed.hostname.toLowerCase() === 'redd.it') {
+    return segments[0] ? { mode: 'thread', link_id: segments[0] } : {};
+  }
+
+  // A post's id follows "comments" or "gallery" wherever it sits in the path;
+  // a permalink to one comment still reads as its thread.
+  const idAt = segments.findIndex((segment) => segment === 'comments' || segment === 'gallery');
+  if (idAt !== -1 && segments[idAt + 1]) {
+    return { mode: 'thread', link_id: segments[idAt + 1] };
+  }
+
+  const call = {};
+  if (segments[0] === 'r' && segments[1]) call.subreddit = segments[1];
+  if ((segments[0] === 'user' || segments[0] === 'u') && segments[1]) {
+    call.author = segments[1];
+    if (segments[2] === 'comments') call.mode = 'comments';
+  }
+  const query = parsed.searchParams.get('q')?.trim();
+  if (query) call.query = query;
+  if (parsed.searchParams.get('type') === 'comment') call.mode = 'comments';
+  return call;
+}
+
+/**
+ * Throw UseRedditSearchError for a reddit.com target. Call before any network
+ * work — the point is that reddit.com never gets a request, not even for its
+ * robots.txt.
+ * @param {string} url
+ */
+export function assertNotRedditUrl(url) {
+  if (isRedditUrl(url)) throw new UseRedditSearchError(url);
+}

package/src/utils/robotsGate.js

@@ -19,6 +19,7 @@
 
 import { RobotsChecker } from './robotsChecker.js';
 import { assertHostAllowed } from './hostBlocklist.js';
+import { assertNotRedditUrl } from './redditHosts.js';
 import { identityHeaders, resolveUserAgent } from './fetchIdentity.js';
 import { throttleHost } from './hostRateLimiter.js';
 import { recordComplianceEvent, apiKeyId } from './complianceAudit.js';
@@ -75,8 +76,13 @@ export async function robotsPreflight(url, options = {}) {
   // costs the caller nothing: we refused, we fetched nothing.
   try {
     assertHostAllowed(url);
+    // reddit.com refuses every non-browser client, so it is never fetched and
+    // the caller is pointed at reddit_search instead. Also not overridable.
+    assertNotRedditUrl(url);
   } catch (error) {
-    if (error?.code === 'HOST_BLOCKED') markPreflightRefusal('HOST_BLOCKED');
+    if (error?.code === 'HOST_BLOCKED' || error?.code === 'USE_REDDIT_SEARCH') {
+      markPreflightRefusal(error.code);
+    }
     throw error;
   }
 
@@ -178,11 +184,29 @@ export async function preflightFetch(url, options = {}) {
   return {
     headers: outboundHeaders(decision.userAgent, signature),
     userAgent: decision.userAgent,
-    warnings: decision.warnings,
+    warnings: [...decision.warnings, ...crawlDelayWarning(url, decision.crawlDelayMs)],
     overridden: decision.overridden
   };
 }
 
+/**
+ * Name a long Crawl-delay, so a slow multi-page call reads as compliance
+ * rather than a stall: eff.org asks every agent for 30 s, and a 10-page
+ * llms.txt run took 13 minutes with nothing in the response saying why (R21,
+ * 2026-09-09).
+ * @param {string} url
+ * @param {number} crawlDelayMs
+ * @returns {string[]}
+ */
+function crawlDelayWarning(url, crawlDelayMs) {
+  if (!(crawlDelayMs >= 5000)) return [];
+  let host = url;
+  try { host = new URL(url).host; } catch { /* keep the raw url */ }
+  return [
+    `robots.txt on ${host} asks for a ${Math.round(crawlDelayMs / 1000)} s crawl delay; requests to it are spaced by that much, so a multi-page call takes about that long per page.`
+  ];
+}
+
 /**
  * The gate for browser paths. Same decision as {@link preflightFetch}, minus
  * the identity and signature headers — those belong on an HTTP fetch, not on a
@@ -213,7 +237,7 @@ export async function browserPreflight(url, options = {}) {
   }
 
   await throttleHost(url, { crawlDelayMs: decision.crawlDelayMs });
-  return decision.warnings;
+  return [...decision.warnings, ...crawlDelayWarning(url, decision.crawlDelayMs)];
 }
 
 /** Test/diagnostic hook: drop every cached robots.txt. */