crawlforge-mcp-server 6.17.1 → 6.18.1

package/src/tools/advanced/BrowserSessionTool.js

@@ -184,6 +184,18 @@ function withJsResult(result) {
   return { ...result, jsResult: result.result.result };
 }
 
+/**
+ * A screenshot action's image travels once, in `screenshots[]`, which the
+ * server publishes as crawlforge://screenshot/{actionId} — the action's id.
+ * Its copy in the action result is replaced by that URI, as scrape_with_actions
+ * does: left in, the same base64 came back inline beside the resource (R24).
+ */
+function withoutScreenshotData(result) {
+  if (result?.type !== 'screenshot' || typeof result.result?.data !== 'string') return result;
+  const { data, ...rest } = result.result;
+  return { ...result, result: { ...rest, resourceUri: `crawlforge://screenshot/${result.id}` } };
+}
+
 /**
  * The gate's warnings — a respect_robots override, a crawl-delay note — are
  * what the shared parameter description promises the caller gets back
@@ -517,7 +529,7 @@ export class BrowserSessionTool {
         : {}),
       ...gateWarningFields(session.page),
       error: result.error,
-      actionResults: result.results.map(withJsResult),
+      actionResults: result.results.map(withJsResult).map(withoutScreenshotData),
       screenshots: result.screenshots,
       ...(result.capturedStates.length > 0 ? { capturedStates: result.capturedStates } : {}),
       stats: result.stats

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -593,7 +593,10 @@ export class ScrapeWithActionsTool extends EventEmitter {
       actionResults,
       attempt: chainResult.attempt ?? attempts.length,
       attempts,
-      totalActions: params.actions.length,
+      // The chain that was run — the caller's actions plus the fills, waits
+      // and submit formAutoFill adds — so it is the denominator of the three
+      // counts below; counting only the caller's own read "1 of 11".
+      totalActions: actionChain.length,
       successfulActions: actionResults.filter(r => r.success).length,
       failedActions: actionResults.filter(r => !r.success).length,
       actionsExecuted: actionResults.length, // Total executed (for validation)
@@ -635,10 +638,18 @@ export class ScrapeWithActionsTool extends EventEmitter {
     if (Array.isArray(formAutoFill.fields)) {
       // Structured shape: { fields: [{selector, value, type, waitAfter}], submitSelector, waitAfterSubmit }
       for (const field of formAutoFill.fields) {
+        // Each field type is filled the way that control is: typing into a
+        // radio, a checkbox or a <select> "succeeds" and changes nothing, so
+        // the form was posted without them and nothing failed (R24). `check`
+        // also carries the types nothing can fill ("file"), which it refuses.
+        const fill = field.type === 'text'
+          ? { type: 'type', text: field.value }
+          : field.type === 'select'
+            ? { type: 'select', value: field.value }
+            : { type: 'check', value: field.value, fieldType: field.type };
         fillActions.push({
-          type: 'type',
+          ...fill,
           selector: field.selector,
-          text: field.value,
           description: `Auto-fill field: ${field.selector}`,
           continueOnError: true,
           retries: 1

package/src/tools/advanced/batchScrape/worker.js

@@ -14,6 +14,7 @@ import { fetchResigned } from '../../../utils/resignedFetch.js';
 import { htmlToMarkdown } from '../../../utils/htmlToMarkdown.js';
 import { elementText } from '../../../utils/elementText.js';
 import { pageTitle } from '../../../utils/pageTitle.js';
+import { classifyContentType, unsupportedContentTypeError } from '../../extract/_fetchAndParse.js';
 
 /**
  * Fetch a URL with AbortController timeout (SSRF-guarded + per-host throttled).
@@ -41,6 +42,13 @@ export async function fetchUrl(url, options = {}) {
     if (response.status === 429 || response.status === 503) {
       noteRetryAfter(url, response.headers?.get?.('retry-after'));
     }
+    // A PDF decoded as text came back success:true with "%PDF-1.4 …" as the
+    // page (R24 1.5). Refused before the body is read, with the wording
+    // `scrape` gives the same URL; an error status keeps its own message.
+    const contentType = response.headers?.get?.('content-type') || null;
+    if (response.ok && classifyContentType(contentType) === 'binary') {
+      throw unsupportedContentTypeError(contentType);
+    }
     // crawlforge-extractors' readBody: same size cap, but decoded with the
     // body's real charset — the local reader this replaces decoded everything
     // as UTF-8, which turned lua.org's ISO-8859-1 "português" into "portugu�s".

package/src/tools/advanced/scrapeWithActions/recorder.js

@@ -168,6 +168,8 @@ export function buildRecordedEntry(action, timestampMsSinceStart) {
   if (action.duration !== undefined) entry.duration = action.duration;
   if (action.url !== undefined) entry.url = action.url;
   if (action.value !== undefined) entry.value = action.value;
+  // check (a formAutoFill checkbox/radio field) does not replay without it
+  if (action.fieldType !== undefined) entry.fieldType = action.fieldType;
   if (action.direction !== undefined) entry.direction = action.direction;
   if (action.distance !== undefined) entry.distance = action.distance;
   // scroll: absolute scroll-to coordinates

package/src/tools/basic/_targetFailure.js

@@ -0,0 +1,54 @@
+/**
+ * Why a fetched document fails extract_links / extract_text, with the same
+ * verdict and wording as the REST routes (fix plan Phase E1).
+ *
+ * A named vendor's wall fails on any status: a challenge arrives as a 403 or
+ * as a 200 with the wall in the body. Any other non-2xx fails with its
+ * status. An empty shell or an error placeholder on a 2xx is not raised here;
+ * extraction proceeds as before.
+ */
+
+import { load } from 'cheerio';
+import { stealthDocumentVerdict } from '../../utils/stealthVerdict.js';
+import { pageTitle } from '../../utils/pageTitle.js';
+
+// Response bodies a verdict can be read from; anything else is a binary. A
+// missing Content-Type is read, as _fetchAndParse.js treats it as HTML.
+const TEXTUAL_BODY = /^(?:text\/(?:html|plain)|application\/xhtml\+xml)\b/i;
+
+/**
+ * The plain fetch's verdict on a document, or null for a body a verdict
+ * cannot be read from (a binary, JSON). The fetch ladder reads it to decide
+ * whether escalation could help (src/utils/fetchLadder.js).
+ * @param {Response & { _body: string }} response - from fetchWithTimeout
+ * @param {string} url - the requested URL
+ */
+export function plainVerdict(response, url) {
+  const contentType = response.headers?.get?.('content-type');
+  if (contentType && !TEXTUAL_BODY.test(contentType)) return null;
+  // A parse of its own, scripts and styles removed, so the handler's
+  // document stays intact.
+  const $ = load(response._body ?? '');
+  $('script, style, noscript').remove();
+  return stealthDocumentVerdict(
+    {
+      url: response.url || url,
+      status: response.status,
+      title: pageTitle($),
+      text: $('body').text().replace(/\s+/g, ' ').trim(),
+      html: response._body
+    },
+    { fetcher: 'a plain fetch', rendered: false, contentReturned: false }
+  );
+}
+
+/**
+ * @param {Response & { _body: string }} response - from fetchWithTimeout
+ * @param {string} url - the requested URL
+ * @param {object|null} [verdict] - plainVerdict's, when the caller already has it
+ * @returns {string|null} the error message, or null when the document is usable
+ */
+export function targetFailure(response, url, verdict = plainVerdict(response, url)) {
+  if (verdict?.blocked) return `Target answered HTTP ${response.status}: ${verdict.error}`;
+  return response.ok ? null : `Target answered HTTP ${response.status}`;
+}

package/src/tools/basic/extractLinks.js

@@ -1,76 +1,84 @@
 /**
  * extract_links — Extract all links from a webpage with optional filtering.
  * Extracted from server.js inline handler.
+ * E3: Link records come from crawlforge-extractors' extractLinkRecords, the
+ * reader the REST route uses, so both surfaces return the same links.
  */
 
 import { load } from 'cheerio';
-import { fetchWithTimeout } from './_fetch.js';
+import { extractLinkRecords } from 'crawlforge-extractors';
+import { fetchLadder, ladderErrorResult, isJsonType } from '../../utils/fetchLadder.js';
 
 /**
  * @param {{ url: string, filter_external?: boolean, base_url?: string,
- *   user_agent?: string, respect_robots?: boolean }} params
+ *   user_agent?: string, respect_robots?: boolean,
+ *   escalate?: boolean, escalate_engine?: string }} params
+ * @param {Function} [escalateFetch] the stealth escalation stage
  */
-export async function extractLinksHandler({ url, filter_external, base_url, user_agent, respect_robots }) {
+async function extractLinks({ url, filter_external, base_url, user_agent, respect_robots, escalate, escalate_engine }, escalateFetch) {
   try {
-    const response = await fetchWithTimeout(url, {
+    const ladder = await fetchLadder(url, {
+      tool: 'extract_links',
       userAgent: user_agent,
       respectRobots: respect_robots,
-      tool: 'extract_links'
+      escalate: escalate === true,
+      escalateEngine: escalate_engine,
+      escalateFetch
     });
-    if (!response.ok) {
-      throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+    if (ladder.html === undefined) return ladderErrorResult('extract_links', 'Failed to extract links: ', ladder);
+
+    // A JSON body carries no <a href>.
+    if (isJsonType(ladder.type)) {
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            links: [],
+            total_count: 0,
+            internal_count: 0,
+            external_count: 0,
+            other_count: 0,
+            base_url: base_url || ladder.url,
+            ...ladder.fields,
+            warnings: [...ladder.warnings, 'the target returned application/json; it has no HTML links']
+          }, null, 2)
+        }]
+      };
     }
 
-    const html = await response.text();
+    const html = ladder.html;
     const $ = load(html);
 
-    const finalUrl = response.url || url;
-    const pageUrl = new URL(finalUrl);
+    const finalUrl = ladder.url;
 
-    // <base href>, if present, overrides the page URL as the resolution base
-    // for relative links (but an explicit base_url override wins over both).
+    // Reported base: an explicit base_url, else <base href> resolved against
+    // the page, else the page URL (extractLinkRecords resolves the same way).
     let docBase = finalUrl;
     const baseHref = $('base[href]').first().attr('href');
     if (baseHref) {
       try { docBase = new URL(baseHref, finalUrl).toString(); } catch { /* ignore invalid <base href> */ }
     }
-
     const baseUrl = base_url || docBase;
-    const links = [];
-
-    $('a[href]').each((_, element) => {
-      const href = $(element).attr('href');
-      const text = $(element).text().trim();
-
-      // A javascript: pseudo-link ("Cookie Settings") is a button, not a
-      // link; it was counted as an external link on boeing.com (R20).
-      if (!href || /^\s*javascript:/i.test(href)) return;
 
-      try {
-        const absoluteUrl = new URL(href, baseUrl).toString();
-        const isExternal = new URL(absoluteUrl).origin !== pageUrl.origin;
-
-        if (filter_external && !isExternal) return;
-
-        links.push({ href: absoluteUrl, text, is_external: isExternal, original_href: href });
-      } catch {
-        // skip invalid URLs
-      }
-    });
-
-    const uniqueLinks = links.filter((link, index, arr) =>
-      arr.findIndex(l => l.href === link.href) === index
-    );
+    // Records are deduplicated on the URL without fragment or trailing slash;
+    // mailto:/tel:/javascript: links are type "other". filter_external drops
+    // only the internal ones.
+    const records = extractLinkRecords($, { pageUrl: finalUrl, baseUrl: base_url });
+    const links = filter_external ? records.filter(l => l.type !== 'internal') : records;
+    const count = (type) => links.filter(l => l.type === type).length;
 
     return {
       content: [{
         type: 'text',
         text: JSON.stringify({
-          links: uniqueLinks,
-          total_count: uniqueLinks.length,
-          internal_count: uniqueLinks.filter(l => !l.is_external).length,
-          external_count: uniqueLinks.filter(l => l.is_external).length,
-          base_url: baseUrl
+          links,
+          total_count: links.length,
+          internal_count: count('internal'),
+          external_count: count('external'),
+          other_count: count('other'),
+          base_url: baseUrl,
+          ...ladder.fields,
+          ...(ladder.warnings.length > 0 ? { warnings: ladder.warnings } : {})
         }, null, 2)
       }]
     };
@@ -81,3 +89,15 @@ export async function extractLinksHandler({ url, filter_external, base_url, user
     };
   }
 }
+
+/**
+ * @param {{ escalateFetch?: (args: { url: string, engine: string, respectRobots?: boolean }) => Promise<object> }} [deps]
+ *   the stealth escalation stage (server.js `stealthEscalation`); without it
+ *   an escalation that would run is reported as unavailable
+ */
+export function createExtractLinksHandler({ escalateFetch } = {}) {
+  return (params) => extractLinks(params, escalateFetch);
+}
+
+/** The handler with no escalation stage wired (tests; server.js wires its own). */
+export const extractLinksHandler = createExtractLinksHandler();

package/src/tools/basic/extractText.js

@@ -3,12 +3,15 @@
  * Extracted from server.js inline handler.
  * D3.1: Added output_format:"markdown" option backed by Turndown.
  * B1: Preserve block structure for text mode; use Readability + GFM for markdown mode.
+ * E3: Text mode reads through crawlforge-extractors' flattenText, as the REST
+ * route does; `selector` and `max_length` take the REST route's meaning.
  */
 
 import { load } from 'cheerio';
+import { flattenText } from 'crawlforge-extractors';
 import { JSDOM } from 'jsdom';
 import { Readability } from '@mozilla/readability';
-import { fetchWithTimeout } from './_fetch.js';
+import { fetchLadder, ladderErrorResult, isJsonType } from '../../utils/fetchLadder.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
 
 // Block-level elements whose boundaries should become paragraph breaks
@@ -73,22 +76,61 @@ export function readabilityToMarkdown(html, pageUrl) {
   return htmlToMarkdown(articleHtml);
 }
 
+/**
+ * Cut text to max_length characters and mark the cut with '...' (the REST rule).
+ * @param {string} text
+ * @param {number} [maxLength]
+ * @returns {string}
+ */
+function truncate(text, maxLength) {
+  return maxLength && text.length > maxLength ? text.substring(0, maxLength) + '...' : text;
+}
+
 /**
  * @param {{ url: string, remove_scripts?: boolean, remove_styles?: boolean,
- *   output_format?: "text"|"markdown", user_agent?: string, respect_robots?: boolean }} params
+ *   output_format?: "text"|"markdown", selector?: string, max_length?: number,
+ *   user_agent?: string, respect_robots?: boolean,
+ *   escalate?: boolean, escalate_engine?: string }} params
+ * @param {Function} [escalateFetch] the stealth escalation stage
  */
-export async function extractTextHandler({ url, remove_scripts, remove_styles, output_format, user_agent, respect_robots }) {
+async function extractText({ url, remove_scripts, remove_styles, output_format, selector, max_length, user_agent, respect_robots, escalate, escalate_engine }, escalateFetch) {
   try {
-    const response = await fetchWithTimeout(url, {
+    const ladder = await fetchLadder(url, {
+      tool: 'extract_text',
       userAgent: user_agent,
       respectRobots: respect_robots,
-      tool: 'extract_text'
+      escalate: escalate === true,
+      escalateEngine: escalate_engine,
+      escalateFetch
     });
-    if (!response.ok) {
-      throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+    if (ladder.html === undefined) return ladderErrorResult('extract_text', 'Failed to extract text: ', ladder);
+
+    const extras = {
+      ...ladder.fields,
+      ...(ladder.warnings.length > 0 ? { warnings: ladder.warnings } : {})
+    };
+
+    // A JSON body has no markup to strip: it is returned as it came.
+    if (isJsonType(ladder.type)) {
+      const body = truncate(ladder.html, max_length);
+      const format = output_format === 'markdown' ? 'markdown' : 'text';
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            url: ladder.url,
+            [format]: body,
+            output_format: format,
+            word_count: body.split(/\s+/).filter(w => w.length > 0).length,
+            char_count: body.length,
+            ...ladder.fields,
+            warnings: [...ladder.warnings, 'the target returned application/json; its body is returned as text']
+          }, null, 2)
+        }]
+      };
     }
 
-    const html = await response.text();
+    const html = ladder.html;
     const $ = load(html);
 
     if (remove_scripts !== false) $('script').remove();
@@ -101,21 +143,32 @@ export async function extractTextHandler({ url, remove_scripts, remove_styles, o
     // enabled never render noscript content, so always strip it.
     $('noscript').remove();
 
-    $('nav, header, footer, aside, .advertisement, .ad, .sidebar').remove();
+    // A selector names the content, so the boilerplate strip is skipped.
+    let $target;
+    if (selector) {
+      $target = $(selector);
+      if ($target.length === 0) throw new Error(`No elements found for selector: ${selector}`);
+    } else {
+      $('nav, header, footer, aside, .advertisement, .ad, .sidebar').remove();
+    }
 
     const result = {
-      url: response.url
+      url: ladder.url
     };
 
     if (output_format === 'markdown') {
-      // Run Readability first to get main content, then convert to GFM markdown
-      result.markdown = readabilityToMarkdown(html, response.url);
+      // Run Readability first to get main content, then convert to GFM markdown;
+      // a selector's matches are converted as they are.
+      const markdown = $target
+        ? htmlToMarkdown($target.toArray().map(el => $.html(el)).join('\n'))
+        : readabilityToMarkdown(html, ladder.url);
+      result.markdown = truncate(markdown, max_length);
       result.output_format = 'markdown';
       const plainText = result.markdown.replace(/[#*`_\[\]]/g, '').replace(/\s+/g, ' ').trim();
       result.word_count = plainText.split(/\s+/).filter(w => w.length > 0).length;
       result.char_count = plainText.length;
     } else {
-      const text = extractBlockText($);
+      const text = truncate(flattenText($, $target), max_length);
       result.text = text;
       result.output_format = 'text';
       result.word_count = text.split(/\s+/).filter(w => w.length > 0).length;
@@ -125,7 +178,7 @@ export async function extractTextHandler({ url, remove_scripts, remove_styles, o
     return {
       content: [{
         type: 'text',
-        text: JSON.stringify(result, null, 2)
+        text: JSON.stringify({ ...result, ...extras }, null, 2)
       }]
     };
   } catch (error) {
@@ -135,3 +188,15 @@ export async function extractTextHandler({ url, remove_scripts, remove_styles, o
     };
   }
 }
+
+/**
+ * @param {{ escalateFetch?: (args: { url: string, engine: string, respectRobots?: boolean }) => Promise<object> }} [deps]
+ *   the stealth escalation stage (server.js `stealthEscalation`); without it
+ *   an escalation that would run is reported as unavailable
+ */
+export function createExtractTextHandler({ escalateFetch } = {}) {
+  return (params) => extractText(params, escalateFetch);
+}
+
+/** The handler with no escalation stage wired (tests; server.js wires its own). */
+export const extractTextHandler = createExtractTextHandler();

package/src/tools/crawl/crawlDeep.js

@@ -261,19 +261,30 @@ export class CrawlDeepTool {
         });
         const duration = Date.now() - startTime;
 
+        // The counters, each defined once: a URL the crawl tried either
+        // produced a page (results) or an error (errors), so
+        // pages_attempted = pages_crawled + error_count. pages_crawled used to
+        // count attempts, which read "3 crawled, 1 found, 2 errors" (R24).
+        const succeeded = results.results.length;
+        const failed = results.errors.length;
+
         // Process and format results
         const response = {
           url: validated.url,
           crawl_depth: effectiveMaxDepth,
-          pages_crawled: results.urls.length,
-          pages_found: results.results.length,
-          error_count: results.errors.length,
+          pages_attempted: succeeded + failed,
+          pages_crawled: succeeded,
+          pages_found: succeeded,
+          error_count: failed,
           duration_ms: duration,
-          pages_per_second: results.urls.length / (duration / 1000),
+          // A seed refused before any I/O finishes in 0 ms; x / 0 is Infinity,
+          // which the output schema rejects as "expected number".
+          pages_per_second: succeeded / (Math.max(duration, 1) / 1000),
           results: this.formatResults(results.results, validated.extract_content, validated.content_max_length),
           errors: results.errors,
           stats: results.stats,
-          site_structure: this.analyzeSiteStructure(results.urls, results.results),
+          // Pages only: a URL that failed is in errors[], not in the site's structure.
+          site_structure: this.analyzeSiteStructure(results.results.map(page => page.url), results.results),
           domain_filter_config: domainFilter ? domainFilter.exportConfig() : null,
           link_analysis: results.linkAnalysis,
           session: sessionContext

package/src/tools/crawl/mapSite.js

@@ -48,6 +48,9 @@ export function searchScore(url, search) {
   return terms.filter((t) => haystack.includes(t)).length;
 }
 
+// A gate refusal is the answer to the request, not a page we failed to read.
+const REFUSAL_CODES = new Set(['ROBOTS_DISALLOWED', 'HOST_BLOCKED', 'USE_REDDIT_SEARCH', 'SSRF_BLOCKED']);
+
 // Lazy singleton — avoids creating a CacheManager timer per request
 let _ranker = null;
 function getRanker() {
@@ -159,13 +162,24 @@ export class MapSiteTool {
         sitemapUrls.forEach(url => urls.add(normalizeUrl(url)));
       }
 
-      // Fetch and parse the main page for additional URLs
-      const pageUrls = await this.fetchPageUrls(validated.url, domainFilter, identity);
-      pageUrls.forEach(url => {
-        if (urls.size < poolLimit) {
-          urls.add(normalizeUrl(url));
+      // Fetch and parse the main page for additional URLs. A start page that
+      // cannot be read is an error when the sitemap gave nothing either: an
+      // unresolvable host used to return a success with total_urls: 0 (R24).
+      try {
+        const pageUrls = await this.fetchPageUrls(validated.url, domainFilter, identity);
+        pageUrls.forEach(url => {
+          if (urls.size < poolLimit) {
+            urls.add(normalizeUrl(url));
+          }
+        });
+      } catch (error) {
+        if (REFUSAL_CODES.has(error.code) || error.message.startsWith('SSRF Protection:')) throw error;
+        const reason = error.cause?.message || error.message;
+        if (urls.size === 0) {
+          throw new Error(`${validated.url} could not be read (${reason}) and no sitemap was found`);
         }
-      });
+        warnings.push(`The start page could not be read (${reason}); the URLs come from the sitemap only.`);
+      }
 
       let pool = Array.from(urls);
       if (scopePath) {
@@ -304,46 +318,40 @@ export class MapSiteTool {
   }
 
   async fetchPageUrls(url, domainFilter = null, identity = {}) {
-    try {
-      const response = await this.fetchWithTimeout(url, identity);
-      if (!response.ok) {
-        return [];
-      }
+    // Throws when the page cannot be read — a gate or SSRF refusal, a network
+    // failure, a non-2xx status. execute() decides which of those end the call.
+    const response = await this.fetchWithTimeout(url, identity);
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}`);
+    }
 
-      const html = await response.text();
-      const $ = load(html);
-      const urls = new Set();
-      const baseUrl = getBaseUrl(url);
-
-      // Extract all links
-      $('a[href]').each((_, element) => {
-        const href = $(element).attr('href');
-        if (href && !href.startsWith('#') && !href.startsWith('javascript:')) {
-          try {
-            const absoluteUrl = new URL(href, url);
-            // Only include URLs from the same domain
-            if (absoluteUrl.origin === new URL(baseUrl).origin) {
-              const urlString = absoluteUrl.toString();
-              
-              // Apply domain filter if provided
-              if (!domainFilter || domainFilter.isAllowed(urlString).allowed) {
-                urls.add(urlString);
-              }
+    const html = await response.text();
+    const $ = load(html);
+    const urls = new Set();
+    const baseUrl = getBaseUrl(url);
+
+    // Extract all links
+    $('a[href]').each((_, element) => {
+      const href = $(element).attr('href');
+      if (href && !href.startsWith('#') && !href.startsWith('javascript:')) {
+        try {
+          const absoluteUrl = new URL(href, url);
+          // Only include URLs from the same domain
+          if (absoluteUrl.origin === new URL(baseUrl).origin) {
+            const urlString = absoluteUrl.toString();
+
+            // Apply domain filter if provided
+            if (!domainFilter || domainFilter.isAllowed(urlString).allowed) {
+              urls.add(urlString);
             }
-          } catch {
-            // Invalid URL, skip
           }
+        } catch {
+          // Invalid URL, skip
         }
-      });
+      }
+    });
 
-      return Array.from(urls);
-    } catch (error) {
-      // A gate refusal is the answer to the request, not a page we failed to
-      // read: surface it instead of returning an emptier map than the caller
-      // would notice.
-      if (error.code === 'ROBOTS_DISALLOWED' || error.code === 'HOST_BLOCKED' || error.code === 'USE_REDDIT_SEARCH') throw error;
-      return [];
-    }
+    return Array.from(urls);
   }
 
   async fetchMetadata(urls, metadataMap, identity = {}) {