crawlforge-mcp-server 6.3.1 → 6.5.0

package/src/utils/htmlToMarkdown.js

@@ -16,10 +16,42 @@
 
 import TurndownService from 'turndown';
 import { gfm } from 'turndown-plugin-gfm';
+import { load } from 'cheerio';
 import { stripHiddenHtml } from './hiddenContent.js';
 
 let _td = null;
 
+/**
+ * turndown-plugin-gfm renders a table as a pipe table only when its first row
+ * is entirely <th>. A corner cell written as an empty <td> — WestJet's fee
+ * table opens `<td> </td><th>1st Bag</th><th>2nd Bag</th>` — fails that test,
+ * so the whole table fell to the layout-table rule and flattened to text
+ * lines with the columns lost (R20, 2026-09-07). Promote empty corner cells
+ * in an otherwise all-<th> first row so the table renders as a table.
+ * @param {string} html
+ * @returns {string}
+ */
+export function promoteCornerHeaderCells(html) {
+  if (!/<th[\s>]/i.test(html)) return html;
+  try {
+    const $ = load(html);
+    let changed = false;
+    $('table').each((_, table) => {
+      const firstRow = $(table).find('tr').first();
+      if (!firstRow.length || firstRow.closest('table')[0] !== table) return;
+      const cells = firstRow.children('th, td');
+      const tds = cells.filter('td');
+      if (cells.filter('th').length === 0 || tds.length === 0) return;
+      if (tds.toArray().some((td) => $(td).text().trim() !== '')) return;
+      tds.each((__, td) => { td.name = 'th'; });
+      changed = true;
+    });
+    return changed ? $.html() : html;
+  } catch {
+    return html;
+  }
+}
+
 // Mirrors turndown-plugin-gfm's own heading-row test, which is what decides
 // whether it converts a table or keeps it as raw HTML.
 function isHeadingRow(tr) {
@@ -112,7 +144,7 @@ export function htmlToMarkdown(html, options = {}) {
     const visible = options.keepHiddenContent
       ? html
       : stripHiddenHtml(html, { css: options.css });
-    return getTurndown().turndown(visible).trim();
+    return getTurndown().turndown(promoteCornerHeaderCells(visible)).trim();
   } catch {
     // Fallback: strip tags, return plain text
     return html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();

package/src/utils/redditHosts.js

@@ -0,0 +1,123 @@
+/**
+ * redditHosts — reddit.com is never fetched.
+ *
+ * reddit.com refuses every non-browser client (403, and the stealth browsers
+ * too — the block is IP/TLS-reputation based), so a scrape or fetch_url
+ * against it always fails and leaves the caller guessing. Reddit is served by
+ * reddit_search, which reads the same posts and comments from the Arctic
+ * Shift community archive (PullPush second). A reddit.com target is therefore
+ * refused before any network work, with the reddit_search call that gets the
+ * same data spelled out — derived from the URL where the URL says enough.
+ *
+ * Runs inside the pre-fetch gate (robotsGate.js), so every fetching tool and
+ * both browser paths get it without knowing. Not overridable: respect_robots
+ * is about robots.txt, and fetching reddit.com fails whatever the caller sends.
+ *
+ * Mirrors the website's `src/lib/tools/reddit-hosts.ts` — same rule, same
+ * message — so a reddit.com URL is answered identically on both surfaces.
+ */
+
+export class UseRedditSearchError extends Error {
+  constructor(url) {
+    const call = redditSearchCallFor(url);
+    const nextStep = Object.keys(call).length
+      ? `reddit_search(${JSON.stringify(call)})`
+      : 'reddit_search with a query, subreddit or author — or mode "thread" with a post\'s link_id';
+    super(
+      `${hostOf(url)} is not fetched: reddit.com refuses every non-browser client, stealth browsers included, ` +
+      `so this call would fail. Reddit is served by reddit_search (5 credits), which reads the same posts ` +
+      `and comments from the Arctic Shift community archive. Next step: ${nextStep}`
+    );
+    this.name = 'UseRedditSearchError';
+    this.code = 'USE_REDDIT_SEARCH';
+    this.url = url;
+    this.redditSearchCall = call;
+  }
+}
+
+/** The hostname of a URL, lowercased, or null if it will not parse. */
+function hostOf(url) {
+  try {
+    return new URL(url).hostname.toLowerCase().replace(/\.$/, '');
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * True for reddit.com and its subdomains (www, old, new, np, sh …) and for the
+ * bare redd.it short-link host. The media hosts (i.redd.it, v.redd.it,
+ * preview.redd.it) serve files to any client and are left alone.
+ * @param {string} url
+ */
+export function isRedditUrl(url) {
+  const host = hostOf(url);
+  if (!host) return false;
+  return host === 'reddit.com' || host.endsWith('.reddit.com') || host === 'redd.it';
+}
+
+/**
+ * The reddit_search call that answers a reddit.com URL, or {} when the URL
+ * names nothing reddit_search can be pointed at (the front page, a wiki, a
+ * settings page).
+ *
+ *   /r/{sub}/comments/{id}/…, /comments/{id}, /gallery/{id}, redd.it/{id}
+ *                                        → mode "thread", link_id
+ *   /r/{sub}/search?q=…                  → query scoped to the subreddit
+ *   /search?q=…                          → an unscoped query
+ *   /r/{sub}[/new|/top|…]                → the subreddit's posts
+ *   /user/{name} or /u/{name}[/comments] → the author's posts (or comments)
+ *   ?type=comment                        → mode "comments"
+ * @param {string} url
+ * @returns {{ mode?: string, query?: string, subreddit?: string, author?: string, link_id?: string }}
+ */
+export function redditSearchCallFor(url) {
+  let parsed;
+  try {
+    parsed = new URL(url);
+  } catch {
+    return {};
+  }
+  const segments = parsed.pathname
+    .split('/')
+    .filter(Boolean)
+    .map((segment) => {
+      try {
+        return decodeURIComponent(segment);
+      } catch {
+        return segment;
+      }
+    });
+
+  if (parsed.hostname.toLowerCase() === 'redd.it') {
+    return segments[0] ? { mode: 'thread', link_id: segments[0] } : {};
+  }
+
+  // A post's id follows "comments" or "gallery" wherever it sits in the path;
+  // a permalink to one comment still reads as its thread.
+  const idAt = segments.findIndex((segment) => segment === 'comments' || segment === 'gallery');
+  if (idAt !== -1 && segments[idAt + 1]) {
+    return { mode: 'thread', link_id: segments[idAt + 1] };
+  }
+
+  const call = {};
+  if (segments[0] === 'r' && segments[1]) call.subreddit = segments[1];
+  if ((segments[0] === 'user' || segments[0] === 'u') && segments[1]) {
+    call.author = segments[1];
+    if (segments[2] === 'comments') call.mode = 'comments';
+  }
+  const query = parsed.searchParams.get('q')?.trim();
+  if (query) call.query = query;
+  if (parsed.searchParams.get('type') === 'comment') call.mode = 'comments';
+  return call;
+}
+
+/**
+ * Throw UseRedditSearchError for a reddit.com target. Call before any network
+ * work — the point is that reddit.com never gets a request, not even for its
+ * robots.txt.
+ * @param {string} url
+ */
+export function assertNotRedditUrl(url) {
+  if (isRedditUrl(url)) throw new UseRedditSearchError(url);
+}

package/src/utils/robotsGate.js

@@ -19,6 +19,7 @@
 
 import { RobotsChecker } from './robotsChecker.js';
 import { assertHostAllowed } from './hostBlocklist.js';
+import { assertNotRedditUrl } from './redditHosts.js';
 import { identityHeaders, resolveUserAgent } from './fetchIdentity.js';
 import { throttleHost } from './hostRateLimiter.js';
 import { recordComplianceEvent, apiKeyId } from './complianceAudit.js';
@@ -75,8 +76,13 @@ export async function robotsPreflight(url, options = {}) {
   // costs the caller nothing: we refused, we fetched nothing.
   try {
     assertHostAllowed(url);
+    // reddit.com refuses every non-browser client, so it is never fetched and
+    // the caller is pointed at reddit_search instead. Also not overridable.
+    assertNotRedditUrl(url);
   } catch (error) {
-    if (error?.code === 'HOST_BLOCKED') markPreflightRefusal('HOST_BLOCKED');
+    if (error?.code === 'HOST_BLOCKED' || error?.code === 'USE_REDDIT_SEARCH') {
+      markPreflightRefusal(error.code);
+    }
     throw error;
   }
 
@@ -178,11 +184,29 @@ export async function preflightFetch(url, options = {}) {
   return {
     headers: outboundHeaders(decision.userAgent, signature),
     userAgent: decision.userAgent,
-    warnings: decision.warnings,
+    warnings: [...decision.warnings, ...crawlDelayWarning(url, decision.crawlDelayMs)],
     overridden: decision.overridden
   };
 }
 
+/**
+ * Name a long Crawl-delay, so a slow multi-page call reads as compliance
+ * rather than a stall: eff.org asks every agent for 30 s, and a 10-page
+ * llms.txt run took 13 minutes with nothing in the response saying why (R21,
+ * 2026-09-09).
+ * @param {string} url
+ * @param {number} crawlDelayMs
+ * @returns {string[]}
+ */
+function crawlDelayWarning(url, crawlDelayMs) {
+  if (!(crawlDelayMs >= 5000)) return [];
+  let host = url;
+  try { host = new URL(url).host; } catch { /* keep the raw url */ }
+  return [
+    `robots.txt on ${host} asks for a ${Math.round(crawlDelayMs / 1000)} s crawl delay; requests to it are spaced by that much, so a multi-page call takes about that long per page.`
+  ];
+}
+
 /**
  * The gate for browser paths. Same decision as {@link preflightFetch}, minus
  * the identity and signature headers — those belong on an HTTP fetch, not on a
@@ -213,7 +237,7 @@ export async function browserPreflight(url, options = {}) {
   }
 
   await throttleHost(url, { crawlDelayMs: decision.crawlDelayMs });
-  return decision.warnings;
+  return [...decision.warnings, ...crawlDelayWarning(url, decision.crawlDelayMs)];
 }
 
 /** Test/diagnostic hook: drop every cached robots.txt. */

package/src/utils/sitemapParser.js

@@ -10,6 +10,23 @@ import { noteRetryAfter } from './hostRateLimiter.js';
 
 const gunzip = promisify(zlib.gunzip);
 
+/**
+ * A sitemap <loc> resolved against the sitemap's own URL and normalized;
+ * null when it is empty or not a URL. The protocol wants absolute locs, but
+ * boeing.com's 1,878-entry sitemap is written with relative paths ("/",
+ * "/commercial"): `normalizeUrl("/")` threw out of the entry loop, the whole
+ * sitemap read as empty, and map_site fell back to crawling links — 75 URLs
+ * (R20, 2026-09-07). One bad entry must not discard the rest either.
+ */
+function resolveLoc(loc, base) {
+  if (!loc) return null;
+  try {
+    return normalizeUrl(new URL(loc, base).href);
+  } catch {
+    return null;
+  }
+}
+
 export class SitemapParser {
   constructor(options = {}) {
     const {
@@ -121,11 +138,11 @@ export class SitemapParser {
       // Parse sitemap index entries
       $('sitemap').each((_, element) => {
         const $sitemap = $(element);
-        const loc = $sitemap.find('loc').text().trim();
-        
+        const loc = resolveLoc($sitemap.find('loc').text().trim(), indexUrl);
+
         if (loc) {
           const sitemap = {
-            url: normalizeUrl(loc),
+            url: loc,
             lastmod: $sitemap.find('lastmod').text().trim() || null
           };
           sitemaps.push(sitemap);
@@ -347,7 +364,9 @@ export class SitemapParser {
     const cacheKey = this.cache?.generateKey(url, { depth: currentDepth });
     if (this.cache && cacheKey) {
       const cached = await this.cache.get(cacheKey);
-      if (cached) {
+      // An empty cached parse is a failure that got remembered (see the
+      // write side below); re-parse rather than serve it for an hour.
+      if (cached && (cached.urls?.length > 0 || cached.sitemaps?.length > 0)) {
         this.stats.cacheHits++;
         return cached;
       }
@@ -378,8 +397,12 @@ export class SitemapParser {
         }
       }
 
-      // Cache the result
-      if (this.cache && cacheKey) {
+      // Cache the result — but never an empty one. A parse that yielded no
+      // URL and no child sitemap is far more likely a failure than a fact
+      // (the relative-<loc> throw above sat in the disk cache for an hour
+      // and map_site kept answering 75 for boeing.com after the parser was
+      // fixed, R20 2026-09-07).
+      if (this.cache && cacheKey && (result.urls.length > 0 || result.sitemaps.length > 0)) {
         await this.cache.set(cacheKey, result);
       }
 
@@ -469,11 +492,11 @@ export class SitemapParser {
     // Parse standard URLs
     $('url').each((_, element) => {
       const $url = $(element);
-      const loc = $url.find('loc').text().trim();
-      
+      const loc = resolveLoc($url.find('loc').text().trim(), url);
+
       if (loc && result.urls.length < this.maxUrlsPerSitemap) {
         const urlData = {
-          loc: normalizeUrl(loc),
+          loc,
           lastmod: $url.find('lastmod').text().trim() || null,
           changefreq: $url.find('changefreq').text().trim() || null,
           priority: $url.find('priority').text().trim() || null
@@ -565,10 +588,10 @@ export class SitemapParser {
 
     $('sitemap').each((_, element) => {
       const $sitemap = $(element);
-      const loc = $sitemap.find('loc').text().trim();
-      
+      const loc = resolveLoc($sitemap.find('loc').text().trim(), url);
+
       if (loc) {
-        result.sitemaps.push(normalizeUrl(loc));
+        result.sitemaps.push(loc);
       }
     });