crawlforge-mcp-server 5.6.7 → 5.6.9

package/src/tools/crawl/mapSite.js

@@ -8,6 +8,7 @@ import { ResultRanker } from '../search/ranking/ResultRanker.js';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 import { CRAWLFORGE_USER_AGENT } from '../../utils/fetchIdentity.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
+import { pageTitle } from '../../utils/pageTitle.js';
 
 // Lazy singleton — avoids creating a CacheManager timer per request
 let _ranker = null;
@@ -280,7 +281,7 @@ export class MapSiteTool {
           const $ = load(html);
           
           metadataMap.set(url, {
-            title: $('title').text().trim(),
+            title: pageTitle($),
             description: $('meta[name="description"]').attr('content') || '',
             keywords: $('meta[name="keywords"]').attr('content') || '',
             h1: $('h1').first().text().trim(),

package/src/tools/extract/_fetchAndParse.js

@@ -11,6 +11,7 @@
  */
 
 import { load } from 'cheerio';
+import { readBody } from 'crawlforge-extractors';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 import { config } from '../../constants/config.js';
 import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
@@ -23,8 +24,15 @@ const DEFAULT_TIMEOUT_MS = 15000;
  * the same cap _fetch.js applies to basic tools, so a large/hostile response
  * (multi-hundred-MB file, endpoint streaming zeros) can't be buffered whole
  * into a JS string and handed to cheerio/JSDOM/Turndown downstream.
- * Content-Length is checked up front; actual bytes read are counted as a
- * backstop for servers that omit or lie about it.
+ *
+ * Decoding goes through crawlforge-extractors' readBody, which honours the
+ * Content-Type charset and the <meta charset> sniff. The local reader it
+ * replaces decoded every body as UTF-8, so scrape, extract_structured,
+ * extract_embedded_state, extract_with_llm and agent all returned mojibake
+ * for Shift_JIS pages (kakaku.com, vector.co.jp, 2ch.sc — R17, 2026-09-04)
+ * while batch_scrape and extract_metadata, already on readBody, read them
+ * correctly. Content-Length is checked up front; actual bytes read are
+ * counted as a backstop for servers that omit or lie about it.
  * @param {Response} response
  * @returns {Promise<string>}
  */
@@ -47,30 +55,7 @@ async function readTextWithSizeCap(response) {
     return response.text();
   }
 
-  const reader = response.body.getReader();
-  const chunks = [];
-  let totalBytes = 0;
-
-  while (true) {
-    const { done, value } = await reader.read();
-    if (done) break;
-    totalBytes += value.byteLength;
-    if (totalBytes > maxBodySize) {
-      reader.cancel();
-      throw new Error(
-        `Response body too large: exceeded limit of ${maxBodySize} bytes`
-      );
-    }
-    chunks.push(value);
-  }
-
-  const mergedBytes = new Uint8Array(totalBytes);
-  let offset = 0;
-  for (const chunk of chunks) {
-    mergedBytes.set(chunk, offset);
-    offset += chunk.byteLength;
-  }
-  return new TextDecoder().decode(mergedBytes);
+  return readBody(response, { maxBytes: maxBodySize });
 }
 
 /**

package/src/tools/extract/extractContent.js

@@ -12,6 +12,8 @@ import { isThinMainContent } from '../scrape/_mainContent.js';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
 import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
+import { readBody } from 'crawlforge-extractors';
+import { config as appConfig } from '../../constants/config.js';
 
 const ExtractContentSchema = z.object({
   url: z.string().url(),
@@ -196,7 +198,9 @@ export class ExtractContentTool {
           throw new Error(`HTTP ${response.status}: ${response.statusText}`);
         }
 
-        html = await response.text();
+        // readBody decodes with the body's real charset; response.text() is
+        // UTF-8-only and mangled Shift_JIS pages (R17, 2026-09-04).
+        html = await readBody(response, { maxBytes: appConfig.fetch.maxBodySize });
         pageTitle = this.extractTitleFromHTML(html);
       }
       }

package/src/tools/extract/extractStructured.js

@@ -56,6 +56,8 @@ function mainContentText($, html, url) {
 
 /** Characters of page text the model is shown; about 6k tokens. */
 const SHOWN_TEXT_BUDGET = 24000;
+/** Characters of the whole-page text kept when main content would crowd it out. */
+const PAGE_HEAD_RESERVE = 6000;
 
 /**
  * The text the model reads: main content first, then the whole page when both
@@ -78,11 +80,19 @@ const SHOWN_TEXT_BUDGET = 24000;
  * @param {string} textContent - whole-page text from fetchAndParse
  * @returns {string}
  */
-function shownText($, html, url, textContent) {
+export function shownText($, html, url, textContent) {
   const main = mainContentText($, html, url);
   if (!main) return textContent;
-  if (main.length + textContent.length > SHOWN_TEXT_BUDGET) return main;
-  return `${main}\n\n${textContent}`;
+  if (main.length + textContent.length <= SHOWN_TEXT_BUDGET) return `${main}\n\n${textContent}`;
+  // Over budget: the article still leads, but the head of the whole-page
+  // text keeps its reserved slice. That head is the site chrome — the nav
+  // that reads "Install (6.3.3)" on swift.org while the main content's table
+  // is titled "Previous Releases" and starts at 6.3.2 — and the link-heavy
+  // lists Readability drops, such as the version list on gradle.org/releases,
+  // where main content alone answered nothing at all (R17, 2026-09-04).
+  const head = textContent.slice(0, PAGE_HEAD_RESERVE);
+  const mainCut = main.slice(0, Math.max(0, SHOWN_TEXT_BUDGET - head.length - 2));
+  return `${mainCut}\n\n${head}`;
 }
 
 /**

package/src/tools/scrape/unifiedScrape.js

@@ -16,6 +16,7 @@ import { extractMainContent, isThinMainContent } from './_mainContent.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
 import { stripHiddenFromDom } from '../../utils/hiddenContent.js';
 import { extractBlockText } from '../basic/extractText.js';
+import { pageTitle } from '../../utils/pageTitle.js';
 
 // ── Schema ────────────────────────────────────────────────────────────────────
 
@@ -132,7 +133,7 @@ function extractMetadataFromDom($, pageUrl) {
 
   const title =
     $('meta[property="og:title"]').attr('content') ||
-    $('title').text().trim() ||
+    pageTitle($) ||
     $('h1').first().text().trim() || '';
 
   const ogTags = {};

package/src/tools/search/redditSearch.js

@@ -396,6 +396,13 @@ export class RedditSearchTool {
     }
     const rows = Array.isArray(data.data) ? data.data : [];
     const results = v.mode === 'posts' ? rows.map(normalizePost) : rows.map(normalizeComment);
+    // Arctic Shift ANDs every term: "Toyota Camry 2026 out the door price paid"
+    // found nothing in r/askcarsales while "Camry OTD" found eight (R18).
+    if (results.length === 0 && v.query && v.query.trim().split(/\s+/).length > 4) {
+      notes.push(
+        `Arctic Shift matches every word of the query, so a long natural-language query (${v.query.trim().split(/\s+/).length} words) usually finds nothing. Retry with two or three keywords.`
+      );
+    }
     return {
       source: 'arctic_shift', mode: v.mode,
       query: v.query ?? null, subreddit: subreddit ?? null, author: author ?? null,

package/src/tools/templates/ScrapeTemplateTool.js

@@ -16,6 +16,7 @@ import { safeFetch } from '../../utils/ssrfGuard.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
 import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
 import { markPreflightRefusal } from '../../server/requestContext.js';
+import { shopifyProductFromJsonLd } from './shopifyJsonLdFallback.js';
 
 /**
  * A caller mistake caught before anything is fetched: no template matches the
@@ -144,6 +145,13 @@ export class ScrapeTemplateTool {
         if (response.status === 429 || response.status === 503) {
           noteRetryAfter(fetchUrl, response.headers.get('retry-after'));
         }
+        // shopify-product reads /products/<handle>.json; a store that refuses
+        // that endpoint (gymshark.com: 403) or a handle that no longer exists
+        // (allbirds.com: 404 after a redirect to a collection) still has the
+        // public product page, whose JSON-LD carries the price (R18).
+        if (templateId === 'shopify-product' && url && fetchUrl !== url && [401, 403, 404, 410].includes(response.status)) {
+          return await this.shopifyPageFallback({ url, tpl, status: response.status, gate, respect_robots, user_agent, timeout });
+        }
         throw new Error(`HTTP ${response.status}: ${response.statusText}`);
       }
       html = await response.text();
@@ -179,6 +187,54 @@ export class ScrapeTemplateTool {
 
     return gate.warnings.length > 0 ? { ...result, warnings: gate.warnings } : result;
   }
+  /**
+   * Read the product page itself when the .json endpoint was refused. The
+   * page goes through the same robots gate and SSRF guard as any fetch.
+   */
+  async shopifyPageFallback({ url, tpl, status, respect_robots, user_agent, timeout }) {
+    const pageGate = await preflightFetch(url, {
+      respectRobots: respect_robots,
+      userAgent: user_agent,
+      tool: 'scrape_template'
+    });
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeout);
+    let response;
+    try {
+      response = await safeFetch(url, { signal: controller.signal, headers: { ...pageGate.headers } });
+    } finally {
+      clearTimeout(timeoutId);
+    }
+    if (!response.ok) {
+      throw new Error(
+        `HTTP ${status} from the products.json endpoint, and the product page ${url} answered HTTP ${response.status} too.`
+      );
+    }
+    const finalUrl = response.url || url;
+    const html = await response.text();
+    const parsed = shopifyProductFromJsonLd(html, finalUrl);
+    if (!parsed.found) {
+      throw new Error(
+        `HTTP ${status} from the products.json endpoint, and ${parsed.reason}` +
+        (finalUrl !== url ? ` (the page redirected to ${finalUrl})` : '') + '.'
+      );
+    }
+    const warnings = [
+      `The store answered /products/<handle>.json with HTTP ${status}; this record was read from the product page's schema.org JSON-LD instead. ` +
+      'Per-variant inventory, compare-at prices and option names are not in JSON-LD, so those fields are null.',
+      ...pageGate.warnings
+    ];
+    return {
+      template: tpl.id,
+      template_name: tpl.name,
+      url,
+      fetchedUrl: finalUrl,
+      data: parsed.data,
+      extractedAt: new Date().toISOString(),
+      warnings
+    };
+  }
+
 }
 
 export default ScrapeTemplateTool;

package/src/tools/templates/shopifyJsonLdFallback.js

@@ -0,0 +1,160 @@
+/**
+ * shopifyJsonLdFallback.js — a Shopify product from the storefront's own
+ * JSON-LD when /products/<handle>.json is refused.
+ *
+ * gymshark.com answers the JSON endpoint with 403 while the product page is
+ * public and carries a schema.org ProductGroup whose hasVariant Products each
+ * carry an Offer (R18, 2026-09-04). allbirds.com redirected a retired handle
+ * to a collection page, so the .json URL was a 404 with no product behind it.
+ * Both used to surface as a bare "HTTP 403/404" from the template. This reads
+ * the page instead and says what it fell back to; per-variant inventory is not
+ * in JSON-LD, so `available` is the offer's availability and variants carry
+ * no stock count.
+ */
+
+import { load } from 'cheerio';
+
+function asArray(value) {
+  if (value == null) return [];
+  return Array.isArray(value) ? value : [value];
+}
+
+function flattenLd(node, out = []) {
+  if (!node || typeof node !== 'object') return out;
+  if (Array.isArray(node)) { node.forEach((n) => flattenLd(n, out)); return out; }
+  out.push(node);
+  if (node['@graph']) flattenLd(node['@graph'], out);
+  return out;
+}
+
+const typed = (re) => (node) => asArray(node?.['@type']).some((t) => re.test(String(t)));
+const isProduct = typed(/^product$/i);
+const isProductGroup = typed(/^productgroup$/i);
+
+function toNumber(value) {
+  if (value == null || value === '') return null;
+  const n = Number(String(value).replace(/[^\d.,-]/g, '').replace(/,(?=\d{3}\b)/g, '').replace(',', '.'));
+  return Number.isFinite(n) ? n : null;
+}
+
+function money(n) {
+  return n == null ? null : n.toFixed(2);
+}
+
+// Gymshark writes its name as "Arrival 5&quot; Shorts" inside the JSON.
+function decodeEntities(value) {
+  if (typeof value !== 'string') return null;
+  const text = /&(?:[a-z]+|#\d+|#x[0-9a-f]+);/i.test(value) ? load(`<x>${value}</x>`)('x').text() : value;
+  return text.replace(/\s+/g, ' ').trim() || null;
+}
+
+function availabilityOf(offer) {
+  const a = String(offer?.availability || '');
+  if (!a) return null;
+  if (/InStock|PreOrder|BackOrder|LimitedAvailability|OnlineOnly/i.test(a)) return true;
+  if (/OutOfStock|SoldOut|Discontinued/i.test(a)) return false;
+  return null;
+}
+
+/** Offers of a node, an AggregateOffer's own list included. */
+function offersOf(node) {
+  return asArray(node?.offers)
+    .flatMap((o) => (o && typed(/AggregateOffer/i)(o) && o.offers ? asArray(o.offers) : [o]))
+    .filter(Boolean);
+}
+
+function offerPrice(offer) {
+  return toNumber(offer.price ?? offer.lowPrice);
+}
+
+/**
+ * @param {string} html - the product page
+ * @param {string} url - the product page URL
+ * @returns {{ found: boolean, data?: object, reason?: string }}
+ */
+export function shopifyProductFromJsonLd(html, url) {
+  const $ = load(html || '');
+  const nodes = [];
+  $('script[type="application/ld+json"]').each((_, el) => {
+    const raw = $(el).html();
+    if (!raw) return;
+    try { flattenLd(JSON.parse(raw), nodes); } catch { /* one bad block does not lose the rest */ }
+  });
+  const product = nodes.find(isProductGroup) || nodes.find(isProduct);
+  if (!product) {
+    return {
+      found: false,
+      reason: `${url} carries no schema.org Product JSON-LD` +
+        (/\/collections\//.test(url) ? ' — the product URL redirected to a collection page, so the product handle no longer exists' : '')
+    };
+  }
+
+  // A ProductGroup (or a Product with hasVariant) prices each variant on the
+  // variant; a plain Product prices its offers directly.
+  const variantNodes = asArray(product.hasVariant).filter(isProduct);
+  const variants = variantNodes.length > 0
+    ? variantNodes.map((v) => {
+      const offers = offersOf(v);
+      const prices = offers.map(offerPrice).filter((n) => n != null);
+      const availabilities = offers.map(availabilityOf).filter((a) => a !== null);
+      const options = ['size', 'color', 'material', 'pattern'].map((k) => decodeEntities(v[k])).filter(Boolean);
+      return {
+        id: v.sku || v.mpn || v.productID || null,
+        title: decodeEntities(v.name) || options.join(' / ') || v.sku || null,
+        price: money(prices.length ? Math.min(...prices) : null),
+        compare_at_price: null,
+        sku: v.sku || null,
+        available: availabilities.length ? availabilities.some(Boolean) : null,
+        inventory_quantity: null,
+        options
+      };
+    })
+    : offersOf(product).map((o) => ({
+      id: o.sku || o.mpn || null,
+      title: decodeEntities(o.name) || o.sku || null,
+      price: money(offerPrice(o)),
+      compare_at_price: null,
+      sku: o.sku || null,
+      available: availabilityOf(o),
+      inventory_quantity: null,
+      options: []
+    }));
+
+  const allOffers = variantNodes.length > 0 ? variantNodes.flatMap(offersOf) : offersOf(product);
+  const prices = allOffers.map(offerPrice).filter((n) => n != null);
+  const highs = allOffers.map((o) => toNumber(o.highPrice)).filter((n) => n != null);
+  const priceMin = prices.length ? Math.min(...prices) : null;
+  const priceMax = prices.length ? Math.max(...prices, ...highs) : (highs.length ? Math.max(...highs) : null);
+  const currency = allOffers.map((o) => o.priceCurrency).find(Boolean) || null;
+  const availabilities = allOffers.map(availabilityOf).filter((v) => v !== null);
+  const available = availabilities.length ? availabilities.some(Boolean) : null;
+
+  let handle = null;
+  try { handle = (new URL(url).pathname.match(/\/products\/([^/?#]+)/) || [])[1] || null; } catch { /* keep null */ }
+
+  const brand = product.brand && typeof product.brand === 'object' ? product.brand.name : product.brand;
+  const images = asArray(product.image).map((i) => (i && typeof i === 'object' ? i.url || i.contentUrl : i)).filter(Boolean);
+  const options = asArray(product.variesBy).map((v) => String(v).split('/').pop().toLowerCase()).filter(Boolean);
+
+  const data = {
+    title: decodeEntities(product.name),
+    vendor: decodeEntities(brand),
+    product_type: decodeEntities(product.category),
+    handle,
+    product_id: product.productGroupID || product.productID || product.sku || null,
+    price: money(priceMin),
+    compare_at_price: null,
+    on_sale: null,
+    currency,
+    price_min: money(priceMin),
+    price_max: money(priceMax),
+    available,
+    variants,
+    options,
+    description: decodeEntities(product.description),
+    images,
+    url: product.url || url,
+    source: 'json-ld'
+  };
+  return { found: true, data };
+}

package/src/utils/challengeDetection.js

@@ -40,6 +40,17 @@ const CHALLENGES = [
     title: /^access denied/i,
     markers: /errors\.edgesuite\.net/i,
     evidence: 'an Akamai access-denied page'
+  },
+  {
+    // Vercel's Attack Challenge Mode answers with HTTP 429, an
+    // x-vercel-mitigated: challenge header and a JavaScript interstitial
+    // titled "Vercel Security Checkpoint" (lesswrong.com, hashicorp.com,
+    // bombas.com, R17 2026-09-04). Chromium solves it and reloads; camoufox
+    // was left on the interstitial, which then read as a successful scrape.
+    vendor: 'vercel',
+    title: /^vercel security checkpoint/i,
+    markers: /vercel\.link\/security-checkpoint|_vercel\/challenge|x-vercel-challenge-token/i,
+    evidence: 'a Vercel Security Checkpoint page'
   }
 ];
 

package/src/utils/pageTitle.js

@@ -0,0 +1,18 @@
+/**
+ * Document title from a cheerio root.
+ *
+ * A bare `$('title')` also matches inline `<svg><title>` elements in the body,
+ * and cheerio's `.text()` concatenates every match: roc-lang.org came back as
+ * "Roc — a fast, friendly, functional languageGitHubYouTubeTwitter..." in the
+ * R17 sweep (2026-09-04), the same defect extract_metadata fixed for MDN on
+ * 2026-08-26. Read the head title first and fall back to the first `<title>`
+ * anywhere for fragments without a head.
+ *
+ * @param {import('cheerio').CheerioAPI} $
+ * @returns {string} trimmed title or ''
+ */
+export function pageTitle($) {
+  return $('head > title').first().text().trim() || $('title').first().text().trim();
+}
+
+export default pageTitle;

package/src/utils/stealthVerdict.js

@@ -0,0 +1,82 @@
+/**
+ * stealthVerdict.js — decide what a stealth-rendered document is.
+ *
+ * A browser scrape that reached a document is not a scrape that succeeded.
+ * Round 18 (2026-09-04, pricing sweep) returned success:true for Edmunds'
+ * "403 - Access Denied" page, Lufthansa's "Page not found", Home Depot's
+ * "Error Page" and Hilton's "Something went wrong" — each an HTTP error or a
+ * soft block served with a title and a few lines of prose, so the R15
+ * challenge-vendor check and the R16 empty-document check both passed it.
+ * The HTTP status of the navigation and a short error-titled document are
+ * now failures too; the content is still returned so the caller can read
+ * what the site served.
+ */
+
+import { detectChallengePage } from './challengeDetection.js';
+
+// A document this short with one of these titles is an error placeholder,
+// not a page. Real pages with these words in a longer title (a news story
+// about an outage) carry far more text than the cap.
+const ERROR_TITLE = /^(?:(?:\d{3}\s*[-–—|:]\s*)?(?:error(?: page)?|access denied|forbidden|(?:page )?not found|service unavailable|internal server error|bad gateway|something went wrong|oops!?[^\n]{0,60}))$/i;
+export const SOFT_ERROR_MAX_CHARS = 1500;
+
+/**
+ * @param {{ url?: string, title?: string, text?: string, html?: string, status?: number|null }} scraped
+ * @param {{ waitedMs?: number, allowEmpty?: boolean }} [options]
+ * @returns {{ success: boolean, status: number|null, error?: string, blocked?: { vendor: string, evidence: string } }}
+ */
+export function stealthDocumentVerdict(scraped, { waitedMs = 0, allowEmpty = false } = {}) {
+  const status = Number.isInteger(scraped?.status) ? scraped.status : null;
+  const url = scraped?.url || '';
+  const title = String(scraped?.title || '').trim();
+  const text = String(scraped?.text || '').trim();
+
+  const challenge = detectChallengePage(scraped || {});
+  if (challenge) {
+    return {
+      success: false,
+      status,
+      blocked: challenge,
+      error: `${challenge.vendor} served a challenge page instead of the content (${challenge.evidence}); the stealth browser did not pass it.`
+    };
+  }
+
+  if (status !== null && status >= 400) {
+    const why = status === 403
+      ? 'A 403 with no challenge vendor on the page is an IP-reputation or WAF block; the site will not serve this network.'
+      : status === 404
+        ? 'The site says the URL does not exist — check the path.'
+        : status === 429
+          ? 'The site is rate-limiting this network; wait before retrying.'
+          : 'Retry later; the server, not the page, failed.';
+    return {
+      success: false,
+      status,
+      error: `HTTP ${status}: ${url} answered with an error page${title ? ` titled "${title}"` : ''}, not the resource; the content returned is that page. ${why}`
+    };
+  }
+
+  if (!title && !text) {
+    if (allowEmpty) return { success: true, status };
+    return {
+      success: false,
+      status,
+      error:
+        `The stealth browser reached ${url} but the document rendered no title and no text` +
+        ` after ${waitedMs}ms of extra wait (${(scraped?.html || '').length} bytes of HTML).` +
+        ' A JavaScript-rendered page needs a longer wait_for; an empty document means the server sent nothing to render.'
+    };
+  }
+
+  if (text.length < SOFT_ERROR_MAX_CHARS && ERROR_TITLE.test(title)) {
+    return {
+      success: false,
+      status,
+      error:
+        `${url} rendered an error page titled "${title}" (${text.length} characters of text) instead of the resource` +
+        ' — a soft block or an application error. Retry later, or with a longer wait_for if the site paints content after a placeholder.'
+    };
+  }
+
+  return { success: true, status };
+}