crawlforge-mcp-server 5.6.8 → 5.6.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.6.8",
+  "version": "5.6.10",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 29 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -116,7 +116,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.6.4",
+    "crawlforge-extractors": "^1.6.5",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",

package/server.js

@@ -34,6 +34,7 @@ import { LocalizationManager } from "./src/core/LocalizationManager.js";
 import * as cheerio from "cheerio";
 import { htmlToMarkdown } from "./src/utils/htmlToMarkdown.js";
 import { detectChallengePage } from "./src/utils/challengeDetection.js";
+import { stealthDocumentVerdict } from "./src/utils/stealthVerdict.js";
 import { browserPreflight } from "./src/utils/robotsGate.js";
 import { memoryMonitor } from "./src/utils/MemoryMonitor.js";
 import { config, validateConfig, getToolConfig } from "./src/constants/config.js";
@@ -106,7 +107,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.6.8",
+  version: "5.6.10",
   description: "Production-ready MCP server with 29 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -1404,28 +1405,20 @@ registerToolIfEnabled("stealth_mode", {
 
         // A bot-wall interstitial arrives as HTTP 200 with a title and prose
         // of its own; reported as a successful scrape it hid the block for
-        // three rounds (R10 Q1 → R15). Name it — the content is still returned
-        // so the caller can see what the site served.
-        const challenge = detectChallengePage(scraped);
+        // three rounds (R10 Q1 → R15). An HTTP error page and a short
+        // error-titled placeholder did the same in R18. Name each — the
+        // content is still returned so the caller can see what the site served.
+        const verdict = stealthDocumentVerdict(scraped, { waitedMs: (wait_for || 0) + (scraped.gracedMs || 0) });
         result = {
-          success: !challenge,
+          success: verdict.success,
           url: scraped.url,
           title: scraped.title,
+          status: verdict.status,
           content: stealthScrapeFormats(formats, scraped)
         };
-        if (challenge) {
-          result.blocked = challenge;
-          result.error = `${challenge.vendor} served a challenge page instead of the content (${challenge.evidence}); the stealth browser did not pass it.`;
-        } else if (!scraped.title && !String(scraped.text || '').trim()) {
-          // A document with no title and no text is not a scraped page; it
-          // is a render that has not happened yet (or an empty response).
-          // Reported as success it reads like a page with nothing on it.
-          result.success = false;
-          result.error =
-            `The stealth browser reached ${scraped.url} but the document rendered no title and no text` +
-            ` after ${wait_for || 0}ms of extra wait (${(scraped.html || '').length} bytes of HTML).` +
-            ' A JavaScript-rendered page needs a longer wait_for; an empty document means the server sent nothing to render.';
-        }
+        if (verdict.blocked) result.blocked = verdict.blocked;
+        if (verdict.error) result.error = verdict.error;
+        if (scraped.gracedMs > 0) result.waited_for_render_ms = scraped.gracedMs;
 
         // Screenshots follow the same crawlforge://screenshot/{id} pattern as
         // scrape and scrape_with_actions, so the base64 never bloats the result.

package/src/core/ActionExecutor.js

@@ -381,6 +381,7 @@ export class ActionExecutor extends EventEmitter {
         url,
         finalUrl: executionContext.finalUrl || url,
         finalHtml: executionContext.finalHtml,
+        navigationStatus: executionContext.page?.__crawlforgeNavigation?.status ?? null,
         executionTime: Date.now() - startTime,
         results: executionContext.results,
         screenshots: executionContext.screenshots,
@@ -1171,7 +1172,7 @@ export class ActionExecutor extends EventEmitter {
     // hostname-based checks alone would miss DNS-rebinding/private-IP targets.
     await assertUrlAllowed(url, { resolveDns: true });
 
-    await page.goto(url, {
+    const response = await page.goto(url, {
       waitUntil: options.waitUntil || 'domcontentloaded',
       timeout: options.timeout || 30000
     });
@@ -1182,6 +1183,14 @@ export class ActionExecutor extends EventEmitter {
     if (/^https?:\/\//i.test(landedUrl)) {
       await assertUrlAllowed(landedUrl, { resolveDns: true });
     }
+
+    // Remembered on the page so the chain result can report the status of
+    // the last navigation: tesla.com's Akamai denial ran a full action chain
+    // and came back success:true (R18, 2026-09-04).
+    let status = null;
+    try { status = response ? response.status() : null; } catch { status = null; }
+    page.__crawlforgeNavigation = { url: landedUrl, status };
+    return { url: landedUrl, status };
   }
 
   /**

package/src/core/LocalizationManager.js

@@ -262,7 +262,7 @@ export class LocalizationManager extends EventEmitter {
       timezone: validatedInput.timezone || countryData.timezone,
       currency: validatedInput.currency || countryData.currency,
       searchDomain: countryData.searchDomain,
-      acceptLanguage: this.buildAcceptLanguageHeader(validatedInput.language || countryData.language),
+      acceptLanguage: this.buildAcceptLanguageHeader(validatedInput.language || countryData.language, validatedInput.countryCode),
       customHeaders: validatedInput.customHeaders || {},
       geoLocation: validatedInput.geoLocation,
       proxySettings: validatedInput.proxySettings,
@@ -601,8 +601,18 @@ export class LocalizationManager extends EventEmitter {
     return await this.configureCountry(countryCode);
   }
   
-  buildAcceptLanguageHeader(language) {
-    const langCode = language.split('-')[0];
+  /**
+   * The header a browser in `countryCode` sends for `language`. A bare
+   * language takes the country as its region — configure_country GB used to
+   * answer en-US (R18, 2026-09-04) because the country never reached here.
+   */
+  buildAcceptLanguageHeader(language, countryCode) {
+    const [langCode, region] = String(language).split('-');
+    const cc = region || (countryCode ? String(countryCode).toUpperCase() : null);
+    if (cc) {
+      const tag = `${langCode}-${cc}`;
+      return langCode === 'en' ? `${tag},en;q=0.9` : `${tag},${langCode};q=0.9,en;q=0.8`;
+    }
     return LANGUAGE_MAPPINGS[langCode] || `${language},${langCode};q=0.9,en;q=0.8`;
   }
   

package/src/core/ResearchOrchestrator.js

@@ -79,6 +79,41 @@ const MAX_CONFLICT_PAIRS = 40;
  * - Advanced provenance tracking and activity logging
  * - Smart URL prioritization based on content quality
  */
+// A research topic is often a paragraph ("Why does X get stuck on Y and what
+// makes Z pass it in 2026: A, B, C, D and known working approaches"). Google
+// answers a 40-word query with nothing: the same run found 1 source from the
+// paragraph and 8 from a six-word restatement (R18, 2026-09-04). The full
+// topic still steers the LLM expansion and the relevance ranking; the search
+// backend gets a keyword-dense form of it.
+export const SEARCH_TOPIC_MAX_WORDS = 12;
+export const SEARCH_QUERY_MAX_WORDS = 16;
+const TOPIC_STOPWORDS = new Set([
+  'a', 'an', 'the', 'and', 'or', 'but', 'of', 'to', 'in', 'on', 'for', 'with', 'from', 'by', 'as', 'at', 'into',
+  'what', 'why', 'how', 'when', 'where', 'which', 'who', 'whom', 'whose', 'that', 'this', 'these', 'those',
+  'does', 'do', 'did', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'it', 'its', 'their', 'there', 'they',
+  'can', 'could', 'should', 'would', 'will', 'may', 'might', 'get', 'gets', 'make', 'makes', 'made', 'about',
+  'known', 'required', 'versus', 'vs', 'i', 'we', 'you', 'my', 'our', 'your', 'me', 'us', 'them', 'not', 'no', 'so'
+]);
+
+export function compactSearchTopic(topic) {
+  const clean = String(topic || '').replace(/\s+/g, ' ').trim();
+  const words = clean.split(' ').filter(Boolean);
+  if (words.length <= SEARCH_TOPIC_MAX_WORDS) return clean;
+  const bare = clean.replace(/[“”"'`()[\]{}:;,?!.]/g, ' ').split(/\s+/).filter(Boolean);
+  const kept = [];
+  for (const word of bare) {
+    if (TOPIC_STOPWORDS.has(word.toLowerCase())) continue;
+    kept.push(word);
+    if (kept.length >= SEARCH_TOPIC_MAX_WORDS) break;
+  }
+  return (kept.length > 0 ? kept : bare.slice(0, SEARCH_TOPIC_MAX_WORDS)).join(' ');
+}
+
+export function clampSearchQuery(query, maxWords = SEARCH_QUERY_MAX_WORDS) {
+  const words = String(query || '').replace(/\s+/g, ' ').trim().split(' ').filter(Boolean);
+  return (words.length > maxWords ? words.slice(0, maxWords) : words).join(' ');
+}
+
 export class ResearchOrchestrator extends EventEmitter {
   constructor(options = {}) {
     super();
@@ -374,9 +409,17 @@ export class ResearchOrchestrator extends EventEmitter {
         });
       }
 
-      // Add research-specific query variations
-      const researchVariations = this.generateResearchVariations(topic);
-      const allQueries = [...new Set([topic, ...expandedQueries, ...researchVariations])];
+      // Add research-specific query variations. The search backend sees the
+      // keyword-dense form of a long topic; every query is clamped to what a
+      // search engine answers.
+      const searchTopic = compactSearchTopic(topic);
+      if (searchTopic !== topic) {
+        this.logger.info('Long research topic compacted for search', { topic, searchTopic });
+      }
+      const researchVariations = this.generateResearchVariations(searchTopic);
+      const allQueries = [...new Set(
+        [searchTopic, ...expandedQueries, ...researchVariations].map((q) => clampSearchQuery(q)).filter(Boolean)
+      )];
 
       // Rank queries by research relevance with semantic understanding
       const rankedQueries = await this.rankResearchQueriesWithSemantics(allQueries, topic);
@@ -390,7 +433,7 @@ export class ResearchOrchestrator extends EventEmitter {
     } catch (error) {
       this.logger.warn('Topic expansion failed, using original topic', { error: error.message });
       this.metrics.queryExpansionTime += Date.now() - startTime;
-      return [topic];
+      return [compactSearchTopic(topic)];
     }
   }
 

package/src/core/StealthBrowserManager.js

@@ -16,6 +16,9 @@ import { BrowserContextPool } from './BrowserContextPool.js';
 import { safeGoto } from '../utils/ssrfGuard.js';
 import { detectChallengePage } from '../utils/challengeDetection.js';
 
+// Grace given to a document that rendered no title and no text (see _waitOutEmptyDocument).
+export const EMPTY_DOCUMENT_GRACE_MS = 8000;
+
 const StealthConfigSchema = z.object({
   level: z.enum(['basic', 'medium', 'advanced']).default('medium'),
   randomizeFingerprint: z.boolean().default(true),
@@ -2024,6 +2027,35 @@ export class StealthBrowserManager {
    * that navigation to reach domcontentloaded. A challenge that never passes
    * is left to challengeDetection to report as blocked.
    */
+  /**
+   * A document with no title and no text right after domcontentloaded is
+   * usually a page that has not painted yet: a self-solving bot wall the
+   * vendor list does not name (booking.com's chal_t redirect), or a client-
+   * rendered shell (carvana.com, chewy.com). Round 18 (2026-09-04) reported
+   * every one of them as "rendered no title and no text after 0ms" while a
+   * 6-second wait in scrape_with_actions returned the full page. Give an
+   * empty document one bounded grace period to fill in — or to navigate on —
+   * before it is read. Returns the milliseconds waited (0 when the document
+   * already had content).
+   */
+  async _waitOutEmptyDocument(page, { timeoutMs = EMPTY_DOCUMENT_GRACE_MS } = {}) {
+    const hasContent = () =>
+      Boolean((document.title && document.title.trim()) || (document.body && document.body.innerText.trim()));
+    let empty;
+    try {
+      empty = !(await page.evaluate(hasContent));
+    } catch {
+      return 0;
+    }
+    if (!empty) return 0;
+    const started = Date.now();
+    await page.waitForFunction(hasContent, null, { timeout: timeoutMs }).catch(() => {});
+    await page.waitForLoadState('domcontentloaded', { timeout: 5000 }).catch(() => {});
+    // What filled the document may itself be a challenge that navigates on.
+    await this._waitOutChallenge(page);
+    return Date.now() - started;
+  }
+
   async _waitOutChallenge(page, { timeoutMs = 8000 } = {}) {
     let title;
     try {
@@ -2046,9 +2078,21 @@ export class StealthBrowserManager {
       const page = await this.createStealthPage(contextId);
       let crashed = false;
       page.on('crash', () => { crashed = true; });
+      // The HTTP status of the document finally read: the navigation's own
+      // response, then whatever a challenge or redirect navigated on to.
+      // Edmunds' "403 - Access Denied" and Lufthansa's 404 came back as
+      // success:true because nothing looked at the status (R18, 2026-09-04).
+      let status = null;
+      page.on('response', (response) => {
+        try {
+          const request = response.request();
+          if (request.isNavigationRequest() && request.frame() === page.mainFrame()) status = response.status();
+        } catch { /* a detached frame; keep the last status */ }
+      });
       // SSRF guard at the navigation boundary: the stealth engine resolves DNS
       // itself, so a URL/host-only check would miss rebinding to private/metadata IPs.
-      await safeGoto(page, url, { waitUntil: 'domcontentloaded' });
+      const response = await safeGoto(page, url, { waitUntil: 'domcontentloaded' });
+      if (response && status === null) status = response.status();
       if (wait_for > 0) await page.waitForTimeout(wait_for);
       // Cloudflare's and Vercel's JavaScript challenges solve themselves in a
       // real browser and then reload the page. camoufox on lesswrong.com was
@@ -2056,6 +2100,7 @@ export class StealthBrowserManager {
       // up, so the interstitial came back as success:true (R17, 2026-09-04).
       // An auto-solving challenge gets one bounded wait to finish first.
       await this._waitOutChallenge(page);
+      const gracedMs = await this._waitOutEmptyDocument(page);
 
       // A failure to read the document is a failure. With every read wrapped
       // in .catch(() => ''), a renderer that crashed or a page closed during
@@ -2082,7 +2127,7 @@ export class StealthBrowserManager {
         ? await page.screenshot({ encoding: 'base64', fullPage: false }).catch(() => null)
         : null;
 
-      return { success: true, url, title, text, html, screenshot: shot };
+      return { success: true, url, title, text, html, screenshot: shot, status, gracedMs };
     } finally {
       await this.closeContext(contextId).catch(() => {});
     }

package/src/core/crawlers/BFSCrawler.js

@@ -41,7 +41,7 @@ export class BFSCrawler {
     // Session context for cookie jar + persistent headers (null = stateless)
     this.sessionContext = sessionContext;
 
-    this.visited = new Set();
+    this.visited = new Map(); // normalized key -> the URL as fetched
     this.results = [];
     this.errors = [];
     this.filterDecisions = []; // Track filtering decisions for analysis
@@ -135,7 +135,7 @@ export class BFSCrawler {
     }
 
     return {
-      urls: Array.from(this.visited),
+      urls: Array.from(this.visited.values()),
       results: this.results,
       errors: this.errors,
       stats: this.getStats(),
@@ -154,6 +154,9 @@ export class BFSCrawler {
     if (this.visited.has(normalizedUrl)) {
       return;
     }
+    // The form that goes on the wire: a bare origin gains its root "/",
+    // everything else (a trailing slash included) stays as written.
+    try { url = new URL(url).href; } catch { /* normalizeUrl accepted it; leave it */ }
 
     // Only the seed is queued at depth 0; children are always depth >= 1.
     const isSeed = depth === 0;
@@ -180,7 +183,11 @@ export class BFSCrawler {
     // Check robots.txt through the shared gate, so this crawl reuses the same
     // cached robots.txt every other tool fetched (and honours the platform
     // blocklist, which no per-request flag can switch off).
-    const gate = await robotsPreflight(normalizedUrl, {
+    // Fetch, resolve links and ask robots about the URL as written: the
+    // normalized form drops a trailing slash, and globalpetrolprices.com
+    // answers /gasoline_prices/ with 200 and /gasoline_prices with 404
+    // (R18, 2026-09-04). The normalized form is the dedupe and cache key only.
+    const gate = await robotsPreflight(url, {
       respectRobots: this.respectRobots,
       userAgent: this.userAgent,
       tool: 'crawl_deep'
@@ -192,9 +199,9 @@ export class BFSCrawler {
       // stay silent: skipping disallowed links is the normal case.
       if (isSeed) {
         this.errors.push({
-          url: normalizedUrl,
+          url,
           depth,
-          error: new RobotsDisallowedError(normalizedUrl).message,
+          error: new RobotsDisallowedError(url).message,
           code: 'ROBOTS_DISALLOWED',
           timestamp: new Date().toISOString()
         });
@@ -209,7 +216,7 @@ export class BFSCrawler {
     if (this.visited.size >= this.maxPages || this.visited.has(normalizedUrl)) {
       return;
     }
-    this.visited.add(normalizedUrl);
+    this.visited.set(normalizedUrl, url);
 
     try {
       // Check cache first
@@ -242,7 +249,7 @@ export class BFSCrawler {
         }
 
         // Fetch the page
-        pageData = await this.fetchPage(normalizedUrl);
+        pageData = await this.fetchPage(url);
         
         // Cache the result
         if (cacheKey) await this.cache.set(cacheKey, pageData);
@@ -255,18 +262,18 @@ export class BFSCrawler {
         const $page = pageData.originalHtml ? load(pageData.originalHtml) : null;
         const pageBodyText = $page ? $page('body').text() : '';
         for (const link of pageData.links) {
-          const absoluteUrl = this.resolveUrl(link, normalizedUrl);
+          const absoluteUrl = this.resolveUrl(link, url);
           if (absoluteUrl) {
             // Extract anchor text and context from link
             const linkMetadata = this.extractLinkMetadata(link, $page, pageBodyText);
-            this.linkAnalyzer.addLink(normalizedUrl, absoluteUrl, linkMetadata);
+            this.linkAnalyzer.addLink(normalizedUrl, normalizeUrl(absoluteUrl), linkMetadata);
           }
         }
       }
 
       // Process the page
       const result = {
-        url: normalizedUrl,
+        url,
         depth,
         title: pageData.title,
         contentLength: pageData.content?.length || 0,
@@ -286,8 +293,8 @@ export class BFSCrawler {
         for (const link of pageData.links) {
           if (this.visited.size >= this.maxPages) break;
           
-          const absoluteUrl = this.resolveUrl(link, normalizedUrl);
-          if (absoluteUrl && !this.visited.has(absoluteUrl)) {
+          const absoluteUrl = this.resolveUrl(link, url);
+          if (absoluteUrl && !this.visited.has(normalizeUrl(absoluteUrl))) {
             // Not awaited: this task already holds a queue slot, so awaiting a
             // child would keep that slot pinned for the rest of the recursive
             // crawl (starving other tasks when concurrency <= depth, and making
@@ -306,7 +313,7 @@ export class BFSCrawler {
       }
     } catch (error) {
       this.errors.push({
-        url: normalizedUrl,
+        url,
         depth,
         error: error.message,
         timestamp: new Date().toISOString()
@@ -425,8 +432,11 @@ export class BFSCrawler {
         if (!this.followExternal && linkUrl.origin !== this.baseUrl.origin) {
           return null;
         }
-        
-        return normalizeUrl(link);
+
+        // Keep the URL as the site wrote it (trailing slash included); only
+        // the fragment goes. Dedupe happens on normalizeUrl() at the caller.
+        linkUrl.hash = '';
+        return linkUrl.toString();
       }
       
       // Handle relative URLs
@@ -437,7 +447,8 @@ export class BFSCrawler {
         return null;
       }
       
-      return normalizeUrl(resolved.toString());
+      resolved.hash = '';
+      return resolved.toString();
     } catch {
       return null;
     }

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -10,6 +10,7 @@ import { load } from 'cheerio';
 
 // Import existing tool for content extraction
 import ExtractContentTool from '../extract/extractContent.js';
+import { stealthDocumentVerdict } from '../../utils/stealthVerdict.js';
 import { elementText } from '../../utils/elementText.js';
 import { pageTitle } from '../../utils/pageTitle.js';
 
@@ -464,14 +465,28 @@ export class ScrapeWithActionsTool extends EventEmitter {
 
     const executionTime = Date.now() - sessionContext.startTime;
 
+    // The chain ran; whether the document it ended on is the page is a
+    // separate question. A bot wall, an HTTP error page or an error-titled
+    // placeholder is named as such (R18, 2026-09-04: tesla.com's Akamai
+    // denial completed every action and reported success).
+    const verdict = stealthDocumentVerdict({
+      url: chainResult?.finalUrl || params.url,
+      title: finalContent?.title || finalContent?.metadata?.title || '',
+      text: finalContent?.content?.text || '',
+      html: chainResult?.finalHtml || '',
+      status: chainResult?.navigationStatus ?? null
+    }, { allowEmpty: true });
+
     return {
-      success: chainResult.success,
+      success: chainResult.success && verdict.success,
       sessionId: sessionContext.id,
       url: params.url,
       executionTime,
 
       // Include error message if action chain failed
-      error: chainResult.error || undefined,
+      error: chainResult.error || verdict.error || undefined,
+      ...(verdict.blocked ? { blocked: verdict.blocked } : {}),
+      ...(verdict.status !== null ? { httpStatus: verdict.status } : {}),
 
       actionResults,
       totalActions: params.actions.length,

package/src/tools/search/redditSearch.js

@@ -396,6 +396,13 @@ export class RedditSearchTool {
     }
     const rows = Array.isArray(data.data) ? data.data : [];
     const results = v.mode === 'posts' ? rows.map(normalizePost) : rows.map(normalizeComment);
+    // Arctic Shift ANDs every term: "Toyota Camry 2026 out the door price paid"
+    // found nothing in r/askcarsales while "Camry OTD" found eight (R18).
+    if (results.length === 0 && v.query && v.query.trim().split(/\s+/).length > 4) {
+      notes.push(
+        `Arctic Shift matches every word of the query, so a long natural-language query (${v.query.trim().split(/\s+/).length} words) usually finds nothing. Retry with two or three keywords.`
+      );
+    }
     return {
       source: 'arctic_shift', mode: v.mode,
       query: v.query ?? null, subreddit: subreddit ?? null, author: author ?? null,

package/src/tools/templates/ScrapeTemplateTool.js

@@ -11,7 +11,7 @@
  * one call — the registry builds the URL, this tool fetches it.
  */
 
-import { TemplateRegistry, retiredTemplate } from 'crawlforge-extractors';
+import { TemplateRegistry, retiredTemplate, shopifyProductFromJsonLd } from 'crawlforge-extractors';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
 import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
@@ -144,6 +144,13 @@ export class ScrapeTemplateTool {
         if (response.status === 429 || response.status === 503) {
           noteRetryAfter(fetchUrl, response.headers.get('retry-after'));
         }
+        // shopify-product reads /products/<handle>.json; a store that refuses
+        // that endpoint (gymshark.com: 403) or a handle that no longer exists
+        // (allbirds.com: 404 after a redirect to a collection) still has the
+        // public product page, whose JSON-LD carries the price (R18).
+        if (templateId === 'shopify-product' && url && fetchUrl !== url && [401, 403, 404, 410].includes(response.status)) {
+          return await this.shopifyPageFallback({ url, tpl, status: response.status, gate, respect_robots, user_agent, timeout });
+        }
         throw new Error(`HTTP ${response.status}: ${response.statusText}`);
       }
       html = await response.text();
@@ -179,6 +186,54 @@ export class ScrapeTemplateTool {
 
     return gate.warnings.length > 0 ? { ...result, warnings: gate.warnings } : result;
   }
+  /**
+   * Read the product page itself when the .json endpoint was refused. The
+   * page goes through the same robots gate and SSRF guard as any fetch.
+   */
+  async shopifyPageFallback({ url, tpl, status, respect_robots, user_agent, timeout }) {
+    const pageGate = await preflightFetch(url, {
+      respectRobots: respect_robots,
+      userAgent: user_agent,
+      tool: 'scrape_template'
+    });
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeout);
+    let response;
+    try {
+      response = await safeFetch(url, { signal: controller.signal, headers: { ...pageGate.headers } });
+    } finally {
+      clearTimeout(timeoutId);
+    }
+    if (!response.ok) {
+      throw new Error(
+        `HTTP ${status} from the products.json endpoint, and the product page ${url} answered HTTP ${response.status} too.`
+      );
+    }
+    const finalUrl = response.url || url;
+    const html = await response.text();
+    const parsed = shopifyProductFromJsonLd(html, finalUrl);
+    if (!parsed.found) {
+      throw new Error(
+        `HTTP ${status} from the products.json endpoint, and ${parsed.reason}` +
+        (finalUrl !== url ? ` (the page redirected to ${finalUrl})` : '') + '.'
+      );
+    }
+    const warnings = [
+      `The store answered /products/<handle>.json with HTTP ${status}; this record was read from the product page's schema.org JSON-LD instead. ` +
+      'Per-variant inventory, compare-at prices and option names are not in JSON-LD, so those fields are null.',
+      ...pageGate.warnings
+    ];
+    return {
+      template: tpl.id,
+      template_name: tpl.name,
+      url,
+      fetchedUrl: finalUrl,
+      data: parsed.data,
+      extractedAt: new Date().toISOString(),
+      warnings
+    };
+  }
+
 }
 
 export default ScrapeTemplateTool;

package/src/utils/stealthVerdict.js

@@ -0,0 +1,82 @@
+/**
+ * stealthVerdict.js — decide what a stealth-rendered document is.
+ *
+ * A browser scrape that reached a document is not a scrape that succeeded.
+ * Round 18 (2026-09-04, pricing sweep) returned success:true for Edmunds'
+ * "403 - Access Denied" page, Lufthansa's "Page not found", Home Depot's
+ * "Error Page" and Hilton's "Something went wrong" — each an HTTP error or a
+ * soft block served with a title and a few lines of prose, so the R15
+ * challenge-vendor check and the R16 empty-document check both passed it.
+ * The HTTP status of the navigation and a short error-titled document are
+ * now failures too; the content is still returned so the caller can read
+ * what the site served.
+ */
+
+import { detectChallengePage } from './challengeDetection.js';
+
+// A document this short with one of these titles is an error placeholder,
+// not a page. Real pages with these words in a longer title (a news story
+// about an outage) carry far more text than the cap.
+const ERROR_TITLE = /^(?:(?:\d{3}\s*[-–—|:]\s*)?(?:error(?: page)?|access denied|forbidden|(?:page )?not found|service unavailable|internal server error|bad gateway|something went wrong|oops!?[^\n]{0,60}))$/i;
+export const SOFT_ERROR_MAX_CHARS = 1500;
+
+/**
+ * @param {{ url?: string, title?: string, text?: string, html?: string, status?: number|null }} scraped
+ * @param {{ waitedMs?: number, allowEmpty?: boolean }} [options]
+ * @returns {{ success: boolean, status: number|null, error?: string, blocked?: { vendor: string, evidence: string } }}
+ */
+export function stealthDocumentVerdict(scraped, { waitedMs = 0, allowEmpty = false } = {}) {
+  const status = Number.isInteger(scraped?.status) ? scraped.status : null;
+  const url = scraped?.url || '';
+  const title = String(scraped?.title || '').trim();
+  const text = String(scraped?.text || '').trim();
+
+  const challenge = detectChallengePage(scraped || {});
+  if (challenge) {
+    return {
+      success: false,
+      status,
+      blocked: challenge,
+      error: `${challenge.vendor} served a challenge page instead of the content (${challenge.evidence}); the stealth browser did not pass it.`
+    };
+  }
+
+  if (status !== null && status >= 400) {
+    const why = status === 403
+      ? 'A 403 with no challenge vendor on the page is an IP-reputation or WAF block; the site will not serve this network.'
+      : status === 404
+        ? 'The site says the URL does not exist — check the path.'
+        : status === 429
+          ? 'The site is rate-limiting this network; wait before retrying.'
+          : 'Retry later; the server, not the page, failed.';
+    return {
+      success: false,
+      status,
+      error: `HTTP ${status}: ${url} answered with an error page${title ? ` titled "${title}"` : ''}, not the resource; the content returned is that page. ${why}`
+    };
+  }
+
+  if (!title && !text) {
+    if (allowEmpty) return { success: true, status };
+    return {
+      success: false,
+      status,
+      error:
+        `The stealth browser reached ${url} but the document rendered no title and no text` +
+        ` after ${waitedMs}ms of extra wait (${(scraped?.html || '').length} bytes of HTML).` +
+        ' A JavaScript-rendered page needs a longer wait_for; an empty document means the server sent nothing to render.'
+    };
+  }
+
+  if (text.length < SOFT_ERROR_MAX_CHARS && ERROR_TITLE.test(title)) {
+    return {
+      success: false,
+      status,
+      error:
+        `${url} rendered an error page titled "${title}" (${text.length} characters of text) instead of the resource` +
+        ' — a soft block or an application error. Retry later, or with a longer wait_for if the site paints content after a placeholder.'
+    };
+  }
+
+  return { success: true, status };
+}