crawlforge-mcp-server 5.6.7 → 5.6.9

package/src/core/StealthBrowserManager.js

@@ -14,6 +14,10 @@ import crypto from 'crypto';
 import HumanBehaviorSimulator from '../utils/HumanBehaviorSimulator.js';
 import { BrowserContextPool } from './BrowserContextPool.js';
 import { safeGoto } from '../utils/ssrfGuard.js';
+import { detectChallengePage } from '../utils/challengeDetection.js';
+
+// Grace given to a document that rendered no title and no text (see _waitOutEmptyDocument).
+export const EMPTY_DOCUMENT_GRACE_MS = 8000;
 
 const StealthConfigSchema = z.object({
   level: z.enum(['basic', 'medium', 'advanced']).default('medium'),
@@ -290,10 +294,19 @@ export class StealthBrowserManager {
       this.browser = null;
     }
 
-    // C2: if the requested engine differs from the running browser, tear it down first.
+    // C2: the requested engine differs from the running browser. Park the
+    // running one instead of closing it: closing took every live context in
+    // it along, so a create_context on Chromium followed by any camoufox
+    // scrape left the caller's contextId pointing at a closed browser
+    // ("Target page, context or browser has been closed", R17 2026-09-04).
+    // Each engine keeps its own browser; cleanup() closes the parked ones.
     if (this.browser && this._launchedEngine && this._launchedEngine !== validatedConfig.engine) {
-      await this.browser.close().catch(() => {});
-      this.browser = null;
+      this._parkedBrowsers ??= new Map();
+      this._parkedBrowsers.set(this._launchedEngine, this.browser);
+      const parked = this._parkedBrowsers.get(validatedConfig.engine);
+      this._parkedBrowsers.delete(validatedConfig.engine);
+      this.browser = parked && parked.isConnected() ? parked : null;
+      this._launchedEngine = this.browser ? validatedConfig.engine : null;
     }
 
     if (this.browser) {
@@ -484,7 +497,14 @@ export class StealthBrowserManager {
       // Policy is "invalid behavior for a normal browser" and rebrowser's
       // bot detector flags it red (R15, 2026-09-04). Every init script here
       // goes through addInitScript/evaluate, which need no CSP bypass.
-      javaScriptEnabled: true
+      javaScriptEnabled: true,
+
+      // Init scripts never reach a service worker, whose navigator therefore
+      // reported the real "HeadlessChrome/151 (Macintosh)" beside a spoofed
+      // Windows Chrome main thread (bot.incolumitas.com, R17 2026-09-04).
+      // Blocking registration removes that vantage point; a page's content
+      // does not depend on its service worker on a first, uncached visit.
+      serviceWorkers: 'block'
     };
 
     // camoufox's Firefox build predates the Browser.setDefaultViewport fields
@@ -499,6 +519,7 @@ export class StealthBrowserManager {
       delete contextOptions.isMobile;
       delete contextOptions.hasTouch;
       delete contextOptions.screen;
+      delete contextOptions.serviceWorkers;
     }
 
     const context = await this.browser.newContext(contextOptions);
@@ -1605,6 +1626,79 @@ export class StealthBrowserManager {
           });
         }
       }, fingerprint.hardware);
+
+      // Init scripts never run inside a dedicated Worker, so a detector that
+      // compares navigator in a worker with the main thread saw the real
+      // platform, deviceMemory and hardwareConcurrency beside the spoofed
+      // ones (bot.incolumitas.com "inconsistentWebWorkerNavigatorPropery",
+      // R17 2026-09-04). At the advanced level, classic workers start from a
+      // blob that patches WorkerNavigator and then importScripts() the real
+      // script; relative importScripts/fetch inside such a worker resolve
+      // against the original script URL. Module workers, data:/blob: worker
+      // URLs and a CSP that refuses blob workers fall through to the native
+      // constructor untouched. camoufox spoofs workers itself.
+      if (config.level === 'advanced' && this._launchedEngine !== 'camoufox') {
+        await context.addInitScript(({ hardware, locale }) => {
+          const NativeWorker = window.Worker;
+          if (typeof NativeWorker !== 'function') return;
+          const primary = locale.split('-')[0];
+          const languages = primary === locale ? [locale] : [locale, primary];
+          const spoof = JSON.stringify({
+            hardwareConcurrency: hardware.hardwareConcurrency,
+            platform: hardware.platform,
+            deviceMemory: hardware.deviceMemory,
+            languages,
+            language: locale
+          });
+          const prelude = (scriptUrl) =>
+            `(() => {
+              const spoof = ${spoof};
+              const proto = self.WorkerNavigator && self.WorkerNavigator.prototype;
+              const define = (key, value) => {
+                try { Object.defineProperty(proto, key, { get: () => value, configurable: true }); } catch (e) {}
+              };
+              if (proto) {
+                define('hardwareConcurrency', spoof.hardwareConcurrency);
+                define('platform', spoof.platform);
+                if ('deviceMemory' in self.navigator) define('deviceMemory', spoof.deviceMemory);
+                define('languages', spoof.languages);
+                define('language', spoof.language);
+              }
+              const base = ${JSON.stringify(scriptUrl)};
+              const resolve = (u) => { try { return new URL(String(u), base).href; } catch (e) { return u; } };
+              const nativeImport = self.importScripts;
+              self.importScripts = (...urls) => nativeImport.apply(self, urls.map(resolve));
+              const nativeFetch = self.fetch;
+              if (typeof nativeFetch === 'function') {
+                self.fetch = (input, init) => nativeFetch.call(self, typeof input === 'string' ? resolve(input) : input, init);
+              }
+              if (self.XMLHttpRequest) {
+                const nativeOpen = self.XMLHttpRequest.prototype.open;
+                self.XMLHttpRequest.prototype.open = function(method, url, ...rest) {
+                  return nativeOpen.call(this, method, resolve(url), ...rest);
+                };
+              }
+            })();
+            importScripts(${JSON.stringify(scriptUrl)});`;
+          const Wrapped = function Worker(scriptURL, options) {
+            try {
+              const isModule = options && options.type === 'module';
+              const absolute = new URL(String(scriptURL), location.href).href;
+              if (!isModule && /^https?:/.test(absolute)) {
+                const blob = new Blob([prelude(absolute)], { type: 'text/javascript' });
+                return new NativeWorker(URL.createObjectURL(blob), options);
+              }
+            } catch (e) {
+              // CSP refused the blob URL or the URL failed to parse: native worker.
+            }
+            return new NativeWorker(scriptURL, options);
+          };
+          Wrapped.prototype = NativeWorker.prototype;
+          Object.defineProperty(Wrapped, 'name', { value: 'Worker' });
+          Wrapped.toString = () => 'function Worker() { [native code] }';
+          window.Worker = Wrapped;
+        }, { hardware: fingerprint.hardware, locale: config.locale });
+      }
     }
 
     // Font spoofing
@@ -1926,6 +2020,56 @@ export class StealthBrowserManager {
    * @param {Object} [params.stealthConfig]     — stealth configuration overrides
    * @returns {Promise<{success:boolean, url:string, title:string, text:string, html:string, screenshot:?string}>}
    */
+  /**
+   * If the document is a self-solving bot-wall interstitial (its title is a
+   * known challenge title), wait up to timeoutMs for the title to change —
+   * the challenge script navigates to the real page when it passes — and for
+   * that navigation to reach domcontentloaded. A challenge that never passes
+   * is left to challengeDetection to report as blocked.
+   */
+  /**
+   * A document with no title and no text right after domcontentloaded is
+   * usually a page that has not painted yet: a self-solving bot wall the
+   * vendor list does not name (booking.com's chal_t redirect), or a client-
+   * rendered shell (carvana.com, chewy.com). Round 18 (2026-09-04) reported
+   * every one of them as "rendered no title and no text after 0ms" while a
+   * 6-second wait in scrape_with_actions returned the full page. Give an
+   * empty document one bounded grace period to fill in — or to navigate on —
+   * before it is read. Returns the milliseconds waited (0 when the document
+   * already had content).
+   */
+  async _waitOutEmptyDocument(page, { timeoutMs = EMPTY_DOCUMENT_GRACE_MS } = {}) {
+    const hasContent = () =>
+      Boolean((document.title && document.title.trim()) || (document.body && document.body.innerText.trim()));
+    let empty;
+    try {
+      empty = !(await page.evaluate(hasContent));
+    } catch {
+      return 0;
+    }
+    if (!empty) return 0;
+    const started = Date.now();
+    await page.waitForFunction(hasContent, null, { timeout: timeoutMs }).catch(() => {});
+    await page.waitForLoadState('domcontentloaded', { timeout: 5000 }).catch(() => {});
+    // What filled the document may itself be a challenge that navigates on.
+    await this._waitOutChallenge(page);
+    return Date.now() - started;
+  }
+
+  async _waitOutChallenge(page, { timeoutMs = 8000 } = {}) {
+    let title;
+    try {
+      title = await page.title();
+    } catch {
+      return;
+    }
+    if (!detectChallengePage({ title })) return;
+    await page
+      .waitForFunction((t) => document.title !== t, title, { timeout: timeoutMs })
+      .catch(() => {});
+    await page.waitForLoadState('domcontentloaded', { timeout: 5000 }).catch(() => {});
+  }
+
   async scrapeWithStealth({ url, engine, wait_for = 0, screenshot = false, stealthConfig = {} } = {}) {
     if (!url) throw new Error('scrapeWithStealth requires a url');
 
@@ -1934,10 +2078,29 @@ export class StealthBrowserManager {
       const page = await this.createStealthPage(contextId);
       let crashed = false;
       page.on('crash', () => { crashed = true; });
+      // The HTTP status of the document finally read: the navigation's own
+      // response, then whatever a challenge or redirect navigated on to.
+      // Edmunds' "403 - Access Denied" and Lufthansa's 404 came back as
+      // success:true because nothing looked at the status (R18, 2026-09-04).
+      let status = null;
+      page.on('response', (response) => {
+        try {
+          const request = response.request();
+          if (request.isNavigationRequest() && request.frame() === page.mainFrame()) status = response.status();
+        } catch { /* a detached frame; keep the last status */ }
+      });
       // SSRF guard at the navigation boundary: the stealth engine resolves DNS
       // itself, so a URL/host-only check would miss rebinding to private/metadata IPs.
-      await safeGoto(page, url, { waitUntil: 'domcontentloaded' });
+      const response = await safeGoto(page, url, { waitUntil: 'domcontentloaded' });
+      if (response && status === null) status = response.status();
       if (wait_for > 0) await page.waitForTimeout(wait_for);
+      // Cloudflare's and Vercel's JavaScript challenges solve themselves in a
+      // real browser and then reload the page. camoufox on lesswrong.com was
+      // read while the "Vercel Security Checkpoint" interstitial was still
+      // up, so the interstitial came back as success:true (R17, 2026-09-04).
+      // An auto-solving challenge gets one bounded wait to finish first.
+      await this._waitOutChallenge(page);
+      const gracedMs = await this._waitOutEmptyDocument(page);
 
       // A failure to read the document is a failure. With every read wrapped
       // in .catch(() => ''), a renderer that crashed or a page closed during
@@ -1964,7 +2127,7 @@ export class StealthBrowserManager {
         ? await page.screenshot({ encoding: 'base64', fullPage: false }).catch(() => null)
         : null;
 
-      return { success: true, url, title, text, html, screenshot: shot };
+      return { success: true, url, title, text, html, screenshot: shot, status, gracedMs };
     } finally {
       await this.closeContext(contextId).catch(() => {});
     }
@@ -2195,9 +2358,18 @@ export class StealthBrowserManager {
     }
 
     // Close browser; if close hangs, kill the process so the OS reclaims it.
+    // Browsers parked by an engine switch are closed the same way.
+    const browsers = [];
     if (this.browser) {
-      const browser = this.browser;
+      browsers.push(this.browser);
       this.browser = null;
+    }
+    if (this._parkedBrowsers) {
+      browsers.push(...this._parkedBrowsers.values());
+      this._parkedBrowsers.clear();
+    }
+    this._launchedEngine = null;
+    for (const browser of browsers) {
       const closed = await withDeadline(browser.close(), 5000);
       if (!closed) {
         try {

package/src/core/analysis/ContentAnalyzer.js

@@ -204,6 +204,9 @@ export class ContentAnalyzer {
       if (analysisOptions.detectLanguage) {
         result.language = await this.detectLanguage(text, analysisOptions);
       }
+      // Topic and keyword extraction pick their tokenizer by language: the
+      // English noun-phrase matcher returns whole clauses for anything else.
+      analysisOptions.detectedLanguage = result.language?.code || null;
 
       // Text summarization
       if (analysisOptions.summarize) {
@@ -440,12 +443,34 @@ export class ContentAnalyzer {
    * @param {Object} options - Extraction options
    * @returns {Promise<Array>} - Array of topics
    */
+  /**
+   * Whether topic/keyword extraction must rank segmented content words
+   * instead of running the English noun-phrase matcher. compromise knows no
+   * grammar but English: on CJK text it emits whole multi-sentence runs as
+   * one "phrase", and on Hindi or Finnish it returned clause-length
+   * "keywords" such as "kaupungissa asuu noin 680 000 ihmistä" (R17,
+   * 2026-09-04). Any detected non-English language, or a text whose letters
+   * are mostly non-Latin, takes the word-frequency path.
+   * @param {string} text
+   * @param {{ detectedLanguage?: string|null }} options
+   * @returns {boolean}
+   */
+  needsSegmentedTerms(text, options = {}) {
+    if (this.isCjkText(text)) return true;
+    const lang = options.detectedLanguage;
+    if (lang && lang !== 'und' && lang !== 'eng' && lang !== 'sco') return true;
+    const letters = text.match(/\p{L}/gu);
+    if (!letters || letters.length === 0) return false;
+    const latin = text.match(/\p{Script=Latin}/gu) || [];
+    return latin.length / letters.length < 0.5;
+  }
+
   async extractTopics(text, options = {}) {
     try {
       // compromise noun-phrase matching is English-only: on CJK text it emits
       // whole multi-sentence runs as one "phrase". Use dictionary-segmented
       // content words, ranked by RAKE-style relative salience like below.
-      if (this.isCjkText(text)) {
+      if (this.needsSegmentedTerms(text, options)) {
         const termFreq = this.cjkContentWordFrequencies(text);
         const maxFreq = Math.max(1, ...Object.values(termFreq));
         return Object.entries(termFreq)
@@ -632,7 +657,7 @@ export class ContentAnalyzer {
     try {
       // compromise is English-only: on CJK text it returns whole multi-sentence
       // runs as single "terms". Rank dictionary-segmented content words instead.
-      if (this.isCjkText(text)) {
+      if (this.needsSegmentedTerms(text, options)) {
         const termFreq = this.cjkContentWordFrequencies(text);
         const totalTerms = Object.values(termFreq).reduce((sum, freq) => sum + freq, 0);
         return Object.entries(termFreq)
@@ -1027,7 +1052,13 @@ export class ContentAnalyzer {
       '也是', '就是', '我们', '你们', '他们', '她们', '它们', '这个', '那个',
       '这些', '那些', '一个', '一些', '以及', '或者', '但是', '因为', '所以',
       '如果', '没有', '可以', '已经', '通过', '由于', '对于', '其中', '并且',
-      '而且', '虽然', '什么', '自己', '这里', '那里'
+      '而且', '虽然', '什么', '自己', '这里', '那里',
+      // Japanese particles, auxiliaries and formal nouns the segmenter emits
+      // as tokens ("ます", "です" led kakaku.com keywords in R17, 2026-09-04)
+      'ます', 'です', 'てい', 'って', 'ので', 'から', 'まで', 'など', 'また',
+      'する', 'いる', 'ある', 'こと', 'これ', 'それ', 'ため', 'よう', 'として',
+      'について', 'という', 'られ', 'れる', 'でき', 'ました', 'ません', 'した',
+      'その', 'この', 'あの', 'もの', 'ところ', 'ここ', 'そこ'
     ];
 
     return cjkStopWords.includes(word);

package/src/core/analysis/sentenceUtils.js

@@ -3,10 +3,13 @@
  * domain names, and other common patterns that contain periods.
  */
 
-// CJK / fullwidth sentence terminators. Unlike the ASCII '.' these are
-// unambiguous — no abbreviation, decimal or initial uses them — and CJK text
-// puts no whitespace after them, so they split on a zero-width boundary.
-const CJK_TERMINATORS = '。．！？；';
+// CJK / fullwidth sentence terminators, plus the Devanagari danda (।) and
+// double danda (॥) that end Hindi, Marathi, Nepali and Sanskrit sentences —
+// a Hindi paragraph of five sentences counted as one (R17, 2026-09-04).
+// Unlike the ASCII '.' these are unambiguous — no abbreviation, decimal or
+// initial uses them — and CJK text puts no whitespace after them, so they
+// split on a zero-width boundary.
+const CJK_TERMINATORS = '。．！？；।॥';
 
 // Common abbreviations that should not trigger sentence splits
 const ABBREVIATIONS = new Set([
@@ -33,16 +36,16 @@ export function splitSentences(text) {
 
   // Split by potential sentence boundaries: . ! ? and the CJK terminators
   // But be smart about abbreviations, numbers, and domain-like patterns
-  const tokens = text.split(/(?<=[.!?])\s+|(?<=[。．！？；])\s*/);
+  const tokens = text.split(/(?<=[.!?])\s+|(?<=[。．！？；।॥])\s*/);
 
   for (const token of tokens) {
     const combined = current ? current + ' ' + token : token;
 
     // Check if the current chunk ends with something that looks like a sentence end
-    const endMatch = combined.match(/([.!?。．！？；])\s*$/);
+    const endMatch = combined.match(/([.!?。．！？；।॥])\s*$/);
     if (endMatch) {
       // Check if the period is likely NOT a sentence boundary
-      const beforePeriod = combined.replace(/[.!?。．！？；]\s*$/, '');
+      const beforePeriod = combined.replace(/[.!?。．！？；।॥]\s*$/, '');
       const lastWord = beforePeriod.split(/\s+/).pop() || '';
       const lastWordLower = lastWord.toLowerCase().replace(/[^a-z]/g, '');
 

package/src/core/crawlers/BFSCrawler.js

@@ -11,6 +11,7 @@ import { safeFetch } from '../../utils/ssrfGuard.js';
 import { robotsPreflight, RobotsDisallowedError } from '../../utils/robotsGate.js';
 import { throttleHost } from '../../utils/hostRateLimiter.js';
 import { CRAWLFORGE_USER_AGENT, identityHeaders } from '../../utils/fetchIdentity.js';
+import { pageTitle } from '../../utils/pageTitle.js';
 
 const logger = new Logger('BFSCrawler');
 
@@ -40,7 +41,7 @@ export class BFSCrawler {
     // Session context for cookie jar + persistent headers (null = stateless)
     this.sessionContext = sessionContext;
 
-    this.visited = new Set();
+    this.visited = new Map(); // normalized key -> the URL as fetched
     this.results = [];
     this.errors = [];
     this.filterDecisions = []; // Track filtering decisions for analysis
@@ -134,7 +135,7 @@ export class BFSCrawler {
     }
 
     return {
-      urls: Array.from(this.visited),
+      urls: Array.from(this.visited.values()),
       results: this.results,
       errors: this.errors,
       stats: this.getStats(),
@@ -153,6 +154,9 @@ export class BFSCrawler {
     if (this.visited.has(normalizedUrl)) {
       return;
     }
+    // The form that goes on the wire: a bare origin gains its root "/",
+    // everything else (a trailing slash included) stays as written.
+    try { url = new URL(url).href; } catch { /* normalizeUrl accepted it; leave it */ }
 
     // Only the seed is queued at depth 0; children are always depth >= 1.
     const isSeed = depth === 0;
@@ -179,7 +183,11 @@ export class BFSCrawler {
     // Check robots.txt through the shared gate, so this crawl reuses the same
     // cached robots.txt every other tool fetched (and honours the platform
     // blocklist, which no per-request flag can switch off).
-    const gate = await robotsPreflight(normalizedUrl, {
+    // Fetch, resolve links and ask robots about the URL as written: the
+    // normalized form drops a trailing slash, and globalpetrolprices.com
+    // answers /gasoline_prices/ with 200 and /gasoline_prices with 404
+    // (R18, 2026-09-04). The normalized form is the dedupe and cache key only.
+    const gate = await robotsPreflight(url, {
       respectRobots: this.respectRobots,
       userAgent: this.userAgent,
       tool: 'crawl_deep'
@@ -191,9 +199,9 @@ export class BFSCrawler {
       // stay silent: skipping disallowed links is the normal case.
       if (isSeed) {
         this.errors.push({
-          url: normalizedUrl,
+          url,
           depth,
-          error: new RobotsDisallowedError(normalizedUrl).message,
+          error: new RobotsDisallowedError(url).message,
           code: 'ROBOTS_DISALLOWED',
           timestamp: new Date().toISOString()
         });
@@ -208,7 +216,7 @@ export class BFSCrawler {
     if (this.visited.size >= this.maxPages || this.visited.has(normalizedUrl)) {
       return;
     }
-    this.visited.add(normalizedUrl);
+    this.visited.set(normalizedUrl, url);
 
     try {
       // Check cache first
@@ -241,7 +249,7 @@ export class BFSCrawler {
         }
 
         // Fetch the page
-        pageData = await this.fetchPage(normalizedUrl);
+        pageData = await this.fetchPage(url);
         
         // Cache the result
         if (cacheKey) await this.cache.set(cacheKey, pageData);
@@ -254,18 +262,18 @@ export class BFSCrawler {
         const $page = pageData.originalHtml ? load(pageData.originalHtml) : null;
         const pageBodyText = $page ? $page('body').text() : '';
         for (const link of pageData.links) {
-          const absoluteUrl = this.resolveUrl(link, normalizedUrl);
+          const absoluteUrl = this.resolveUrl(link, url);
           if (absoluteUrl) {
             // Extract anchor text and context from link
             const linkMetadata = this.extractLinkMetadata(link, $page, pageBodyText);
-            this.linkAnalyzer.addLink(normalizedUrl, absoluteUrl, linkMetadata);
+            this.linkAnalyzer.addLink(normalizedUrl, normalizeUrl(absoluteUrl), linkMetadata);
           }
         }
       }
 
       // Process the page
       const result = {
-        url: normalizedUrl,
+        url,
         depth,
         title: pageData.title,
         contentLength: pageData.content?.length || 0,
@@ -285,8 +293,8 @@ export class BFSCrawler {
         for (const link of pageData.links) {
           if (this.visited.size >= this.maxPages) break;
           
-          const absoluteUrl = this.resolveUrl(link, normalizedUrl);
-          if (absoluteUrl && !this.visited.has(absoluteUrl)) {
+          const absoluteUrl = this.resolveUrl(link, url);
+          if (absoluteUrl && !this.visited.has(normalizeUrl(absoluteUrl))) {
             // Not awaited: this task already holds a queue slot, so awaiting a
             // child would keep that slot pinned for the rest of the recursive
             // crawl (starving other tasks when concurrency <= depth, and making
@@ -305,7 +313,7 @@ export class BFSCrawler {
       }
     } catch (error) {
       this.errors.push({
-        url: normalizedUrl,
+        url,
         depth,
         error: error.message,
         timestamp: new Date().toISOString()
@@ -381,7 +389,7 @@ export class BFSCrawler {
     const $ = load(html);
     
     // Extract title
-    const title = $('title').text().trim() || $('h1').first().text().trim() || '';
+    const title = pageTitle($) || $('h1').first().text().trim() || '';
     
     // Extract main content
     $('script, style, noscript').remove();
@@ -424,8 +432,11 @@ export class BFSCrawler {
         if (!this.followExternal && linkUrl.origin !== this.baseUrl.origin) {
           return null;
         }
-        
-        return normalizeUrl(link);
+
+        // Keep the URL as the site wrote it (trailing slash included); only
+        // the fragment goes. Dedupe happens on normalizeUrl() at the caller.
+        linkUrl.hash = '';
+        return linkUrl.toString();
       }
       
       // Handle relative URLs
@@ -436,7 +447,8 @@ export class BFSCrawler {
         return null;
       }
       
-      return normalizeUrl(resolved.toString());
+      resolved.hash = '';
+      return resolved.toString();
     } catch {
       return null;
     }

package/src/core/integrations/PerformanceIntegration.js

@@ -8,6 +8,7 @@ import WorkerPool from '../workers/WorkerPool.js';
 import ConnectionPool from '../connections/ConnectionPool.js';
 import StreamProcessor from '../processing/StreamProcessor.js';
 import { config } from '../../constants/config.js';
+import { pageTitle } from '../../utils/pageTitle.js';
 
 let performanceManagerInstance = null;
 let workerPoolInstance = null;
@@ -279,7 +280,7 @@ async function parseHtmlSync(html, options = {}) {
 
   if (options.extractText !== false) {
     result.text = $('body').text().trim();
-    result.title = $('title').text().trim();
+    result.title = pageTitle($);
   }
 
   if (options.extractLinks) {

package/src/core/processing/ContentProcessor.js

@@ -9,6 +9,7 @@ import * as cheerio from 'cheerio';
 import { z } from 'zod';
 import { ContentQualityAssessor } from '../../utils/contentUtils.js';
 import { recoverDroppedTables } from '../../tools/scrape/_mainContent.js';
+import { pageTitle } from '../../utils/pageTitle.js';
 
 const ContentProcessorSchema = z.object({
   html: z.string(),
@@ -303,7 +304,7 @@ export class ContentProcessor {
   extractMetadata(html) {
     const $ = cheerio.load(html);
     const metadata = {
-      title: $('title').text().trim() || null,
+      title: pageTitle($) || null,
       description: null,
       keywords: null,
       author: null,
@@ -474,7 +475,7 @@ export class ContentProcessor {
 
     return {
       content: mainContent,
-      title: $('title').text().trim() || null,
+      title: pageTitle($) || null,
       headings: this.extractHeadings($),
       length: mainContent ? mainContent.length : 0
     };

package/src/core/workers/worker.js

@@ -9,6 +9,7 @@ import { JSDOM } from 'jsdom';
 import { Readability } from '@mozilla/readability';
 import compromise from 'compromise';
 import { franc } from 'franc';
+import { pageTitle } from '../../utils/pageTitle.js';
 
 // Task handlers
 const taskHandlers = {
@@ -93,7 +94,7 @@ async function handleParseHtml(data) {
   // Extract text content
   if (extractText) {
     result.text = $('body').text().trim();
-    result.title = $('title').text().trim();
+    result.title = pageTitle($);
   }
 
   // Extract links

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -10,7 +10,9 @@ import { load } from 'cheerio';
 
 // Import existing tool for content extraction
 import ExtractContentTool from '../extract/extractContent.js';
+import { stealthDocumentVerdict } from '../../utils/stealthVerdict.js';
 import { elementText } from '../../utils/elementText.js';
+import { pageTitle } from '../../utils/pageTitle.js';
 
 // Recording / replay helpers
 import {
@@ -463,14 +465,28 @@ export class ScrapeWithActionsTool extends EventEmitter {
 
     const executionTime = Date.now() - sessionContext.startTime;
 
+    // The chain ran; whether the document it ended on is the page is a
+    // separate question. A bot wall, an HTTP error page or an error-titled
+    // placeholder is named as such (R18, 2026-09-04: tesla.com's Akamai
+    // denial completed every action and reported success).
+    const verdict = stealthDocumentVerdict({
+      url: chainResult?.finalUrl || params.url,
+      title: finalContent?.title || finalContent?.metadata?.title || '',
+      text: finalContent?.content?.text || '',
+      html: chainResult?.finalHtml || '',
+      status: chainResult?.navigationStatus ?? null
+    }, { allowEmpty: true });
+
     return {
-      success: chainResult.success,
+      success: chainResult.success && verdict.success,
       sessionId: sessionContext.id,
       url: params.url,
       executionTime,
 
       // Include error message if action chain failed
-      error: chainResult.error || undefined,
+      error: chainResult.error || verdict.error || undefined,
+      ...(verdict.blocked ? { blocked: verdict.blocked } : {}),
+      ...(verdict.status !== null ? { httpStatus: verdict.status } : {}),
 
       actionResults,
       totalActions: params.actions.length,
@@ -625,7 +641,7 @@ export class ScrapeWithActionsTool extends EventEmitter {
         const state = {
           capturePoint: stateData.afterActionIndex + 1,
           url: stateData.url,
-          title: $('title').text().trim(),
+          title: pageTitle($),
           timestamp: stateData.timestamp,
           content: {}
         };

package/src/tools/advanced/batchScrape/worker.js

@@ -12,6 +12,7 @@ import { noteRetryAfter } from '../../../utils/hostRateLimiter.js';
 import { preflightFetch } from '../../../utils/robotsGate.js';
 import { htmlToMarkdown } from '../../../utils/htmlToMarkdown.js';
 import { elementText } from '../../../utils/elementText.js';
+import { pageTitle } from '../../../utils/pageTitle.js';
 
 /**
  * Fetch a URL with AbortController timeout (SSRF-guarded + per-host throttled).
@@ -133,7 +134,7 @@ export function generateFormats($, html, formats) {
   if (formats.includes('markdown')) content.markdown = buildMarkdown($);
   if (formats.includes('json')) {
     content.json = {
-      title: $('title').text().trim(),
+      title: pageTitle($),
       headings: extractHeadings($),
       links: extractLinks($),
       images: extractImages($),
@@ -144,7 +145,7 @@ export function generateFormats($, html, formats) {
 }
 
 function buildMarkdown($) {
-  const title = $('title').text().trim();
+  const title = pageTitle($);
 
   const selectors = ['article', 'main', '.content', '#content', '.post-content', '.entry-content'];
   let $body = null;
@@ -203,7 +204,7 @@ function extractImages($) {
 
 function extractMetadata($) {
   const m = {};
-  m.title = $('title').text().trim();
+  m.title = pageTitle($);
   m.description = $('meta[name="description"]').attr('content') || '';
   m.og = {};
   $('meta[property^="og:"]').each((_, el) => {