crawlforge-mcp-server 5.6.7 → 5.6.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.6.7",
+  "version": "5.6.8",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 29 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -116,7 +116,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.6.3",
+    "crawlforge-extractors": "^1.6.4",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",

package/server.js

@@ -106,7 +106,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.6.7",
+  version: "5.6.8",
   description: "Production-ready MCP server with 29 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",

package/src/core/AgentOrchestrator.js

@@ -34,7 +34,11 @@ const DEFAULT_MAX_URLS = 10;
 function isRelevant(text, query) {
   if (!text || !query) return true; // fail-open
   const lc = text.toLowerCase();
-  return query.toLowerCase().split(/\s+/).some(term => term.length > 3 && lc.includes(term));
+  // Terms are compared bare: the quoted entity in `npm package "commander"?`
+  // arrived as `"commander"?` and matched nothing (R17, 2026-09-04).
+  return query.toLowerCase().split(/\s+/)
+    .map(term => term.replace(/[^\p{L}\p{N}]/gu, ''))
+    .some(term => term.length > 3 && lc.includes(term));
 }
 
 /**
@@ -45,6 +49,46 @@ function truncate(text, maxChars = 8000) {
   return text.slice(0, maxChars) + '\n[...truncated]';
 }
 
+/**
+ * Literal provenance check: every version, date and large number in the
+ * answer must appear somewhere in the fetched source text. Returns the raw
+ * strings that do not (deduplicated, in answer order). Comparison is
+ * case-insensitive with whitespace and thousands separators removed, so
+ * "6.3.3" matches "6.3.3" and "1,234" matches "1 234". URLs in the answer are
+ * ignored: they are citations, not claims.
+ */
+const MONTHS = '(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*';
+const VALUE_PATTERNS = [
+  /\b\d+\.\d+(?:\.\d+){0,2}\b/g,                                   // versions 1.2 / 1.2.3 / 1.2.3.4
+  /\b\d{4}-\d{2}-\d{2}\b/g,                                          // ISO dates
+  new RegExp(`\\b${MONTHS}\\.?\\s+\\d{1,2}(?:st|nd|rd|th)?,?\\s+\\d{4}\\b`, 'gi'), // January 18, 2024
+  new RegExp(`\\b\\d{1,2}(?:st|nd|rd|th)?\\s+${MONTHS}\\.?\\s+\\d{4}\\b`, 'gi'),   // 18 January 2024
+  /\b\d{1,3}(?:,\d{3})+\b|\b\d{4,}\b/g,                              // 1,234 / 12345
+];
+
+function normalizeValue(value) {
+  return value.toLowerCase().replace(/[\s,]/g, '');
+}
+
+export function unverifiedValues(answer, sourceText) {
+  if (!answer) return [];
+  const stripped = answer.replace(/https?:\/\/\S+/g, ' ');
+  const haystack = normalizeValue(sourceText || '');
+  const missing = [];
+  const seen = new Set();
+  for (const pattern of VALUE_PATTERNS) {
+    for (const match of stripped.matchAll(pattern)) {
+      const raw = match[0];
+      const key = normalizeValue(raw);
+      if (!key || seen.has(key)) continue;
+      seen.add(key);
+      if (!haystack.includes(key)) missing.push(raw);
+    }
+  }
+  // "2024" inside an already-flagged "January 18, 2024" is the same claim.
+  return missing.filter((v, i) => !missing.some((o, j) => j !== i && o.length > v.length && o.includes(v)));
+}
+
 /**
  * Current-state task gate: prompts about the live "now" ("right now",
  * "currently", "today", "latest", "#1 … now") go stale through search alone —
@@ -222,6 +266,17 @@ export class AgentOrchestrator {
     } catch {
       // Sampling unavailable — use raw prompt
     }
+    // A quoted term in the prompt ("commander") is the thing being asked
+    // about. A planner told to search the bare entity reduced
+    // 'npm package "commander"' to "npm", fetched only npm's own docs and
+    // answered from memory (R17, 2026-09-04). Every query carries the quoted
+    // terms.
+    const quotedTerms = [...prompt.matchAll(/"([^"\n]{2,60})"/g)].map(m => m[1].trim()).filter(Boolean);
+    if (quotedTerms.length > 0) {
+      searchQueries = searchQueries.map(q =>
+        quotedTerms.every(t => q.toLowerCase().includes(t.toLowerCase())) ? q : `${q} ${quotedTerms.join(' ')}`.trim()
+      );
+    }
 
     // ── GATHER (search) ───────────────────────────────────────────────────────
     const urlQueue = [...seedUrls]; // start with any user-provided seeds
@@ -238,6 +293,8 @@ export class AgentOrchestrator {
       if (!priorityUrls.includes(url)) priorityUrls.push(url);
     }
     const searchResults = [];
+    /** url -> { title, text }: the engine's excerpt, evidence of last resort for a page that cannot be fetched. */
+    const excerpts = new Map();
 
     if (urlQueue.length < capUrls) {
       try {
@@ -253,6 +310,11 @@ export class AgentOrchestrator {
               for (const r of parsed.results) {
                 if (r.link && !urlQueue.includes(r.link)) urlQueue.push(r.link);
                 searchResults.push({ query: q, title: r.title || '', url: r.link || '', snippet: r.snippet || '' });
+                // What the engine saw of the page: the snippet plus the
+                // page's own meta description, which for npmjs.com carries
+                // "Latest version: 15.0.0" while the snippet does not.
+                const excerpt = [r.snippet, r.pagemap?.metatags?.description].filter(Boolean).join(' ');
+                if (r.link && excerpt && !excerpts.has(r.link)) excerpts.set(r.link, { title: r.title || '', text: excerpt });
               }
             }
           } catch { /* skip failed search */ }
@@ -318,7 +380,17 @@ export class AgentOrchestrator {
           text: truncate(textContent),
           step
         });
-      } catch { /* skip unreachable URL */ }
+      } catch {
+        // The page is out of reach (challenge page, 403, timeout) but the
+        // search engine's excerpt of it is not: npmjs.com's "Latest version:
+        // 15.0.0" sat in the snippet while every fetch of the package page was
+        // challenged, and the run ended with no evidence at all (R17,
+        // 2026-09-04). Keep a relevant snippet as evidence, labelled as one.
+        const excerpt = excerpts.get(url);
+        if (excerpt && isRelevant(excerpt.text, prompt)) {
+          evidence.push({ url, title: excerpt.title, text: excerpt.text, snippet: true, step });
+        }
+      }
     }
 
     // ── SHAPE ─────────────────────────────────────────────────────────────────
@@ -340,7 +412,7 @@ export class AgentOrchestrator {
       .sort((a, b) => b._score - a._score);
     const perSourceCap = Math.max(1500, Math.floor(12000 / Math.max(evidence.length, 1)));
     const combinedText = orderedEvidence
-      .map(e => `--- Source: ${e.url} ---\n${truncate(e.text, perSourceCap)}`)
+      .map(e => `--- Source: ${e.url}${e.snippet ? ' (search-result snippet; the page itself could not be fetched)' : ''} ---\n${truncate(e.text, perSourceCap)}`)
       .join('\n\n');
 
     if (!combinedText.trim()) {
@@ -386,6 +458,7 @@ export class AgentOrchestrator {
     let answer = null;
     let degraded = false;
     let degradedReason;
+    let unverified = [];
 
     try {
       // Wording matters for small local models (llama3.2-class): without the
@@ -414,6 +487,34 @@ export class AgentOrchestrator {
 
       const { text } = await this._getSamplingClient().complete(synthesisPrompt, { maxTokens: 1024 });
       answer = text;
+
+      // A small model fills gaps from memory: "Swift 5.9.3", "commander 8.6.2
+      // published January 18, 2024" — none of it on any fetched page (R17,
+      // 2026-09-04). Versions, dates and counts in the answer must appear in
+      // the source text; what does not gets one corrective rewrite, and
+      // whatever still remains is named in the answer and in `provenance`.
+      unverified = unverifiedValues(answer, combinedText);
+      if (unverified.length > 0) {
+        const retryPrompt =
+          `${synthesisPrompt}\n\nYour previous answer was:\n${fenceUntrusted(answer, 'previous answer')}\n` +
+          `It contains values that appear nowhere in the source text: ${unverified.map(v => `"${v}"`).join(', ')}. ` +
+          `Rewrite the answer. Keep only values that the sources state, spelled as the sources spell them. ` +
+          `Where the sources do not state a value, say that it is not stated instead of guessing.`;
+        try {
+          const { text: rewritten } = await this._getSamplingClient().complete(retryPrompt, { maxTokens: 1024 });
+          if (rewritten && rewritten.trim()) {
+            answer = rewritten;
+            unverified = unverifiedValues(answer, combinedText);
+          }
+        } catch {
+          // Keep the first answer; it is flagged below.
+        }
+      }
+      if (unverified.length > 0) {
+        answer +=
+          `\n\nProvenance warning: ${unverified.map(v => `"${v}"`).join(', ')} ` +
+          `${unverified.length === 1 ? 'does' : 'do'} not appear in the fetched sources and may be invented.`;
+      }
     } catch (err) {
       degraded = true;
       degradedReason = `LLM synthesis unavailable: ${err.message}`;
@@ -425,11 +526,12 @@ export class AgentOrchestrator {
       success: true,
       answer,
       search_results: searchResults,
-      evidence: degraded ? evidence : evidence.map(e => ({ url: e.url })),
+      evidence: degraded ? evidence : evidence.map(e => (e.snippet ? { url: e.url, snippet: true } : { url: e.url })),
       degraded,
       reason: degradedReason,
       steps: step,
-      urls_fetched: urlsFetched
+      urls_fetched: urlsFetched,
+      provenance: { checked: !degraded, unverified }
     };
   }
 

package/src/core/ChangeTracker.js

@@ -13,6 +13,7 @@ import { load } from 'cheerio';
 import { diffWords, diffLines, diffChars } from 'diff';
 import { structureSignature, structuralSimilarity } from 'crawlforge-extractors';
 import { calculateSimilarity as calculateContentSimilarity } from '../tools/tracking/trackChanges/differ.js';
+import { pageTitle } from '../utils/pageTitle.js';
 
 const ChangeTrackingSchema = z.object({
   url: z.string().url(),
@@ -892,7 +893,7 @@ export class ChangeTracker extends EventEmitter {
   
   extractMetadata($, options) {
     const metadata = {
-      title: $('title').text() || '',
+      title: pageTitle($) || '',
       headings: [],
       links: [],
       images: [],

package/src/core/LLMsTxtAnalyzer.js

@@ -9,6 +9,7 @@ import { safeFetch } from '../utils/ssrfGuard.js';
 import { resolveUserAgent } from '../utils/fetchIdentity.js';
 import { preflightFetch } from '../utils/robotsGate.js';
 import { noteRetryAfter } from '../utils/hostRateLimiter.js';
+import { pageTitle } from '../utils/pageTitle.js';
 
 const logger = new Logger('LLMsTxtAnalyzer');
 
@@ -722,7 +723,7 @@ export class LLMsTxtAnalyzer {
         type: 'static', 
         confidence: 0.7,
         metadata: { 
-          title: $('title').text().trim(),
+          title: pageTitle($),
           contentLength: html.length 
         }
       };

package/src/core/LocalizationManager.js

@@ -679,11 +679,15 @@ export class LocalizationManager extends EventEmitter {
       return this.timezoneCache.get(timezone);
     }
     
-    // Calculate dynamically if not cached
+    // Calculate dynamically if not cached. Compare the target zone against
+    // UTC, never against the host zone: the old form added the host offset
+    // once more, so America/Sao_Paulo came out as -420 from a UTC-4 host
+    // (R17, 2026-09-04). Both wall-clock strings are parsed the same way, so
+    // the difference is the zone's current offset in minutes.
     const now = new Date();
-    const utc = new Date(now.getTime() + (now.getTimezoneOffset() * 60000));
-    const targetTime = new Date(utc.toLocaleString('en-US', { timeZone: timezone }));
-    const offset = (targetTime.getTime() - utc.getTime()) / (1000 * 60);
+    const asTarget = new Date(now.toLocaleString('en-US', { timeZone: timezone }));
+    const asUtc = new Date(now.toLocaleString('en-US', { timeZone: 'UTC' }));
+    const offset = Math.round((asTarget.getTime() - asUtc.getTime()) / 60000);
     
     this.timezoneCache.set(timezone, offset);
     return offset;
@@ -1344,10 +1348,13 @@ export class LocalizationManager extends EventEmitter {
     }
     
     // CJK script detection
+    // Kana and Hangul are tested before the unified ideographs: Japanese
+    // text is full of kanji, so a chinese-first order labelled every
+    // Japanese page "chinese" (R17, 2026-09-04).
     const cjkPatterns = {
-      chinese: /[\u4E00-\u9FFF]/,
       japanese: /[\u3040-\u309F\u30A0-\u30FF]/,
-      korean: /[\uAC00-\uD7AF]/
+      korean: /[\uAC00-\uD7AF]/,
+      chinese: /[\u4E00-\u9FFF]/
     };
     
     for (const [script, pattern] of Object.entries(cjkPatterns)) {

package/src/core/StealthBrowserManager.js

@@ -14,6 +14,7 @@ import crypto from 'crypto';
 import HumanBehaviorSimulator from '../utils/HumanBehaviorSimulator.js';
 import { BrowserContextPool } from './BrowserContextPool.js';
 import { safeGoto } from '../utils/ssrfGuard.js';
+import { detectChallengePage } from '../utils/challengeDetection.js';
 
 const StealthConfigSchema = z.object({
   level: z.enum(['basic', 'medium', 'advanced']).default('medium'),
@@ -290,10 +291,19 @@ export class StealthBrowserManager {
       this.browser = null;
     }
 
-    // C2: if the requested engine differs from the running browser, tear it down first.
+    // C2: the requested engine differs from the running browser. Park the
+    // running one instead of closing it: closing took every live context in
+    // it along, so a create_context on Chromium followed by any camoufox
+    // scrape left the caller's contextId pointing at a closed browser
+    // ("Target page, context or browser has been closed", R17 2026-09-04).
+    // Each engine keeps its own browser; cleanup() closes the parked ones.
     if (this.browser && this._launchedEngine && this._launchedEngine !== validatedConfig.engine) {
-      await this.browser.close().catch(() => {});
-      this.browser = null;
+      this._parkedBrowsers ??= new Map();
+      this._parkedBrowsers.set(this._launchedEngine, this.browser);
+      const parked = this._parkedBrowsers.get(validatedConfig.engine);
+      this._parkedBrowsers.delete(validatedConfig.engine);
+      this.browser = parked && parked.isConnected() ? parked : null;
+      this._launchedEngine = this.browser ? validatedConfig.engine : null;
     }
 
     if (this.browser) {
@@ -484,7 +494,14 @@ export class StealthBrowserManager {
       // Policy is "invalid behavior for a normal browser" and rebrowser's
       // bot detector flags it red (R15, 2026-09-04). Every init script here
       // goes through addInitScript/evaluate, which need no CSP bypass.
-      javaScriptEnabled: true
+      javaScriptEnabled: true,
+
+      // Init scripts never reach a service worker, whose navigator therefore
+      // reported the real "HeadlessChrome/151 (Macintosh)" beside a spoofed
+      // Windows Chrome main thread (bot.incolumitas.com, R17 2026-09-04).
+      // Blocking registration removes that vantage point; a page's content
+      // does not depend on its service worker on a first, uncached visit.
+      serviceWorkers: 'block'
     };
 
     // camoufox's Firefox build predates the Browser.setDefaultViewport fields
@@ -499,6 +516,7 @@ export class StealthBrowserManager {
       delete contextOptions.isMobile;
       delete contextOptions.hasTouch;
       delete contextOptions.screen;
+      delete contextOptions.serviceWorkers;
     }
 
     const context = await this.browser.newContext(contextOptions);
@@ -1605,6 +1623,79 @@ export class StealthBrowserManager {
           });
         }
       }, fingerprint.hardware);
+
+      // Init scripts never run inside a dedicated Worker, so a detector that
+      // compares navigator in a worker with the main thread saw the real
+      // platform, deviceMemory and hardwareConcurrency beside the spoofed
+      // ones (bot.incolumitas.com "inconsistentWebWorkerNavigatorPropery",
+      // R17 2026-09-04). At the advanced level, classic workers start from a
+      // blob that patches WorkerNavigator and then importScripts() the real
+      // script; relative importScripts/fetch inside such a worker resolve
+      // against the original script URL. Module workers, data:/blob: worker
+      // URLs and a CSP that refuses blob workers fall through to the native
+      // constructor untouched. camoufox spoofs workers itself.
+      if (config.level === 'advanced' && this._launchedEngine !== 'camoufox') {
+        await context.addInitScript(({ hardware, locale }) => {
+          const NativeWorker = window.Worker;
+          if (typeof NativeWorker !== 'function') return;
+          const primary = locale.split('-')[0];
+          const languages = primary === locale ? [locale] : [locale, primary];
+          const spoof = JSON.stringify({
+            hardwareConcurrency: hardware.hardwareConcurrency,
+            platform: hardware.platform,
+            deviceMemory: hardware.deviceMemory,
+            languages,
+            language: locale
+          });
+          const prelude = (scriptUrl) =>
+            `(() => {
+              const spoof = ${spoof};
+              const proto = self.WorkerNavigator && self.WorkerNavigator.prototype;
+              const define = (key, value) => {
+                try { Object.defineProperty(proto, key, { get: () => value, configurable: true }); } catch (e) {}
+              };
+              if (proto) {
+                define('hardwareConcurrency', spoof.hardwareConcurrency);
+                define('platform', spoof.platform);
+                if ('deviceMemory' in self.navigator) define('deviceMemory', spoof.deviceMemory);
+                define('languages', spoof.languages);
+                define('language', spoof.language);
+              }
+              const base = ${JSON.stringify(scriptUrl)};
+              const resolve = (u) => { try { return new URL(String(u), base).href; } catch (e) { return u; } };
+              const nativeImport = self.importScripts;
+              self.importScripts = (...urls) => nativeImport.apply(self, urls.map(resolve));
+              const nativeFetch = self.fetch;
+              if (typeof nativeFetch === 'function') {
+                self.fetch = (input, init) => nativeFetch.call(self, typeof input === 'string' ? resolve(input) : input, init);
+              }
+              if (self.XMLHttpRequest) {
+                const nativeOpen = self.XMLHttpRequest.prototype.open;
+                self.XMLHttpRequest.prototype.open = function(method, url, ...rest) {
+                  return nativeOpen.call(this, method, resolve(url), ...rest);
+                };
+              }
+            })();
+            importScripts(${JSON.stringify(scriptUrl)});`;
+          const Wrapped = function Worker(scriptURL, options) {
+            try {
+              const isModule = options && options.type === 'module';
+              const absolute = new URL(String(scriptURL), location.href).href;
+              if (!isModule && /^https?:/.test(absolute)) {
+                const blob = new Blob([prelude(absolute)], { type: 'text/javascript' });
+                return new NativeWorker(URL.createObjectURL(blob), options);
+              }
+            } catch (e) {
+              // CSP refused the blob URL or the URL failed to parse: native worker.
+            }
+            return new NativeWorker(scriptURL, options);
+          };
+          Wrapped.prototype = NativeWorker.prototype;
+          Object.defineProperty(Wrapped, 'name', { value: 'Worker' });
+          Wrapped.toString = () => 'function Worker() { [native code] }';
+          window.Worker = Wrapped;
+        }, { hardware: fingerprint.hardware, locale: config.locale });
+      }
     }
 
     // Font spoofing
@@ -1926,6 +2017,27 @@ export class StealthBrowserManager {
    * @param {Object} [params.stealthConfig]     — stealth configuration overrides
    * @returns {Promise<{success:boolean, url:string, title:string, text:string, html:string, screenshot:?string}>}
    */
+  /**
+   * If the document is a self-solving bot-wall interstitial (its title is a
+   * known challenge title), wait up to timeoutMs for the title to change —
+   * the challenge script navigates to the real page when it passes — and for
+   * that navigation to reach domcontentloaded. A challenge that never passes
+   * is left to challengeDetection to report as blocked.
+   */
+  async _waitOutChallenge(page, { timeoutMs = 8000 } = {}) {
+    let title;
+    try {
+      title = await page.title();
+    } catch {
+      return;
+    }
+    if (!detectChallengePage({ title })) return;
+    await page
+      .waitForFunction((t) => document.title !== t, title, { timeout: timeoutMs })
+      .catch(() => {});
+    await page.waitForLoadState('domcontentloaded', { timeout: 5000 }).catch(() => {});
+  }
+
   async scrapeWithStealth({ url, engine, wait_for = 0, screenshot = false, stealthConfig = {} } = {}) {
     if (!url) throw new Error('scrapeWithStealth requires a url');
 
@@ -1938,6 +2050,12 @@ export class StealthBrowserManager {
       // itself, so a URL/host-only check would miss rebinding to private/metadata IPs.
       await safeGoto(page, url, { waitUntil: 'domcontentloaded' });
       if (wait_for > 0) await page.waitForTimeout(wait_for);
+      // Cloudflare's and Vercel's JavaScript challenges solve themselves in a
+      // real browser and then reload the page. camoufox on lesswrong.com was
+      // read while the "Vercel Security Checkpoint" interstitial was still
+      // up, so the interstitial came back as success:true (R17, 2026-09-04).
+      // An auto-solving challenge gets one bounded wait to finish first.
+      await this._waitOutChallenge(page);
 
       // A failure to read the document is a failure. With every read wrapped
       // in .catch(() => ''), a renderer that crashed or a page closed during
@@ -2195,9 +2313,18 @@ export class StealthBrowserManager {
     }
 
     // Close browser; if close hangs, kill the process so the OS reclaims it.
+    // Browsers parked by an engine switch are closed the same way.
+    const browsers = [];
     if (this.browser) {
-      const browser = this.browser;
+      browsers.push(this.browser);
       this.browser = null;
+    }
+    if (this._parkedBrowsers) {
+      browsers.push(...this._parkedBrowsers.values());
+      this._parkedBrowsers.clear();
+    }
+    this._launchedEngine = null;
+    for (const browser of browsers) {
       const closed = await withDeadline(browser.close(), 5000);
       if (!closed) {
         try {

package/src/core/analysis/ContentAnalyzer.js

@@ -204,6 +204,9 @@ export class ContentAnalyzer {
       if (analysisOptions.detectLanguage) {
         result.language = await this.detectLanguage(text, analysisOptions);
       }
+      // Topic and keyword extraction pick their tokenizer by language: the
+      // English noun-phrase matcher returns whole clauses for anything else.
+      analysisOptions.detectedLanguage = result.language?.code || null;
 
       // Text summarization
       if (analysisOptions.summarize) {
@@ -440,12 +443,34 @@ export class ContentAnalyzer {
    * @param {Object} options - Extraction options
    * @returns {Promise<Array>} - Array of topics
    */
+  /**
+   * Whether topic/keyword extraction must rank segmented content words
+   * instead of running the English noun-phrase matcher. compromise knows no
+   * grammar but English: on CJK text it emits whole multi-sentence runs as
+   * one "phrase", and on Hindi or Finnish it returned clause-length
+   * "keywords" such as "kaupungissa asuu noin 680 000 ihmistä" (R17,
+   * 2026-09-04). Any detected non-English language, or a text whose letters
+   * are mostly non-Latin, takes the word-frequency path.
+   * @param {string} text
+   * @param {{ detectedLanguage?: string|null }} options
+   * @returns {boolean}
+   */
+  needsSegmentedTerms(text, options = {}) {
+    if (this.isCjkText(text)) return true;
+    const lang = options.detectedLanguage;
+    if (lang && lang !== 'und' && lang !== 'eng' && lang !== 'sco') return true;
+    const letters = text.match(/\p{L}/gu);
+    if (!letters || letters.length === 0) return false;
+    const latin = text.match(/\p{Script=Latin}/gu) || [];
+    return latin.length / letters.length < 0.5;
+  }
+
   async extractTopics(text, options = {}) {
     try {
       // compromise noun-phrase matching is English-only: on CJK text it emits
       // whole multi-sentence runs as one "phrase". Use dictionary-segmented
       // content words, ranked by RAKE-style relative salience like below.
-      if (this.isCjkText(text)) {
+      if (this.needsSegmentedTerms(text, options)) {
         const termFreq = this.cjkContentWordFrequencies(text);
         const maxFreq = Math.max(1, ...Object.values(termFreq));
         return Object.entries(termFreq)
@@ -632,7 +657,7 @@ export class ContentAnalyzer {
     try {
       // compromise is English-only: on CJK text it returns whole multi-sentence
       // runs as single "terms". Rank dictionary-segmented content words instead.
-      if (this.isCjkText(text)) {
+      if (this.needsSegmentedTerms(text, options)) {
         const termFreq = this.cjkContentWordFrequencies(text);
         const totalTerms = Object.values(termFreq).reduce((sum, freq) => sum + freq, 0);
         return Object.entries(termFreq)
@@ -1027,7 +1052,13 @@ export class ContentAnalyzer {
       '也是', '就是', '我们', '你们', '他们', '她们', '它们', '这个', '那个',
       '这些', '那些', '一个', '一些', '以及', '或者', '但是', '因为', '所以',
       '如果', '没有', '可以', '已经', '通过', '由于', '对于', '其中', '并且',
-      '而且', '虽然', '什么', '自己', '这里', '那里'
+      '而且', '虽然', '什么', '自己', '这里', '那里',
+      // Japanese particles, auxiliaries and formal nouns the segmenter emits
+      // as tokens ("ます", "です" led kakaku.com keywords in R17, 2026-09-04)
+      'ます', 'です', 'てい', 'って', 'ので', 'から', 'まで', 'など', 'また',
+      'する', 'いる', 'ある', 'こと', 'これ', 'それ', 'ため', 'よう', 'として',
+      'について', 'という', 'られ', 'れる', 'でき', 'ました', 'ません', 'した',
+      'その', 'この', 'あの', 'もの', 'ところ', 'ここ', 'そこ'
     ];
 
     return cjkStopWords.includes(word);

package/src/core/analysis/sentenceUtils.js

@@ -3,10 +3,13 @@
  * domain names, and other common patterns that contain periods.
  */
 
-// CJK / fullwidth sentence terminators. Unlike the ASCII '.' these are
-// unambiguous — no abbreviation, decimal or initial uses them — and CJK text
-// puts no whitespace after them, so they split on a zero-width boundary.
-const CJK_TERMINATORS = '。．！？；';
+// CJK / fullwidth sentence terminators, plus the Devanagari danda (।) and
+// double danda (॥) that end Hindi, Marathi, Nepali and Sanskrit sentences —
+// a Hindi paragraph of five sentences counted as one (R17, 2026-09-04).
+// Unlike the ASCII '.' these are unambiguous — no abbreviation, decimal or
+// initial uses them — and CJK text puts no whitespace after them, so they
+// split on a zero-width boundary.
+const CJK_TERMINATORS = '。．！？；।॥';
 
 // Common abbreviations that should not trigger sentence splits
 const ABBREVIATIONS = new Set([
@@ -33,16 +36,16 @@ export function splitSentences(text) {
 
   // Split by potential sentence boundaries: . ! ? and the CJK terminators
   // But be smart about abbreviations, numbers, and domain-like patterns
-  const tokens = text.split(/(?<=[.!?])\s+|(?<=[。．！？；])\s*/);
+  const tokens = text.split(/(?<=[.!?])\s+|(?<=[。．！？；।॥])\s*/);
 
   for (const token of tokens) {
     const combined = current ? current + ' ' + token : token;
 
     // Check if the current chunk ends with something that looks like a sentence end
-    const endMatch = combined.match(/([.!?。．！？；])\s*$/);
+    const endMatch = combined.match(/([.!?。．！？；।॥])\s*$/);
     if (endMatch) {
       // Check if the period is likely NOT a sentence boundary
-      const beforePeriod = combined.replace(/[.!?。．！？；]\s*$/, '');
+      const beforePeriod = combined.replace(/[.!?。．！？；।॥]\s*$/, '');
       const lastWord = beforePeriod.split(/\s+/).pop() || '';
       const lastWordLower = lastWord.toLowerCase().replace(/[^a-z]/g, '');
 

package/src/core/crawlers/BFSCrawler.js

@@ -11,6 +11,7 @@ import { safeFetch } from '../../utils/ssrfGuard.js';
 import { robotsPreflight, RobotsDisallowedError } from '../../utils/robotsGate.js';
 import { throttleHost } from '../../utils/hostRateLimiter.js';
 import { CRAWLFORGE_USER_AGENT, identityHeaders } from '../../utils/fetchIdentity.js';
+import { pageTitle } from '../../utils/pageTitle.js';
 
 const logger = new Logger('BFSCrawler');
 
@@ -381,7 +382,7 @@ export class BFSCrawler {
     const $ = load(html);
     
     // Extract title
-    const title = $('title').text().trim() || $('h1').first().text().trim() || '';
+    const title = pageTitle($) || $('h1').first().text().trim() || '';
     
     // Extract main content
     $('script, style, noscript').remove();

package/src/core/integrations/PerformanceIntegration.js

@@ -8,6 +8,7 @@ import WorkerPool from '../workers/WorkerPool.js';
 import ConnectionPool from '../connections/ConnectionPool.js';
 import StreamProcessor from '../processing/StreamProcessor.js';
 import { config } from '../../constants/config.js';
+import { pageTitle } from '../../utils/pageTitle.js';
 
 let performanceManagerInstance = null;
 let workerPoolInstance = null;
@@ -279,7 +280,7 @@ async function parseHtmlSync(html, options = {}) {
 
   if (options.extractText !== false) {
     result.text = $('body').text().trim();
-    result.title = $('title').text().trim();
+    result.title = pageTitle($);
   }
 
   if (options.extractLinks) {

package/src/core/processing/ContentProcessor.js

@@ -9,6 +9,7 @@ import * as cheerio from 'cheerio';
 import { z } from 'zod';
 import { ContentQualityAssessor } from '../../utils/contentUtils.js';
 import { recoverDroppedTables } from '../../tools/scrape/_mainContent.js';
+import { pageTitle } from '../../utils/pageTitle.js';
 
 const ContentProcessorSchema = z.object({
   html: z.string(),
@@ -303,7 +304,7 @@ export class ContentProcessor {
   extractMetadata(html) {
     const $ = cheerio.load(html);
     const metadata = {
-      title: $('title').text().trim() || null,
+      title: pageTitle($) || null,
       description: null,
       keywords: null,
       author: null,
@@ -474,7 +475,7 @@ export class ContentProcessor {
 
     return {
       content: mainContent,
-      title: $('title').text().trim() || null,
+      title: pageTitle($) || null,
       headings: this.extractHeadings($),
       length: mainContent ? mainContent.length : 0
     };

package/src/core/workers/worker.js

@@ -9,6 +9,7 @@ import { JSDOM } from 'jsdom';
 import { Readability } from '@mozilla/readability';
 import compromise from 'compromise';
 import { franc } from 'franc';
+import { pageTitle } from '../../utils/pageTitle.js';
 
 // Task handlers
 const taskHandlers = {
@@ -93,7 +94,7 @@ async function handleParseHtml(data) {
   // Extract text content
   if (extractText) {
     result.text = $('body').text().trim();
-    result.title = $('title').text().trim();
+    result.title = pageTitle($);
   }
 
   // Extract links

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -11,6 +11,7 @@ import { load } from 'cheerio';
 // Import existing tool for content extraction
 import ExtractContentTool from '../extract/extractContent.js';
 import { elementText } from '../../utils/elementText.js';
+import { pageTitle } from '../../utils/pageTitle.js';
 
 // Recording / replay helpers
 import {
@@ -625,7 +626,7 @@ export class ScrapeWithActionsTool extends EventEmitter {
         const state = {
           capturePoint: stateData.afterActionIndex + 1,
           url: stateData.url,
-          title: $('title').text().trim(),
+          title: pageTitle($),
           timestamp: stateData.timestamp,
           content: {}
         };

package/src/tools/advanced/batchScrape/worker.js

@@ -12,6 +12,7 @@ import { noteRetryAfter } from '../../../utils/hostRateLimiter.js';
 import { preflightFetch } from '../../../utils/robotsGate.js';
 import { htmlToMarkdown } from '../../../utils/htmlToMarkdown.js';
 import { elementText } from '../../../utils/elementText.js';
+import { pageTitle } from '../../../utils/pageTitle.js';
 
 /**
  * Fetch a URL with AbortController timeout (SSRF-guarded + per-host throttled).
@@ -133,7 +134,7 @@ export function generateFormats($, html, formats) {
   if (formats.includes('markdown')) content.markdown = buildMarkdown($);
   if (formats.includes('json')) {
     content.json = {
-      title: $('title').text().trim(),
+      title: pageTitle($),
       headings: extractHeadings($),
       links: extractLinks($),
       images: extractImages($),
@@ -144,7 +145,7 @@ export function generateFormats($, html, formats) {
 }
 
 function buildMarkdown($) {
-  const title = $('title').text().trim();
+  const title = pageTitle($);
 
   const selectors = ['article', 'main', '.content', '#content', '.post-content', '.entry-content'];
   let $body = null;
@@ -203,7 +204,7 @@ function extractImages($) {
 
 function extractMetadata($) {
   const m = {};
-  m.title = $('title').text().trim();
+  m.title = pageTitle($);
   m.description = $('meta[name="description"]').attr('content') || '';
   m.og = {};
   $('meta[property^="og:"]').each((_, el) => {

package/src/tools/crawl/mapSite.js

@@ -8,6 +8,7 @@ import { ResultRanker } from '../search/ranking/ResultRanker.js';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 import { CRAWLFORGE_USER_AGENT } from '../../utils/fetchIdentity.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
+import { pageTitle } from '../../utils/pageTitle.js';
 
 // Lazy singleton — avoids creating a CacheManager timer per request
 let _ranker = null;
@@ -280,7 +281,7 @@ export class MapSiteTool {
           const $ = load(html);
           
           metadataMap.set(url, {
-            title: $('title').text().trim(),
+            title: pageTitle($),
             description: $('meta[name="description"]').attr('content') || '',
             keywords: $('meta[name="keywords"]').attr('content') || '',
             h1: $('h1').first().text().trim(),

package/src/tools/extract/_fetchAndParse.js

@@ -11,6 +11,7 @@
  */
 
 import { load } from 'cheerio';
+import { readBody } from 'crawlforge-extractors';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 import { config } from '../../constants/config.js';
 import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
@@ -23,8 +24,15 @@ const DEFAULT_TIMEOUT_MS = 15000;
  * the same cap _fetch.js applies to basic tools, so a large/hostile response
  * (multi-hundred-MB file, endpoint streaming zeros) can't be buffered whole
  * into a JS string and handed to cheerio/JSDOM/Turndown downstream.
- * Content-Length is checked up front; actual bytes read are counted as a
- * backstop for servers that omit or lie about it.
+ *
+ * Decoding goes through crawlforge-extractors' readBody, which honours the
+ * Content-Type charset and the <meta charset> sniff. The local reader it
+ * replaces decoded every body as UTF-8, so scrape, extract_structured,
+ * extract_embedded_state, extract_with_llm and agent all returned mojibake
+ * for Shift_JIS pages (kakaku.com, vector.co.jp, 2ch.sc — R17, 2026-09-04)
+ * while batch_scrape and extract_metadata, already on readBody, read them
+ * correctly. Content-Length is checked up front; actual bytes read are
+ * counted as a backstop for servers that omit or lie about it.
  * @param {Response} response
  * @returns {Promise<string>}
  */
@@ -47,30 +55,7 @@ async function readTextWithSizeCap(response) {
     return response.text();
   }
 
-  const reader = response.body.getReader();
-  const chunks = [];
-  let totalBytes = 0;
-
-  while (true) {
-    const { done, value } = await reader.read();
-    if (done) break;
-    totalBytes += value.byteLength;
-    if (totalBytes > maxBodySize) {
-      reader.cancel();
-      throw new Error(
-        `Response body too large: exceeded limit of ${maxBodySize} bytes`
-      );
-    }
-    chunks.push(value);
-  }
-
-  const mergedBytes = new Uint8Array(totalBytes);
-  let offset = 0;
-  for (const chunk of chunks) {
-    mergedBytes.set(chunk, offset);
-    offset += chunk.byteLength;
-  }
-  return new TextDecoder().decode(mergedBytes);
+  return readBody(response, { maxBytes: maxBodySize });
 }
 
 /**

package/src/tools/extract/extractContent.js

@@ -12,6 +12,8 @@ import { isThinMainContent } from '../scrape/_mainContent.js';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
 import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
+import { readBody } from 'crawlforge-extractors';
+import { config as appConfig } from '../../constants/config.js';
 
 const ExtractContentSchema = z.object({
   url: z.string().url(),
@@ -196,7 +198,9 @@ export class ExtractContentTool {
           throw new Error(`HTTP ${response.status}: ${response.statusText}`);
         }
 
-        html = await response.text();
+        // readBody decodes with the body's real charset; response.text() is
+        // UTF-8-only and mangled Shift_JIS pages (R17, 2026-09-04).
+        html = await readBody(response, { maxBytes: appConfig.fetch.maxBodySize });
         pageTitle = this.extractTitleFromHTML(html);
       }
       }

package/src/tools/extract/extractStructured.js

@@ -56,6 +56,8 @@ function mainContentText($, html, url) {
 
 /** Characters of page text the model is shown; about 6k tokens. */
 const SHOWN_TEXT_BUDGET = 24000;
+/** Characters of the whole-page text kept when main content would crowd it out. */
+const PAGE_HEAD_RESERVE = 6000;
 
 /**
  * The text the model reads: main content first, then the whole page when both
@@ -78,11 +80,19 @@ const SHOWN_TEXT_BUDGET = 24000;
  * @param {string} textContent - whole-page text from fetchAndParse
  * @returns {string}
  */
-function shownText($, html, url, textContent) {
+export function shownText($, html, url, textContent) {
   const main = mainContentText($, html, url);
   if (!main) return textContent;
-  if (main.length + textContent.length > SHOWN_TEXT_BUDGET) return main;
-  return `${main}\n\n${textContent}`;
+  if (main.length + textContent.length <= SHOWN_TEXT_BUDGET) return `${main}\n\n${textContent}`;
+  // Over budget: the article still leads, but the head of the whole-page
+  // text keeps its reserved slice. That head is the site chrome — the nav
+  // that reads "Install (6.3.3)" on swift.org while the main content's table
+  // is titled "Previous Releases" and starts at 6.3.2 — and the link-heavy
+  // lists Readability drops, such as the version list on gradle.org/releases,
+  // where main content alone answered nothing at all (R17, 2026-09-04).
+  const head = textContent.slice(0, PAGE_HEAD_RESERVE);
+  const mainCut = main.slice(0, Math.max(0, SHOWN_TEXT_BUDGET - head.length - 2));
+  return `${mainCut}\n\n${head}`;
 }
 
 /**

package/src/tools/scrape/unifiedScrape.js

@@ -16,6 +16,7 @@ import { extractMainContent, isThinMainContent } from './_mainContent.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
 import { stripHiddenFromDom } from '../../utils/hiddenContent.js';
 import { extractBlockText } from '../basic/extractText.js';
+import { pageTitle } from '../../utils/pageTitle.js';
 
 // ── Schema ────────────────────────────────────────────────────────────────────
 
@@ -132,7 +133,7 @@ function extractMetadataFromDom($, pageUrl) {
 
   const title =
     $('meta[property="og:title"]').attr('content') ||
-    $('title').text().trim() ||
+    pageTitle($) ||
     $('h1').first().text().trim() || '';
 
   const ogTags = {};

package/src/utils/challengeDetection.js

@@ -40,6 +40,17 @@ const CHALLENGES = [
     title: /^access denied/i,
     markers: /errors\.edgesuite\.net/i,
     evidence: 'an Akamai access-denied page'
+  },
+  {
+    // Vercel's Attack Challenge Mode answers with HTTP 429, an
+    // x-vercel-mitigated: challenge header and a JavaScript interstitial
+    // titled "Vercel Security Checkpoint" (lesswrong.com, hashicorp.com,
+    // bombas.com, R17 2026-09-04). Chromium solves it and reloads; camoufox
+    // was left on the interstitial, which then read as a successful scrape.
+    vendor: 'vercel',
+    title: /^vercel security checkpoint/i,
+    markers: /vercel\.link\/security-checkpoint|_vercel\/challenge|x-vercel-challenge-token/i,
+    evidence: 'a Vercel Security Checkpoint page'
   }
 ];
 

package/src/utils/pageTitle.js

@@ -0,0 +1,18 @@
+/**
+ * Document title from a cheerio root.
+ *
+ * A bare `$('title')` also matches inline `<svg><title>` elements in the body,
+ * and cheerio's `.text()` concatenates every match: roc-lang.org came back as
+ * "Roc — a fast, friendly, functional languageGitHubYouTubeTwitter..." in the
+ * R17 sweep (2026-09-04), the same defect extract_metadata fixed for MDN on
+ * 2026-08-26. Read the head title first and fall back to the first `<title>`
+ * anywhere for fragments without a head.
+ *
+ * @param {import('cheerio').CheerioAPI} $
+ * @returns {string} trimmed title or ''
+ */
+export function pageTitle($) {
+  return $('head > title').first().text().trim() || $('title').first().text().trim();
+}
+
+export default pageTitle;