crawlforge-mcp-server 5.5.0 → 5.5.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.5.0",
+  "version": "5.5.2",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 29 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -114,7 +114,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.5.0",
+    "crawlforge-extractors": "^1.5.1",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",

package/server.js

@@ -105,7 +105,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.5.0",
+  version: "5.5.2",
   description: "Production-ready MCP server with 29 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",

package/src/core/LocalizationManager.js

@@ -7,6 +7,7 @@
 import { z } from 'zod';
 import { EventEmitter } from 'events';
 import { identityHeaders } from '../utils/fetchIdentity.js';
+import { detectLanguage, toIso6391 } from '../utils/languageDetection.js';
 
 // ISO 3166-1 alpha-2 country codes with associated settings (Expanded to 15+ countries)
 const SUPPORTED_COUNTRIES = {
@@ -1372,35 +1373,26 @@ export class LocalizationManager extends EventEmitter {
    * Analyze text for language detection
    */
   async analyzeTextLanguage(text) {
-    // Simple heuristic-based language detection
-    // In a real implementation, this could use a proper language detection library
-    const patterns = {
-      'en': /\b(the|and|is|in|to|of|a|that|it|with|for|as|was|on|are|you)\b/gi,
-      'es': /\b(el|la|de|que|y|en|un|es|se|no|te|lo|le|da|su|por|son)\b/gi,
-      'fr': /\b(le|de|et|à|un|il|être|et|en|avoir|que|pour|dans|ce|son|une)\b/gi,
-      'de': /\b(der|die|und|in|den|von|zu|das|mit|sich|des|auf|für|ist|im)\b/gi,
-      'it': /\b(il|di|che|e|la|per|in|un|è|da|sono|con|non|si|una|su)\b/gi
-    };
-    
-    let bestMatch = null;
-    let maxMatches = 0;
-    const wordCount = text.split(/\s+/).filter(Boolean).length;
-
-    for (const [lang, pattern] of Object.entries(patterns)) {
-      const matches = (text.match(pattern) || []).length;
-      if (matches > maxMatches) {
-        maxMatches = matches;
-        // Confidence from stop-word DENSITY (matches per word), not the
-        // absolute match count — absolute counts made confidence depend on
-        // sample length (a 500-char English sample scored only ~35%).
-        // Stop-words are ~25-40% of natural text in the matching language,
-        // so density * 250 puts an unambiguous match in the 60-95 range.
-        const density = matches / Math.max(1, wordCount);
-        bestMatch = { language: lang, confidence: Math.min(95, Math.round(density * 250)) };
-      }
-    }
+    // Delegates to the server's single detector (src/utils/languageDetection.js).
+    //
+    // This used to be a stop-word regex stub covering five languages, and it
+    // was wrong in both directions: Japanese matched no pattern and returned
+    // null, and French returned 'es' because the `fr` and `es` lists share
+    // "de", "la", "en", "un" and "le" and whichever matched more tokens won.
+    // analyze_content answered the same strings correctly the whole time,
+    // because it used franc — so the bug was really that two detectors existed
+    // and only one got fixed. Anything language-shaped goes through the shared
+    // module now.
+    //
+    // The shared detector speaks ISO 639-3; this surface reports 639-1 (what
+    // html@lang and Accept-Language use), so codes are converted on the way out.
+    const detected = detectLanguage(text);
+    if (!detected) return null;
 
-    return bestMatch;
+    return {
+      language: toIso6391(detected.code),
+      confidence: Math.round(detected.confidence * 100)
+    };
   }
 
   /**

package/src/core/StealthBrowserManager.js

@@ -60,6 +60,25 @@ const StealthConfigSchema = z.object({
   engine: z.enum(['chromium', 'camoufox']).optional().default('chromium')
 });
 
+/**
+ * True when a page is driven by Chromium, the only engine that speaks CDP.
+ *
+ * Playwright exposes the engine as `browser.browserType().name()`. A context
+ * created over a persistent/connected browser can report a null browser, so an
+ * unknown engine is treated as not-Chromium: skipping an optional emulation is
+ * cheap, and calling CDP on Firefox throws.
+ *
+ * @param {import('playwright').Page} page
+ * @returns {boolean}
+ */
+function isChromium(page) {
+  try {
+    return page.context().browser()?.browserType().name() === 'chromium';
+  } catch {
+    return false;
+  }
+}
+
 export class StealthBrowserManager {
   constructor(options = {}) {
     this.browser = null;
@@ -1822,15 +1841,31 @@ export class StealthBrowserManager {
     // Add request headers
     await page.setExtraHTTPHeaders(fingerprint.headers);
 
-    // Emulate realistic network conditions
-    if (config.level === 'advanced') {
-      const client = await page.context().newCDPSession(page);
-      await client.send('Network.emulateNetworkConditions', {
-        offline: false,
-        downloadThroughput: (1.5 + Math.random() * 2) * 1024 * 1024 / 8, // 1.5-3.5 Mbps
-        uploadThroughput: (0.75 + Math.random() * 1.25) * 1024 * 1024 / 8, // 0.75-2 Mbps  
-        latency: 40 + Math.random() * 60 // 40-100ms
-      });
+    // Emulate realistic network conditions.
+    //
+    // Network.emulateNetworkConditions is a Chrome DevTools Protocol command,
+    // and CDP is Chromium-only — Playwright throws "CDP session is only
+    // available in Chromium" on Firefox and WebKit, and there is no
+    // cross-browser equivalent. Unguarded, this made `engine: "camoufox"` fail
+    // 100% of the time, because Camoufox is Firefox-based: every advanced-level
+    // camoufox scrape died here before it ever reached the page.
+    //
+    // The emulation is cosmetic realism, not a stealth requirement, so on a
+    // non-Chromium engine it is skipped rather than fatal.
+    if (config.level === 'advanced' && isChromium(page)) {
+      try {
+        const client = await page.context().newCDPSession(page);
+        await client.send('Network.emulateNetworkConditions', {
+          offline: false,
+          downloadThroughput: (1.5 + Math.random() * 2) * 1024 * 1024 / 8, // 1.5-3.5 Mbps
+          uploadThroughput: (0.75 + Math.random() * 1.25) * 1024 * 1024 / 8, // 0.75-2 Mbps
+          latency: 40 + Math.random() * 60 // 40-100ms
+        });
+      } catch (error) {
+        // A browser build that reports chromium but refuses CDP must not take
+        // the scrape down with it.
+        console.warn(`Network condition emulation skipped: ${error.message}`);
+      }
     }
 
     // Set up human behavior if enabled

package/src/core/analysis/ContentAnalyzer.js

@@ -3,7 +3,11 @@
  * Uses multiple NLP libraries for comprehensive content analysis
  */
 
-import { franc, francAll } from 'franc';
+import {
+  LANGUAGE_NAMES,
+  cjkScriptCounts as sharedCjkScriptCounts,
+  detectLanguage as sharedDetectLanguage
+} from '../../utils/languageDetection.js';
 import nlp from 'compromise';
 import { z } from 'zod';
 import { splitSentences } from './sentenceUtils.js';
@@ -98,86 +102,6 @@ const AnalysisResult = z.object({
 });
 
 // Language code to name mapping
-const LANGUAGE_NAMES = {
-  'eng': 'English',
-  'spa': 'Spanish',
-  'fra': 'French',
-  'deu': 'German',
-  'ita': 'Italian',
-  'por': 'Portuguese',
-  'rus': 'Russian',
-  'jpn': 'Japanese',
-  'kor': 'Korean',
-  'cmn': 'Chinese',
-  'arb': 'Arabic',
-  'hin': 'Hindi',
-  'nld': 'Dutch',
-  'swe': 'Swedish',
-  'nob': 'Norwegian',
-  'dan': 'Danish',
-  'fin': 'Finnish',
-  'pol': 'Polish',
-  'ces': 'Czech',
-  'hun': 'Hungarian',
-  'tur': 'Turkish',
-  'ell': 'Greek',
-  'heb': 'Hebrew',
-  'tha': 'Thai',
-  'vie': 'Vietnamese',
-  'ind': 'Indonesian',
-  'zlm': 'Malay',
-  'zsm': 'Malay',
-  'tgl': 'Tagalog',
-  'ukr': 'Ukrainian',
-  'bul': 'Bulgarian',
-  'hrv': 'Croatian',
-  'slv': 'Slovenian',
-  'ron': 'Romanian',
-  'lit': 'Lithuanian',
-  'lav': 'Latvian',
-  'est': 'Estonian',
-  'slk': 'Slovak',
-  'cat': 'Catalan',
-  'eus': 'Basque',
-  'glg': 'Galician',
-  'gle': 'Irish',
-  'cym': 'Welsh',
-  'isl': 'Icelandic',
-  'mlt': 'Maltese',
-  'sqi': 'Albanian',
-  'mkd': 'Macedonian',
-  'srp': 'Serbian',
-  'bos': 'Bosnian',
-  'mon': 'Mongolian',
-  'uzb': 'Uzbek',
-  'kaz': 'Kazakh',
-  'aze': 'Azerbaijani',
-  'geo': 'Georgian',
-  'arm': 'Armenian',
-  'fas': 'Persian',
-  'urd': 'Urdu',
-  'ben': 'Bengali',
-  'tam': 'Tamil',
-  'tel': 'Telugu',
-  'kan': 'Kannada',
-  'mal': 'Malayalam',
-  'guj': 'Gujarati',
-  'pan': 'Punjabi',
-  'ori': 'Odia',
-  'mar': 'Marathi',
-  'nep': 'Nepali',
-  'sin': 'Sinhala',
-  'mya': 'Burmese',
-  'khm': 'Khmer',
-  'lao': 'Lao',
-  'amh': 'Amharic',
-  'som': 'Somali',
-  'swa': 'Swahili',
-  'hau': 'Hausa',
-  'yor': 'Yoruba',
-  'ibo': 'Igbo',
-  'afr': 'Afrikaans'
-};
 
 export class ContentAnalyzer {
   constructor() {
@@ -205,17 +129,7 @@ export class ContentAnalyzer {
    * @returns {Object} - { letters, han, kana, hangul, share }
    */
   cjkScriptCounts(text) {
-    const letters = (text.match(/\p{L}/gu) || []).length;
-    const han = (text.match(/\p{Script=Han}/gu) || []).length;
-    const kana = (text.match(/[\p{Script=Hiragana}\p{Script=Katakana}]/gu) || []).length;
-    const hangul = (text.match(/\p{Script=Hangul}/gu) || []).length;
-    return {
-      letters,
-      han,
-      kana,
-      hangul,
-      share: letters > 0 ? (han + kana + hangul) / letters : 0
-    };
+    return sharedCjkScriptCounts(text);
   }
 
   /**
@@ -346,80 +260,12 @@ export class ContentAnalyzer {
    * @returns {Promise<Object>} - Language detection result
    */
   async detectLanguage(text, options = {}) {
-    try {
-      // franc scores the single most common script, so a Chinese, Japanese or
-      // Korean page carrying the usual run of English product names and code
-      // samples is detected as English. Those scripts never appear in
-      // Latin-script prose, so a meaningful share of them settles the question
-      // before trigram scoring gets a say.
-      const { letters, han, kana, hangul, share } = this.cjkScriptCounts(text);
-      if (letters > 0) {
-        if (share >= 0.1) {
-          const code = kana > 0 ? 'jpn' : hangul > han ? 'kor' : 'cmn';
-          return {
-            code,
-            name: LANGUAGE_NAMES[code],
-            confidence: 0.9,
-            alternative: [],
-            detectionMethod: 'script'
-          };
-        }
-      }
-
-      // Use franc for language detection
-      const detected = franc(text, {
-        minLength: 10,
-        whitelist: Object.keys(LANGUAGE_NAMES)
-      });
-
-      if (detected === 'und') {
-        // Fallback: check if text is predominantly ASCII Latin characters (likely English)
-        const latinChars = (text.match(/[a-zA-Z]/g) || []).length;
-        const totalChars = text.replace(/\s/g, '').length;
-        if (totalChars > 0 && latinChars / totalChars > 0.7) {
-          // Check for common English words as a heuristic
-          const lower = text.toLowerCase();
-          const englishMarkers = ['the ', 'is ', 'are ', 'was ', 'and ', 'for ', 'that ', 'with ', 'this ', 'from '];
-          const matchCount = englishMarkers.filter(w => lower.includes(w)).length;
-          if (matchCount >= 2) {
-            return {
-              code: 'eng',
-              name: 'English',
-              confidence: 0.6,
-              alternative: [],
-              detectionMethod: 'heuristic'
-            };
-          }
-        }
-        return null; // Truly undetermined language
-      }
-
-      // Get confidence score based on text length and detection certainty
-      const confidence = Math.min(1, 0.5 + (text.length / 500) * 0.5);
-
-      // Get alternative languages using franc.all
-      const alternatives = francAll(text, {
-        minLength: 10,
-        whitelist: Object.keys(LANGUAGE_NAMES)
-      })
-      .slice(1, 4) // Top 3 alternatives
-      .map(([code, score]) => ({
-        code,
-        name: LANGUAGE_NAMES[code] || code,
-        confidence: Math.round(score * 100) / 100
-      }));
-
-      return {
-        code: detected,
-        name: LANGUAGE_NAMES[detected] || detected,
-        confidence: Math.round(confidence * 100) / 100,
-        alternative: alternatives
-      };
-
-    } catch (error) {
-      console.warn('Language detection failed:', error.message);
-      return null;
-    }
+    // The detector itself lives in src/utils/languageDetection.js so that this
+    // class and LocalizationManager cannot drift apart again — they did, and
+    // the localization surface spent that time reporting Japanese as null and
+    // French as Spanish. Behaviour here is unchanged: same franc call, same CJK
+    // script pre-check, same result shape.
+    return sharedDetectLanguage(text, options);
   }
 
   /**

package/src/tools/extract/extractStructured.js

@@ -190,33 +190,76 @@ export class ExtractStructuredTool {
       // the Apple MacBook Air page Readability keeps the FAQ block and every
       // price is left behind in an embedded JSON blob, so checking against what
       // the model was shown would null every correct price.
-      let provenance = { enabled: false };
-      if (extractionResult && extractionMethod === 'llm' && verify_numbers) {
-        const checked = verifyNumericProvenance(extractionResult.data || {}, `${html}\n${textContent}`);
+      const fullSource = `${html}\n${textContent}`;
+
+      /** Run the guard over one extraction and describe what it did. */
+      const applyGuard = (result) => {
+        const checked = verifyNumericProvenance(result.data || {}, fullSource);
         // The model's own `valid` flag described the data before the guard ran.
         // A required field the guard nulled is not filled in any more, so that
         // flag cannot stand or the response reports a fabrication as valid.
         const nulledRequired = checked.unverified
           .map((entry) => entry.path)
           .filter((path) => (schema.required || []).includes(path));
-        extractionResult = {
-          ...extractionResult,
-          data: checked.data,
-          ...(nulledRequired.length > 0 ? {
-            valid: false,
-            validationErrors: [
-              ...(extractionResult.validationErrors || []),
-              ...nulledRequired.map((field) => `Field "${field}" was not found in the page source`)
-            ]
-          } : {})
+        return {
+          checked,
+          nulledRequired,
+          result: {
+            ...result,
+            data: checked.data,
+            ...(nulledRequired.length > 0 ? {
+              valid: false,
+              validationErrors: [
+                ...(result.validationErrors || []),
+                ...nulledRequired.map((field) => `Field "${field}" was not found in the page source`)
+              ]
+            } : {})
+          }
         };
+      };
+
+      let provenance = { enabled: false };
+      if (extractionResult && extractionMethod === 'llm' && verify_numbers) {
+        let guarded = applyGuard(extractionResult);
+
+        // Step 3c: the model is shown main content, but the guard checks the
+        // FULL source — so a value can be nulled for being absent from what the
+        // model saw while sitting in the page all along. sqlite.org is the case
+        // that found this: Readability drops the "Version 3.53.4" line, the
+        // model was handed no version at all and answered "3.34.0" from memory,
+        // and the guard correctly nulled it — leaving a caller with nothing on a
+        // page that plainly states the answer.
+        //
+        // So when a REQUIRED field is nulled, retry once on the full page text,
+        // which is exactly what the guard will accept. Only a strictly better
+        // result is kept, so a retry can never make the answer worse.
+        const shownText = mainContentText($, html, url) || textContent;
+        if (guarded.nulledRequired.length > 0 && shownText !== textContent && textContent) {
+          try {
+            const llm = this._ensureLLMManager(llmConfig || {});
+            const retry = await llm.extractStructured(textContent, schema, {
+              prompt: prompt || '',
+              maxContentLength: 6000
+            });
+            if (retry?.method === 'llm') {
+              const retryGuarded = applyGuard(retry);
+              if (retryGuarded.nulledRequired.length < guarded.nulledRequired.length) {
+                guarded = retryGuarded;
+              }
+            }
+          } catch {
+            // The first answer already stands; a failed retry changes nothing.
+          }
+        }
+
+        extractionResult = guarded.result;
         provenance = {
           enabled: true,
-          verified: checked.verified,
-          nulled: checked.nulled,
-          unverified: checked.unverified
+          verified: guarded.checked.verified,
+          nulled: guarded.checked.nulled,
+          unverified: guarded.checked.unverified
         };
-        if (checked.skipped) provenance.skipped = checked.skipped;
+        if (guarded.checked.skipped) provenance.skipped = guarded.checked.skipped;
       }
 
       // Step 4: CSS selector fallback if LLM unavailable or failed

package/src/tools/scrape/_mainContent.js

@@ -1,14 +1,39 @@
 /**
  * _mainContent.js — Readability main-content extraction, plus recovery of the
- * data tables Readability drops.
+ * content Readability drops.
  *
- * Readability keeps one article candidate and discards everything outside it.
- * On a table-led page that silently loses the payload: Wikipedia's *List of
- * S&P 500 companies* came back with zero table rows at scrape's default
- * onlyMainContent:true, and 505 pipe-table lines with it off. No Readability
- * option recovers them (charThreshold:100 and nbTopCandidates:20 were both
- * probed) — the tables are simply not in the candidate — so they are
- * re-attached afterwards.
+ * Two independent losses are repaired here.
+ *
+ * 1. Readability keeps one article candidate and discards everything outside
+ *    it. On a table-led page that silently loses the payload: Wikipedia's
+ *    *List of S&P 500 companies* came back with zero table rows at scrape's
+ *    default onlyMainContent:true, and 505 pipe-table lines with it off. No
+ *    Readability option recovers them (charThreshold:100 and nbTopCandidates:20
+ *    were both probed) — the tables are simply not in the candidate — so they
+ *    are re-attached afterwards.
+ *
+ * 2. Readability's class regexes are unanchored, so they fire on words their
+ *    author never meant. Two collisions between them deleted every code
+ *    example on next-intl.dev/docs/routing/setup:
+ *
+ *      - `unlikelyCandidates` contains `extra`, which matches inside `nextra-*`
+ *        — the class prefix on every element Nextra emits, and Nextra builds a
+ *        large share of the JS ecosystem's documentation. None of those classes
+ *        match `okMaybeItsACandidate`, so the elements are deleted before
+ *        scoring runs: all 96 inline `<code>` spans and the "setRequestLocale
+ *        is legacy" callout went with them.
+ *      - `negative` contains `hidden`, which matches Tailwind's
+ *        `overflow-hidden`. That is only -25 class weight, but it is enough for
+ *        `_cleanConditionally` to drop the wrapper around all 12 `<pre>` blocks.
+ *
+ *    Both losses were silent, and the second is the dangerous one: the prose
+ *    survived, so the page read as complete while every code example and the
+ *    warning callout were gone, and sentences came back with holes where their
+ *    inline code had been ("In order to use unique pathnames …, can be used to
+ *    handle"). `sanitizeAccidentalClasses` strips just those class tokens
+ *    before Readability sees the document, restoring 11 of the 12 blocks — the
+ *    12th sits inside a collapsed `<details>`, which Readability drops whole
+ *    and which this deliberately does not try to rescue.
  *
  * Shared by unifiedScrape (wants HTML) and extractStructured (wants text).
  */
@@ -20,10 +45,108 @@ import { Readability } from '@mozilla/readability';
 // Readability already kept it. Long enough to be unique to that table.
 const SIGNATURE_LENGTH = 120;
 
+// Copied verbatim from @mozilla/readability's REGEXPS. Two of them delete
+// content, and both are unanchored: `unlikelyCandidates` removes an element
+// outright, `negative` docks 25 from its class weight, which is enough for
+// `_cleanConditionally` to drop it. The unit test asserts the real library
+// still behaves this way, so a Readability upgrade that fixes it upstream
+// fails loudly here rather than leaving a no-op pass behind.
+const UNLIKELY_SOURCE =
+  '-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote';
+const NEGATIVE_SOURCE =
+  '-ad-|hidden|^hid$| hid$| hid |^hidden|banner|combx|comment|com-|contact|footer|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|widget';
+const REMOVAL_SOURCES = [UNLIKELY_SOURCE, NEGATIVE_SOURCE];
+// Readability's `positive` regex. A token matching it carries a signal worth
+// keeping, so it is never stripped even if it also matches mid-word.
+const POSITIVE_SIGNAL =
+  /article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story/i;
+
+// Utility classes that collide with the `negative` regex on a whole word while
+// meaning something else entirely. Tailwind's `overflow-hidden` clips overflow;
+// it does not mark the element hidden. The mid-word rule below cannot catch
+// these — `hidden` really is a whole segment of `overflow-hidden` — and the
+// cost is real: -25 class weight is enough for `_cleanConditionally` to delete
+// the element, which is what removed every code block on next-intl.dev after
+// the `nextra-` collision had already been handled. Listed explicitly rather
+// than inferred, because `page-footer` and `main-sidebar` are the same shape
+// and *are* genuine.
+const UTILITY_FALSE_POSITIVES = new Set([
+  'overflow-hidden',
+  'overflow-x-hidden',
+  'overflow-y-hidden',
+  'overflow-scroll',
+  'overflow-x-scroll',
+  'overflow-y-scroll',
+  'scroll-smooth',
+  'scroll-auto'
+]);
+
 function normalizeWhitespace(text) {
   return text.replace(/\s+/g, ' ').trim();
 }
 
+/**
+ * True when `token` only ever trips Readability's removal regexes inside a
+ * longer word — `extra` within `nextra`, `hidden` within Tailwind's
+ * `overflow-hidden`.
+ *
+ * Only a *preceding* letter or digit counts as mid-word. A trailing one does
+ * not, because that is how English suffixes attach: rescuing `comments`,
+ * `banners` or `footers` would defeat the regexes on exactly the clutter they
+ * were written to catch, whereas a letter run in front of the match means a
+ * different word entirely.
+ *
+ * @param {string} token - one class name
+ * @returns {boolean}
+ */
+function isAccidentalToken(token) {
+  if (UTILITY_FALSE_POSITIVES.has(token.toLowerCase())) return true;
+  if (POSITIVE_SIGNAL.test(token)) return false;
+  let sawMatch = false;
+  for (const source of REMOVAL_SOURCES) {
+    const scan = new RegExp(source, 'gi');
+    let match;
+    while ((match = scan.exec(token)) !== null) {
+      if (match[0].length === 0) {
+        scan.lastIndex += 1;
+        continue;
+      }
+      sawMatch = true;
+      const before = token[match.index - 1];
+      if (before === undefined || !/[a-z0-9]/i.test(before)) return false;
+    }
+  }
+  return sawMatch;
+}
+
+/**
+ * Strip the class tokens that would make Readability discard an element over a
+ * word the class does not actually contain.
+ *
+ * Only whole tokens that are accidental on every match are removed, so a class
+ * carrying a real signal survives: an element still genuinely marked `sidebar`
+ * or `hidden` is still dropped, as it should be. Ids are left alone — they
+ * anchor in-page links, and the class attribute is what these frameworks put
+ * their names on.
+ *
+ * @param {Document} document - mutated in place
+ * @returns {number} how many elements were sanitized
+ */
+export function sanitizeAccidentalClasses(document) {
+  let sanitized = 0;
+  for (const el of Array.from(document.querySelectorAll('[class]'))) {
+    const className = el.getAttribute('class') || '';
+    const tokens = className.split(/\s+/).filter(Boolean);
+    const kept = tokens.filter((token) => !isAccidentalToken(token));
+    if (kept.length === tokens.length) continue;
+
+    if (kept.length > 0) el.setAttribute('class', kept.join(' '));
+    else el.removeAttribute('class');
+    sanitized += 1;
+  }
+  return sanitized;
+}
+
 /**
  * Readability's own data-table test (`_markDataTables`): 10+ rows or more than
  * 4 columns means the table carries data rather than layout, counting
@@ -61,6 +184,7 @@ export function extractMainContent(html, url) {
   let article;
   try {
     const dom = new JSDOM(html, { url });
+    sanitizeAccidentalClasses(dom.window.document);
     article = new Readability(dom.window.document).parse();
   } catch {
     return { html: null, title: '', tablesRecovered: 0 };

package/src/utils/languageDetection.js

@@ -0,0 +1,249 @@
+/**
+ * languageDetection -- the server's single language detector.
+ *
+ * There used to be two. `ContentAnalyzer` detected with franc plus a CJK script
+ * pre-check; `LocalizationManager.analyzeTextLanguage` was a stub whose own
+ * comment said "In a real implementation, this could use a proper language
+ * detection library" and which matched stop-word regexes for exactly five
+ * languages. It reported Japanese as `null` (no pattern covers the script) and
+ * French as `es` (both the `fr` and `es` lists match "de", "la", "en", "un",
+ * "le", so whichever happened to match more tokens won), while analyze_content
+ * answered the same three strings correctly.
+ *
+ * That split is the actual bug: the 2026-08-26 franc + CJK fix was applied to
+ * ContentAnalyzer and never reached the localization path, so the same class of
+ * failure shipped twice. Both callers now come through this module, so a fix
+ * here reaches every surface and there is no second detector to drift.
+ *
+ * Codes are ISO 639-3 (what franc emits). `toIso6391()` converts for callers
+ * that speak 639-1, which is what Accept-Language and html@lang use.
+ */
+
+import { franc, francAll } from 'franc';
+
+/** Languages the detector will consider, ISO 639-3 -> English name. */
+export const LANGUAGE_NAMES = {
+  'eng': 'English',
+  'spa': 'Spanish',
+  'fra': 'French',
+  'deu': 'German',
+  'ita': 'Italian',
+  'por': 'Portuguese',
+  'rus': 'Russian',
+  'jpn': 'Japanese',
+  'kor': 'Korean',
+  'cmn': 'Chinese',
+  'arb': 'Arabic',
+  'hin': 'Hindi',
+  'nld': 'Dutch',
+  'swe': 'Swedish',
+  'nob': 'Norwegian',
+  'dan': 'Danish',
+  'fin': 'Finnish',
+  'pol': 'Polish',
+  'ces': 'Czech',
+  'hun': 'Hungarian',
+  'tur': 'Turkish',
+  'ell': 'Greek',
+  'heb': 'Hebrew',
+  'tha': 'Thai',
+  'vie': 'Vietnamese',
+  'ind': 'Indonesian',
+  'zlm': 'Malay',
+  'zsm': 'Malay',
+  'tgl': 'Tagalog',
+  'ukr': 'Ukrainian',
+  'bul': 'Bulgarian',
+  'hrv': 'Croatian',
+  'slv': 'Slovenian',
+  'ron': 'Romanian',
+  'lit': 'Lithuanian',
+  'lav': 'Latvian',
+  'est': 'Estonian',
+  'slk': 'Slovak',
+  'cat': 'Catalan',
+  'eus': 'Basque',
+  'glg': 'Galician',
+  'gle': 'Irish',
+  'cym': 'Welsh',
+  'isl': 'Icelandic',
+  'mlt': 'Maltese',
+  'sqi': 'Albanian',
+  'mkd': 'Macedonian',
+  'srp': 'Serbian',
+  'bos': 'Bosnian',
+  'mon': 'Mongolian',
+  'uzb': 'Uzbek',
+  'kaz': 'Kazakh',
+  'aze': 'Azerbaijani',
+  'geo': 'Georgian',
+  'arm': 'Armenian',
+  'fas': 'Persian',
+  'urd': 'Urdu',
+  'ben': 'Bengali',
+  'tam': 'Tamil',
+  'tel': 'Telugu',
+  'kan': 'Kannada',
+  'mal': 'Malayalam',
+  'guj': 'Gujarati',
+  'pan': 'Punjabi',
+  'ori': 'Odia',
+  'mar': 'Marathi',
+  'nep': 'Nepali',
+  'sin': 'Sinhala',
+  'mya': 'Burmese',
+  'khm': 'Khmer',
+  'lao': 'Lao',
+  'amh': 'Amharic',
+  'som': 'Somali',
+  'swa': 'Swahili',
+  'hau': 'Hausa',
+  'yor': 'Yoruba',
+  'ibo': 'Igbo',
+  'afr': 'Afrikaans'
+};
+
+/**
+ * ISO 639-3 -> ISO 639-1. Only codes with a 639-1 equivalent appear; the rest
+ * have none, and callers keep the 639-3 code rather than invent one.
+ */
+export const ISO_639_3_TO_1 = {
+  eng: 'en', spa: 'es', fra: 'fr', deu: 'de', ita: 'it', por: 'pt', rus: 'ru',
+  jpn: 'ja', kor: 'ko', cmn: 'zh', arb: 'ar', hin: 'hi', nld: 'nl', swe: 'sv',
+  nob: 'nb', dan: 'da', fin: 'fi', pol: 'pl', ces: 'cs', hun: 'hu', tur: 'tr',
+  ell: 'el', heb: 'he', tha: 'th', vie: 'vi', ind: 'id', zlm: 'ms', zsm: 'ms',
+  tgl: 'tl', ukr: 'uk', bul: 'bg', hrv: 'hr', slv: 'sl', ron: 'ro', lit: 'lt',
+  lav: 'lv', est: 'et', slk: 'sk', cat: 'ca', eus: 'eu', glg: 'gl', gle: 'ga',
+  cym: 'cy', isl: 'is', mlt: 'mt', sqi: 'sq', mkd: 'mk', srp: 'sr', bos: 'bs',
+  mon: 'mn', uzb: 'uz', kaz: 'kk', aze: 'az', geo: 'ka', arm: 'hy', fas: 'fa',
+  urd: 'ur', ben: 'bn', tam: 'ta', tel: 'te', kan: 'kn', mal: 'ml', guj: 'gu',
+  pan: 'pa', ori: 'or', mar: 'mr', nep: 'ne', sin: 'si', mya: 'my', khm: 'km',
+  lao: 'lo', amh: 'am', som: 'so', swa: 'sw', hau: 'ha', yor: 'yo', ibo: 'ig',
+  afr: 'af'
+};
+
+/** The CJK share at or above which script alone settles the language. */
+const CJK_SCRIPT_THRESHOLD = 0.1;
+
+/**
+ * Count CJK-script letters and their share of all letters. This is the script
+ * signal detection uses; the ContentAnalyzer tokenizer reuses it to decide when
+ * whitespace splitting cannot work.
+ *
+ * @param {string} text
+ * @returns {{letters: number, han: number, kana: number, hangul: number, share: number}}
+ */
+export function cjkScriptCounts(text) {
+  const letters = (text.match(/\p{L}/gu) || []).length;
+  const han = (text.match(/\p{Script=Han}/gu) || []).length;
+  const kana = (text.match(/[\p{Script=Hiragana}\p{Script=Katakana}]/gu) || []).length;
+  const hangul = (text.match(/\p{Script=Hangul}/gu) || []).length;
+  return {
+    letters,
+    han,
+    kana,
+    hangul,
+    share: letters > 0 ? (han + kana + hangul) / letters : 0
+  };
+}
+
+/**
+ * True when a meaningful share of the text is in a CJK script, i.e. words are
+ * not whitespace-delimited and must be segmented by dictionary.
+ *
+ * @param {string} text
+ * @returns {boolean}
+ */
+export function isCjkText(text) {
+  return cjkScriptCounts(text).share >= CJK_SCRIPT_THRESHOLD;
+}
+
+/**
+ * Detect the language of a piece of text.
+ *
+ * @param {string} text
+ * @param {{minLength?: number}} [options]
+ * @returns {{code: string, name: string, confidence: number,
+ *            alternative: Array<{code: string, name: string, confidence: number}>,
+ *            detectionMethod?: string} | null} null when undetermined
+ */
+export function detectLanguage(text, options = {}) {
+  if (typeof text !== 'string' || text.trim() === '') return null;
+  const minLength = options.minLength ?? 10;
+
+  try {
+    // franc scores the single most common script, so a Chinese, Japanese or
+    // Korean page carrying the usual run of English product names and code
+    // samples is detected as English. Those scripts never appear in
+    // Latin-script prose, so a meaningful share of them settles the question
+    // before trigram scoring gets a say.
+    const { letters, han, kana, hangul, share } = cjkScriptCounts(text);
+    if (letters > 0 && share >= CJK_SCRIPT_THRESHOLD) {
+      const code = kana > 0 ? 'jpn' : hangul > han ? 'kor' : 'cmn';
+      return {
+        code,
+        name: LANGUAGE_NAMES[code],
+        confidence: 0.9,
+        alternative: [],
+        detectionMethod: 'script'
+      };
+    }
+
+    const detected = franc(text, { minLength, whitelist: Object.keys(LANGUAGE_NAMES) });
+
+    if (detected === 'und') {
+      // Latin-script text franc could not place. Two English markers is a weak
+      // signal, but it beats returning nothing for short English strings.
+      const latinChars = (text.match(/[a-zA-Z]/g) || []).length;
+      const totalChars = text.replace(/\s/g, '').length;
+      if (totalChars > 0 && latinChars / totalChars > 0.7) {
+        const lower = text.toLowerCase();
+        const englishMarkers = ['the ', 'is ', 'are ', 'was ', 'and ', 'for ', 'that ', 'with ', 'this ', 'from '];
+        if (englishMarkers.filter(w => lower.includes(w)).length >= 2) {
+          return {
+            code: 'eng',
+            name: 'English',
+            confidence: 0.6,
+            alternative: [],
+            detectionMethod: 'heuristic'
+          };
+        }
+      }
+      return null; // Truly undetermined language
+    }
+
+    const confidence = Math.min(1, 0.5 + (text.length / 500) * 0.5);
+
+    const alternative = francAll(text, { minLength, whitelist: Object.keys(LANGUAGE_NAMES) })
+      .slice(1, 4)
+      .map(([code, score]) => ({
+        code,
+        name: LANGUAGE_NAMES[code] || code,
+        confidence: Math.round(score * 100) / 100
+      }));
+
+    return {
+      code: detected,
+      name: LANGUAGE_NAMES[detected] || detected,
+      confidence: Math.round(confidence * 100) / 100,
+      alternative
+    };
+  } catch (error) {
+    console.warn('Language detection failed:', error.message);
+    return null;
+  }
+}
+
+/**
+ * The ISO 639-1 code for a 639-3 code, or the input unchanged when the language
+ * has no 639-1 equivalent.
+ *
+ * @param {string|null|undefined} code
+ * @returns {string|null}
+ */
+export function toIso6391(code) {
+  if (!code) return null;
+  return ISO_639_3_TO_1[code] || code;
+}
+
+export default detectLanguage;

package/src/utils/provenance.js

@@ -1,7 +1,7 @@
 /**
- * provenance -- numeric provenance guard for LLM-extracted data.
+ * provenance -- provenance guard for LLM-extracted data.
  *
- * A model handed page text that does not contain the number it was asked for
+ * A model handed page text that does not contain the value it was asked for
  * does not say so: it writes a plausible one. On
  * https://www.apple.com/shop/buy-mac/macbook-air every MacBook Air price lives
  * only inside the page's embedded PRODUCT_SELECTION_BOOTSTRAP JSON — the
@@ -25,6 +25,22 @@
  *    "1.299,00", "1 299", "1299.00" and in a value split across markup. Every
  *    ambiguous reading of a source number is admitted, because an extra reading
  *    can only make the guard more permissive, never null a real value.
+ *
+ * Two kinds of value are checked, both of which have one correct spelling:
+ *
+ * - numbers, matched on normalised numeric readings (see `valueReadings`);
+ * - digit-bearing literals — versions, dates, ISBNs, SKUs, model numbers —
+ *   matched by literal substring (see `literalReadings`).
+ *
+ * The literal class was added after `extract_structured` returned SQLite
+ * "3.34.0" on sqlite.org for three runs running. Readability drops the
+ * "Version 3.53.4" line, so the model saw no version and answered from memory
+ * with a real-but-wrong release; the guard reported `verified: 0` because a
+ * three-segment dotted string is not a number and nothing else looked at it.
+ *
+ * Prose is never checked. A value containing whitespace, or no digit at all,
+ * is left alone: a model may legitimately re-word a description, and nulling
+ * good text is the one failure this guard must not introduce.
  */
 
 /** Spaces (incl. NBSP / narrow NBSP) and the Swiss apostrophe group digits. */
@@ -46,6 +62,31 @@ const CURRENCY_SYMBOLS = /\p{Sc}/gu;
  */
 const NUMERIC_STRING = /^[+-]?\d+(?:[.,]\d{3})*(?:[.,]\d+)?$/;
 
+/**
+ * A digit-bearing literal that is not a plain number: a version ("3.53.4"), a
+ * date ("2026-07-24"), an ISBN, a SKU, a model number ("A2338"), an id.
+ *
+ * These are the other thing a model fabricates, and NUMERIC_STRING cannot see
+ * them — "3.34.0" has three dotted segments, so it fails that pattern, and
+ * `valueReadings` returned null, and the walker skipped the field. On
+ * sqlite.org that is exactly what happened: Readability drops the "Version
+ * 3.53.4" line, so the model was shown no version at all and answered "3.34.0"
+ * from memory — a real SQLite release, absent from the page, reported as
+ * verified because the guard never looked at it.
+ *
+ * The shape is deliberately tight, and the rule is "no whitespace": a value
+ * with a space in it is prose, which a model may legitimately re-word, and
+ * comparing it literally would null good extractions. What is left is only
+ * tokens that have one correct spelling, so a literal lookup is a fair test.
+ */
+const LITERAL_SEPARATED = /^[A-Za-z0-9]+(?:[.\-_/:+][A-Za-z0-9]+)+$/;
+
+/** A mixed alphanumeric run with no separator: "A2338", "RTX4090". */
+const LITERAL_ALNUM = /^(?=.*[A-Za-z])(?=.*\d)[A-Za-z0-9]+$/;
+
+/** A leading "v" is a way of writing a version, not part of it. */
+const VERSION_PREFIX = /^v(?=\d)/i;
+
 /** Chained markup between digits (`<span>1</span><span>299</span>`) is welded. */
 const MARKUP_BETWEEN_DIGITS = /(\d)(?:(?:\s*<[^>]{0,120}>)+\s*)([\d.,])/g;
 const MAX_WELD_PASSES = 3;
@@ -96,7 +137,7 @@ function readings(token) {
  * @param {string} source - raw html and/or page text
  * @returns {Set<string>}
  */
-function numbersInSource(source) {
+function sourceVariants(source) {
   const variants = [source];
   if (source.includes('<')) {
     let welded = source;
@@ -107,9 +148,12 @@ function numbersInSource(source) {
     }
     if (welded !== source) variants.push(welded);
   }
+  return variants;
+}
 
+function numbersInSource(source) {
   const found = new Set();
-  for (const variant of variants) {
+  for (const variant of sourceVariants(source)) {
     for (const [token] of variant.matchAll(GROUPED_TOKEN)) {
       for (const reading of readings(token)) found.add(reading);
     }
@@ -120,6 +164,19 @@ function numbersInSource(source) {
   return found;
 }
 
+/**
+ * Lowercased source variants for literal lookup. A literal is checked by
+ * substring, so "3.53.4" is found inside "Version 3.53.4 (2026-07-24)" and
+ * inside "v3.53.4", and the welded variant covers a token the page splits
+ * across markup.
+ *
+ * @param {string} source
+ * @returns {string[]}
+ */
+function literalHaystacks(source) {
+  return sourceVariants(source).map((variant) => variant.toLowerCase());
+}
+
 /**
  * The numeric readings of an extracted value, or null when the value is not a
  * numeric field at all.
@@ -143,6 +200,34 @@ function valueReadings(value) {
   return readings(stripped.replace(/^\+/, ''));
 }
 
+/**
+ * The literal spellings of a digit-bearing identifier, or null when the value
+ * is not one. Checked only after `valueReadings` declines, so a plain number
+ * never reaches here.
+ *
+ * Two shapes qualify, both of which have exactly one correct spelling:
+ * separated ("3.53.4", "2026-07-24", "978-0-596-51774-8") and mixed
+ * alphanumeric ("A2338"). Everything else — anything with whitespace, and any
+ * string with no digit in it — is prose and is left alone, because a model may
+ * legitimately re-word prose and a literal comparison would null good data.
+ *
+ * @param {*} value
+ * @returns {string[]|null} lowercased spellings to look for
+ */
+function literalReadings(value) {
+  if (typeof value !== 'string') return null;
+  const token = value.trim();
+  if (!token || /\s/.test(token)) return null;
+  if (!/\d/.test(token)) return null;
+  if (!LITERAL_SEPARATED.test(token) && !LITERAL_ALNUM.test(token)) return null;
+
+  const lower = token.toLowerCase();
+  const spellings = new Set([lower]);
+  // "v3.53.4" on a page that writes "3.53.4", and the reverse.
+  spellings.add(lower.replace(VERSION_PREFIX, ''));
+  return [...spellings];
+}
+
 /**
  * Replace every numeric value that is not present in the source with null.
  *
@@ -168,6 +253,13 @@ export function verifyNumericProvenance(data, source) {
   const found = numbersInSource(source);
   const unverified = [];
   let verified = 0;
+  // Built lazily: most extractions are all numbers and never need it, and
+  // lowercasing a multi-megabyte page is not free.
+  let haystacks = null;
+  const inSource = (spellings) => {
+    if (haystacks === null) haystacks = literalHaystacks(source);
+    return spellings.some((s) => haystacks.some((h) => h.includes(s)));
+  };
 
   const walk = (node, path) => {
     if (Array.isArray(node)) {
@@ -182,8 +274,20 @@ export function verifyNumericProvenance(data, source) {
     }
 
     const candidates = valueReadings(node);
-    if (candidates === null) return node;
-    if (candidates.some((c) => found.has(c))) {
+    if (candidates !== null) {
+      if (candidates.some((c) => found.has(c))) {
+        verified++;
+        return node;
+      }
+      unverified.push({ path: path || '(root)', value: node, reason: 'not_found_in_source' });
+      return null;
+    }
+
+    // Not a number, but possibly a version/date/SKU-shaped literal, which is
+    // the other thing a model invents when the page does not say.
+    const literals = literalReadings(node);
+    if (literals === null) return node;
+    if (inSource(literals)) {
       verified++;
       return node;
     }