crawlforge-mcp-server 5.2.4 → 5.2.6

package/CLAUDE.md

@@ -62,7 +62,7 @@ These guidelines are working if: fewer unnecessary changes in diffs, fewer rewri
 
 CrawlForge MCP Server - A professional MCP (Model Context Protocol) server providing 28 web scraping, crawling, and content processing tools (5 inline + 23 advanced).
 
-**Current Version:** 5.2.4
+**Current Version:** 5.2.6
 
 ## Development Commands
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.2.4",
+  "version": "5.2.6",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 28 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -113,7 +113,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.2.0",
+    "crawlforge-extractors": "^1.2.2",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",

package/server.js

@@ -100,7 +100,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.2.4",
+  version: "5.2.6",
   description: "Production-ready MCP server with 28 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -191,6 +191,7 @@ const serpRankTool = new SerpRankTool();
 // provider search_web uses, then reads those posts from the archive.
 const redditSearchTool = new RedditSearchTool({
   searchApiKey: searchWebToolConfig.apiKey,
+  searchApiBaseUrl: searchWebToolConfig.apiBaseUrl,
 });
 const crawlDeepTool = new CrawlDeepTool(getToolConfig('crawl_deep'));
 const mapSiteTool = new MapSiteTool(getToolConfig('map_site'));

package/src/core/AgentOrchestrator.js

@@ -44,6 +44,21 @@ function truncate(text, maxChars = 8000) {
   return text.slice(0, maxChars) + '\n[...truncated]';
 }
 
+/**
+ * Current-state task gate: prompts about the live "now" ("right now",
+ * "currently", "today", "latest", "#1 … now") go stale through search alone —
+ * dated articles rank for the task's words while the live page does not
+ * (live repro 2026-08-26: "#1 story on Hacker News right now" answered from
+ * a January thread; news.ycombinator.com was never fetched). Detected
+ * deterministically here (never LLM-trusted) so PLAN steers search toward
+ * the live entity page and SHAPE answers from it, not from dated results.
+ */
+const CURRENT_STATE_RE = /\b(right now|currently|today|tonight|at the moment|as of (now|today)|latest|this (week|month|morning))\b|#\d+[^.?!]*\bnow\b/i;
+
+export function isCurrentStateTask(prompt) {
+  return CURRENT_STATE_RE.test(prompt || '');
+}
+
 // ── Orchestrator ──────────────────────────────────────────────────────────────
 
 export class AgentOrchestrator {
@@ -172,10 +187,17 @@ export class AgentOrchestrator {
     }
 
     // ── PLAN ──────────────────────────────────────────────────────────────────
+    const currentState = isCurrentStateTask(prompt);
     let searchQueries = [prompt]; // fallback: use raw prompt as query
     try {
       const planPrompt =
         `Decompose this research task into 1-3 concise web search queries. ` +
+        // Current-state tasks: a query made of the task's words surfaces dated
+        // articles ABOUT the topic; the bare entity name surfaces the live
+        // official page as the top result, which GATHER then prioritizes.
+        (currentState
+          ? `The task asks about a CURRENT live state: the FIRST query must be ONLY the name of the site or thing whose current state is asked, nothing else. `
+          : '') +
         `Output ONLY the queries, one per line, no preamble or numbering:\n\n${prompt}`;
       const { text } = await this._getSamplingClient().complete(planPrompt, { maxTokens: 200 });
       const lines = text.split('\n')
@@ -231,6 +253,38 @@ export class AgentOrchestrator {
       } catch { /* search tool init failed */ }
     }
 
+    // Current-state tasks: search results are LEADS, not answers — the answer
+    // must come from the authoritative live page. The raw top result is NOT a
+    // safe proxy (live retest 2026-08-26: CSE ranks thehackernews.com above
+    // news.ycombinator.com for "Hacker News"), so vote by domain across all
+    // results — PLAN's bare entity query makes the official site dominate —
+    // and put that domain's root (its live front page) first in the fetch
+    // queue and first at SHAPE time so synthesis answers from it, never from
+    // a dated article. A wrong root self-heals: it fails the relevance gate
+    // and never enters evidence.
+    if (currentState && searchResults.length > 0) {
+      const originCounts = new Map();
+      for (const s of searchResults) {
+        try {
+          const origin = new URL(s.url).origin;
+          originCounts.set(origin, (originCounts.get(origin) || 0) + 1);
+        } catch { /* unparsable result url */ }
+      }
+      let bestOrigin = null;
+      let bestCount = 0;
+      for (const [origin, count] of originCounts) {
+        // Strict > keeps the earliest-seen (top-ranked) origin on ties.
+        if (count > bestCount) { bestOrigin = origin; bestCount = count; }
+      }
+      if (bestOrigin) {
+        const liveRoot = `${bestOrigin}/`;
+        if (!priorityUrls.includes(liveRoot)) priorityUrls.push(liveRoot);
+        const qi = urlQueue.findIndex(u => u === liveRoot || `${u}/` === liveRoot);
+        if (qi > 0) urlQueue.splice(qi, 1);
+        if (qi !== 0) urlQueue.unshift(liveRoot);
+      }
+    }
+
     // ── ACT loop ──────────────────────────────────────────────────────────────
     // urlsFetched (capUrls) and step (capSteps) are deliberately decoupled:
     // urlsFetched counts every fetch attempt (gates how many URLs we try),
@@ -336,6 +390,11 @@ export class AgentOrchestrator {
         `Task: ${prompt}\n\n` +
         `${combinedText}\n\n` +
         `Rules:\n` +
+        // Short and imperative on purpose: the executing model is a small
+        // local one (gemma3:4b-class) and ignores hedged phrasing.
+        (currentState
+          ? `- The task asks about the CURRENT state. Answer from the FIRST source below (the live page). NEVER present older or dated content as the current answer.\n`
+          : '') +
         `- Answer ONLY from the provided sources; do not use outside knowledge.\n` +
         `- Read the sources carefully before concluding anything is missing from them.\n` +
         `- Cite the exact source URL(s) you used.\n` +

package/src/core/ChangeTracker.js

@@ -312,14 +312,22 @@ export class ChangeTracker extends EventEmitter {
       
       this.emit('changeDetected', changeRecord);
       
+      const ignoredOptions = this.findIgnoredCompareOptions(options, baseline.options);
+
       return {
         hasChanges: significance !== 'none',
         significance,
         changeType: changeRecord.changeType,
-        summary: this.generateChangeSummary(changeAnalysis),
+        summary: this.generateChangeSummary(changeAnalysis, significance),
         details: changeAnalysis,
         metrics: changeRecord.metrics,
-        recommendations: this.generateChangeRecommendations(changeRecord)
+        recommendations: this.generateChangeRecommendations(changeRecord),
+        ...(ignoredOptions.length ? {
+          warnings: [
+            `${ignoredOptions.join(', ')} passed to this compare ${ignoredOptions.length === 1 ? 'was' : 'were'} ignored — ` +
+            `the baseline's options are applied to both sides of the diff. Recreate the baseline to change them.`
+          ]
+        } : {})
       };
       
     } catch (error) {
@@ -328,6 +336,27 @@ export class ChangeTracker extends EventEmitter {
     }
   }
   
+  /**
+   * Report analysis options supplied at compare time that differ from the
+   * baseline's and were therefore not applied.
+   *
+   * Ignoring them is deliberate — both sides of a diff have to be analyzed
+   * identically, and once customSelectors scoped the baseline it no longer
+   * holds the full document to re-scope. But doing it silently let a caller
+   * scope a compare and read the resulting whole-page churn as real change:
+   * the result is byte-identical to an unscoped run, with nothing saying so.
+   *
+   * @param {Object} callerOptions - trackingOptions passed to this compare
+   * @param {Object} baselineOptions - options stored with the baseline
+   * @returns {string[]} - names of the ignored options
+   */
+  findIgnoredCompareOptions(callerOptions = {}, baselineOptions = {}) {
+    return ['granularity', 'customSelectors', 'excludeSelectors'].filter(key => {
+      if (callerOptions[key] === undefined) return false;
+      return JSON.stringify(callerOptions[key]) !== JSON.stringify(baselineOptions[key]);
+    });
+  }
+  
   /**
    * Analyze content structure and create hierarchical hashes
    * @param {string} content - Content to analyze
@@ -701,6 +730,29 @@ export class ChangeTracker extends EventEmitter {
         }
       });
     });
+
+    // Scoping to a tag outside that list (address, td, li, tr, dd) otherwise
+    // indexes ZERO elements: the scoped document is hashed, but nothing in it
+    // matches the allowlist, so every compare sees an empty element map and
+    // can never report an element-level change. Hash the scoped elements
+    // themselves for any tag the loop above does not already cover.
+    if (options.customSelectors?.length) {
+      const alreadyHashed = new Set(importantElements);
+      options.customSelectors.forEach((selector, selectorIndex) => {
+        $(selector).each((index, element) => {
+          const tag = element.tagName?.toLowerCase();
+          if (!tag || alreadyHashed.has(tag)) return;
+
+          const elementKey = `custom_${selectorIndex}_${index}`;
+          analysis.hashes.elements[elementKey] = this.hashContent($(element).html() || '');
+
+          if (options.trackAttributes) {
+            const attributes = element.attribs || {};
+            analysis.hashes.elements[`${elementKey}_attr`] = this.hashContent(JSON.stringify(attributes));
+          }
+        });
+      });
+    }
   }
   
   async analyzeTextLevel($, analysis, options) {
@@ -1160,7 +1212,7 @@ export class ChangeTracker extends EventEmitter {
     return 'text_change';
   }
   
-  generateChangeSummary(changeAnalysis) {
+  generateChangeSummary(changeAnalysis, significance) {
     const { addedElements, removedElements, modifiedElements, similarity } = changeAnalysis;
     
     const total = addedElements.length + removedElements.length + modifiedElements.length;
@@ -1171,7 +1223,13 @@ export class ChangeTracker extends EventEmitter {
       added: addedElements.length,
       removed: removedElements.length,
       modified: modifiedElements.length,
-      changeDescription: this.generateChangeDescription(changeAnalysis)
+      // Sub-threshold text noise (a rotating session token, a base64 timestamp)
+      // still lands in textChanges, so the description read "Text content
+      // changed" on a compare that reported hasChanges:false and
+      // totalChanges:0. Defer to the verdict the caller is given.
+      changeDescription: significance === 'none'
+        ? 'No significant changes detected'
+        : this.generateChangeDescription(changeAnalysis)
     };
   }
   

package/src/core/ResearchOrchestrator.js

@@ -1515,12 +1515,22 @@ export class ResearchOrchestrator extends EventEmitter {
       .filter(group => group.avgCredibility >= this.credibilityThreshold)
       .sort((a, b) => b.consensusStrength - a.consensusStrength)
       .slice(0, 10)
-      .map(group => ({
-        finding: group.keywords.join(' '),
-        supportingClaims: group.claims.length,
-        credibility: group.avgCredibility,
-        sources: group.claims.map(c => c.source)
-      }));
+      .map(group => {
+        // Findings must be readable prose. Claims are extractive sentences
+        // from the source content, so surface the most credible one verbatim
+        // — joining the group's keywords here produced stopword-stripped
+        // gibberish ("scraping server model context protocol server that...").
+        const representative = group.claims.reduce(
+          (best, c) => ((c.credibility || 0) > (best.credibility || 0) ? c : best),
+          group.claims[0]
+        );
+        return {
+          finding: representative.claim,
+          supportingClaims: group.claims.length,
+          credibility: group.avgCredibility,
+          sources: group.claims.map(c => c.source)
+        };
+      });
   }
 
   compileSupportingEvidence(sources) {

package/src/core/analysis/ContentAnalyzer.js

@@ -195,6 +195,55 @@ export class ContentAnalyzer {
     };
   }
 
+  /**
+   * Count CJK-script letters and their share of all letters. This is the
+   * script signal language detection uses; the tokenizer reuses it to decide
+   * when whitespace splitting cannot work.
+   * @param {string} text - Text to analyze
+   * @returns {Object} - { letters, han, kana, hangul, share }
+   */
+  cjkScriptCounts(text) {
+    const letters = (text.match(/\p{L}/gu) || []).length;
+    const han = (text.match(/\p{Script=Han}/gu) || []).length;
+    const kana = (text.match(/[\p{Script=Hiragana}\p{Script=Katakana}]/gu) || []).length;
+    const hangul = (text.match(/\p{Script=Hangul}/gu) || []).length;
+    return {
+      letters,
+      han,
+      kana,
+      hangul,
+      share: letters > 0 ? (han + kana + hangul) / letters : 0
+    };
+  }
+
+  /**
+   * True when a meaningful share of the text is in a CJK script, i.e. words
+   * are not whitespace-delimited and must be segmented by dictionary.
+   * @param {string} text - Text to analyze
+   * @returns {boolean}
+   */
+  isCjkText(text) {
+    return this.cjkScriptCounts(text).share >= 0.1;
+  }
+
+  /**
+   * Tokenize text into words with Intl.Segmenter (dictionary-based for CJK
+   * scripts, whitespace/boundary-based otherwise). Only word-like segments
+   * are returned — punctuation and whitespace segments are dropped.
+   * @param {string} text - Text to tokenize
+   * @returns {string[]} - Word tokens
+   */
+  segmentWords(text) {
+    if (!this._wordSegmenter) {
+      this._wordSegmenter = new Intl.Segmenter(undefined, { granularity: 'word' });
+    }
+    const words = [];
+    for (const seg of this._wordSegmenter.segment(text)) {
+      if (seg.isWordLike) words.push(seg.segment);
+    }
+    return words;
+  }
+
   /**
    * Analyze text content with multiple NLP techniques
    * @param {Object} params - Analysis parameters
@@ -288,12 +337,9 @@ export class ContentAnalyzer {
       // samples is detected as English. Those scripts never appear in
       // Latin-script prose, so a meaningful share of them settles the question
       // before trigram scoring gets a say.
-      const letters = (text.match(/\p{L}/gu) || []).length;
+      const { letters, han, kana, hangul, share } = this.cjkScriptCounts(text);
       if (letters > 0) {
-        const han = (text.match(/\p{Script=Han}/gu) || []).length;
-        const kana = (text.match(/[\p{Script=Hiragana}\p{Script=Katakana}]/gu) || []).length;
-        const hangul = (text.match(/\p{Script=Hangul}/gu) || []).length;
-        if ((han + kana + hangul) / letters >= 0.1) {
+        if (share >= 0.1) {
           const code = kana > 0 ? 'jpn' : hangul > han ? 'kor' : 'cmn';
           return {
             code,
@@ -532,8 +578,25 @@ export class ContentAnalyzer {
    */
   async extractTopics(text, options = {}) {
     try {
+      // compromise noun-phrase matching is English-only: on CJK text it emits
+      // whole multi-sentence runs as one "phrase". Use dictionary-segmented
+      // content words, ranked by RAKE-style relative salience like below.
+      if (this.isCjkText(text)) {
+        const termFreq = this.cjkContentWordFrequencies(text);
+        const maxFreq = Math.max(1, ...Object.values(termFreq));
+        return Object.entries(termFreq)
+          .map(([topic, frequency]) => ({
+            topic,
+            confidence: Math.round((frequency / maxFreq) * 100) / 100,
+            keywords: [topic]
+          }))
+          .filter(topic => topic.confidence >= options.minConfidence)
+          .sort((a, b) => b.confidence - a.confidence)
+          .slice(0, options.maxTopics);
+      }
+
       const doc = nlp(text);
-      
+
       // Extract noun phrases as potential topics
       const nounPhrases = doc.nouns().out('array');
       const adjNounPhrases = doc.match('#Adjective+ #Noun+').out('array');
@@ -700,8 +763,24 @@ export class ContentAnalyzer {
    */
   async extractKeywords(text, options = {}) {
     try {
+      // compromise is English-only: on CJK text it returns whole multi-sentence
+      // runs as single "terms". Rank dictionary-segmented content words instead.
+      if (this.isCjkText(text)) {
+        const termFreq = this.cjkContentWordFrequencies(text);
+        const totalTerms = Object.values(termFreq).reduce((sum, freq) => sum + freq, 0);
+        return Object.entries(termFreq)
+          .map(([keyword, frequency]) => ({
+            keyword,
+            frequency,
+            relevance: totalTerms > 0 ? frequency / totalTerms : 0,
+            type: 'word'
+          }))
+          .sort((a, b) => b.relevance - a.relevance)
+          .slice(0, options.maxKeywords);
+      }
+
       const doc = nlp(text);
-      
+
       // Extract different types of terms
       const nouns = doc.nouns().out('array');
       const verbs = doc.verbs().out('array');
@@ -860,7 +939,10 @@ export class ContentAnalyzer {
   calculateStatistics(text) {
     const characters = text.length;
     const charactersNoSpaces = text.replace(/\s/g, '').length;
-    const words = text.split(/\s+/).filter(w => w.length > 0);
+    // CJK text has no whitespace between words — segment by dictionary instead
+    const words = this.isCjkText(text)
+      ? this.segmentWords(text)
+      : text.split(/\s+/).filter(w => w.length > 0);
     const sentences = splitSentences(text);
     const paragraphs = text.split(/\n\s*\n/).filter(p => p.trim().length > 0);
     
@@ -1034,6 +1116,45 @@ export class ContentAnalyzer {
     return orgHeadedAlias.test(text);
   }
 
+  /**
+   * Frequency table of content words for CJK text, built from dictionary
+   * segmentation. Single-character CJK tokens are dropped (overwhelmingly
+   * particles: 的, 是, 了…), as are short/stop-worded Latin tokens mixed in
+   * and common two-character CJK function words.
+   * @param {string} text - Text to analyze
+   * @returns {Object} - Map of word -> frequency
+   */
+  cjkContentWordFrequencies(text) {
+    const freq = {};
+    for (const raw of this.segmentWords(text)) {
+      const word = raw.toLowerCase();
+      const isCjkWord = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u.test(word);
+      const keep = isCjkWord
+        ? word.length >= 2 && !this.isCjkStopWord(word)
+        : word.length > 2 && !this.isStopWord(word);
+      if (keep) {
+        freq[word] = (freq[word] || 0) + 1;
+      }
+    }
+    return freq;
+  }
+
+  /**
+   * Check if a segmented CJK word is a common function word
+   * @param {string} word - Word to check
+   * @returns {boolean} - True if stop word
+   */
+  isCjkStopWord(word) {
+    const cjkStopWords = [
+      '也是', '就是', '我们', '你们', '他们', '她们', '它们', '这个', '那个',
+      '这些', '那些', '一个', '一些', '以及', '或者', '但是', '因为', '所以',
+      '如果', '没有', '可以', '已经', '通过', '由于', '对于', '其中', '并且',
+      '而且', '虽然', '什么', '自己', '这里', '那里'
+    ];
+
+    return cjkStopWords.includes(word);
+  }
+
   /**
    * Check if word is a stop word
    * @param {string} word - Word to check

package/src/core/llm/LLMManager.js

@@ -320,13 +320,37 @@ ${findingsText}
 Synthesize these findings into a comprehensive analysis:`;
 
     try {
+      // Constrain the output shape (Ollama structured outputs; other providers
+      // ignore `format`). Small local models otherwise wrap the JSON in
+      // markdown fences, drift the key names, or emit an empty object — any of
+      // which blanked the synthesis users see.
+      const synthesisSchema = {
+        type: 'object',
+        properties: {
+          summary: { type: 'string' },
+          keyInsights: { type: 'array', items: { type: 'string' } },
+          themes: { type: 'array', items: { type: 'string' } },
+          confidence: { type: 'number' },
+          gaps: { type: 'array', items: { type: 'string' } },
+          recommendations: { type: 'array', items: { type: 'string' } }
+        },
+        required: ['summary', 'keyInsights', 'themes', 'confidence']
+      };
+
       const response = await this.generateCompletion(prompt, {
         systemPrompt,
         maxTokens: 800,
-        temperature: 0.4
+        temperature: 0.4,
+        format: synthesisSchema
       });
 
-      return JSON.parse(response);
+      // Strip markdown code fences if present
+      const cleaned = response.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '').trim();
+      const parsed = JSON.parse(cleaned);
+      if (!parsed || typeof parsed.summary !== 'string' || parsed.summary.length === 0) {
+        throw new Error('Synthesis response missing summary');
+      }
+      return parsed;
     } catch (error) {
       this.logger.warn('LLM synthesis failed, using fallback', { error: error.message });
       return this.fallbackSynthesis(findings, topic);

package/src/core/processing/PDFProcessor.js

@@ -16,6 +16,8 @@ const PDFProcessorSchema = z.object({
   options: z.object({
     extractMetadata: z.boolean().default(true),
     extractText: z.boolean().default(true),
+    // Detect grid tables from the text layer (positioned text items).
+    extractTables: z.boolean().default(false),
     maxPages: z.number().min(1).max(1000).default(100),
     // For decrypting password-protected PDFs (pdf-parse 2.x / pdfjs-dist honors this).
     password: z.string().optional(),
@@ -36,6 +38,10 @@ const PDFResult = z.object({
   source: z.string(),
   sourceType: z.string(),
   text: z.string().optional(),
+  tables: z.array(z.object({
+    page: z.number(),
+    rows: z.array(z.array(z.string()))
+  })).optional(),
   metadata: z.object({
     title: z.string().nullable(),
     author: z.string().nullable(),
@@ -161,6 +167,16 @@ export class PDFProcessor {
           }
         }
 
+        // Extract tables from the text layer. getInfo() above already loaded
+        // the document, so pdf-parse's parser.doc (a plain property in the
+        // compiled build) holds the pdfjs PDFDocumentProxy — reuse it instead
+        // of parsing the buffer a second time.
+        if (processingOptions.extractTables) {
+          const tableStart = pageRange?.start || 1;
+          const tableEnd = Math.min(pageRange?.end || processingOptions.maxPages, totalPages);
+          result.tables = await this.extractTablesFromDocument(parser.doc, tableStart, tableEnd);
+        }
+
         // Extract metadata
         if (processingOptions.extractMetadata) {
           result.metadata = this.extractPDFMetadata(info);
@@ -435,6 +451,195 @@ export class PDFProcessor {
       .trim();
   }
 
+  /**
+   * Extract tables from a loaded pdfjs document's text layer.
+   * Walks the requested pages (1-based, inclusive), reads positioned text
+   * items via getTextContent(), and runs layout-based table detection on each.
+   * @param {Object} doc - pdfjs PDFDocumentProxy (pdf-parse's parser.doc)
+   * @param {number} startPage - First page to scan
+   * @param {number} endPage - Last page to scan
+   * @returns {Promise<Array>} - Detected tables as {page, rows: [[cell, ...], ...]}
+   */
+  async extractTablesFromDocument(doc, startPage, endPage) {
+    const tables = [];
+    const last = Math.min(endPage, doc.numPages);
+    for (let pageNum = Math.max(1, startPage); pageNum <= last; pageNum++) {
+      const page = await doc.getPage(pageNum);
+      const textContent = await page.getTextContent();
+      tables.push(...this.detectTablesFromTextItems(textContent.items, pageNum));
+    }
+    return tables;
+  }
+
+  /**
+   * Detect grid tables in one page's positioned text items (pdfjs
+   * getTextContent() shape: {str, transform, width, height}; transform[4]/[5]
+   * carry x/y). Pure layout analysis:
+   *   1. cluster items into rows by y proximity (sub/superscripts sit ~0.35em
+   *      off the baseline, so they fold into their base row),
+   *   2. split each row into cell segments wherever the x-gap exceeds a
+   *      threshold well above word spacing,
+   *   3. take runs of >= 3 consecutive multi-cell rows and derive column
+   *      boundaries from x-regions that almost no row's text crosses.
+   * Ordinary paragraphs yield single-segment rows (word gaps stay under the
+   * threshold), so they never form a run; coincidental misaligned gaps leave
+   * no shared low-coverage x-region, so no columns emerge.
+   * @param {Array} items - pdfjs text items for one page
+   * @param {number} pageNumber - 1-based page number for the emitted tables
+   * @returns {Array} - Tables as {page, rows: [[cell, ...], ...]}
+   */
+  detectTablesFromTextItems(items, pageNumber) {
+    const texts = (items || [])
+      .filter(item => item.str && item.str.trim().length > 0 && Array.isArray(item.transform))
+      .map(item => ({
+        str: item.str,
+        x: item.transform[4],
+        y: item.transform[5],
+        width: item.width || 0,
+        height: item.height || 10
+      }));
+    if (texts.length === 0) {
+      return [];
+    }
+
+    const heights = texts.map(t => t.height).sort((a, b) => a - b);
+    const medianHeight = heights[Math.floor(heights.length / 2)] || 10;
+    const yTolerance = Math.max(2, medianHeight * 0.4);
+    const gapThreshold = Math.max(3, medianHeight * 0.5);
+
+    // 1. Cluster items into rows, top to bottom.
+    texts.sort((a, b) => b.y - a.y || a.x - b.x);
+    const rows = [];
+    let currentRow = null;
+    let previousY;
+    for (const t of texts) {
+      if (currentRow && previousY - t.y <= yTolerance) {
+        currentRow.items.push(t);
+      } else {
+        currentRow = { y: t.y, items: [t] };
+        rows.push(currentRow);
+      }
+      previousY = t.y;
+    }
+
+    // 2. Split each row into cell segments on x-gaps. PDF text items are often
+    // fragments ("1", ".", "4", "·", "10"), so fragments closer than the gap
+    // threshold join one segment, with a space when they don't visually touch.
+    const spaceGap = medianHeight * 0.12;
+    const segmentedRows = rows.map(row => {
+      const sorted = [...row.items].sort((a, b) => a.x - b.x);
+      const segments = [];
+      let segment = null;
+      for (const t of sorted) {
+        if (segment && t.x - segment.xEnd < gapThreshold) {
+          if (t.x - segment.xEnd > spaceGap) {
+            segment.text += ' ';
+          }
+          segment.text += t.str;
+          segment.xEnd = Math.max(segment.xEnd, t.x + t.width);
+        } else {
+          segment = { xStart: t.x, xEnd: t.x + t.width, text: t.str };
+          segments.push(segment);
+        }
+      }
+      return { y: row.y, segments };
+    });
+
+    // 3. Collect runs of consecutive multi-cell rows with table-like spacing.
+    const maxRowGap = medianHeight * 2.5;
+    const runs = [];
+    let run = null;
+    for (const row of segmentedRows) {
+      if (row.segments.length >= 2) {
+        const previous = run && run[run.length - 1];
+        if (previous && previous.y - row.y <= maxRowGap) {
+          run.push(row);
+        } else {
+          run = [row];
+          runs.push(run);
+        }
+      } else {
+        run = null;
+      }
+    }
+
+    const tables = [];
+    for (const runRows of runs) {
+      if (runRows.length < 3) {
+        continue;
+      }
+      const tableRows = this.buildTableFromRun(runRows, gapThreshold);
+      if (tableRows) {
+        tables.push({ page: pageNumber, rows: tableRows });
+      }
+    }
+    return tables;
+  }
+
+  /**
+   * Turn a run of multi-cell rows into a rows-of-cells grid, or null when the
+   * rows don't align into at least two shared columns.
+   * Column separators are x-regions at least gapThreshold wide that at most
+   * ~20% of the rows' text crosses — a lone spanning header can't merge two
+   * otherwise-separate columns, while misaligned prose yields no separator at
+   * all (the genuine-alignment requirement).
+   * @param {Array} runRows - Rows of {y, segments: [{xStart, xEnd, text}]}
+   * @param {number} gapThreshold - Minimum column-separator width
+   * @returns {Array|null} - Rows as arrays of cell strings, or null
+   */
+  buildTableFromRun(runRows, gapThreshold) {
+    // Sweep segment x-extents to find low-coverage separator regions.
+    const events = [];
+    for (const row of runRows) {
+      for (const segment of row.segments) {
+        events.push({ x: segment.xStart, delta: 1 });
+        events.push({ x: segment.xEnd, delta: -1 });
+      }
+    }
+    events.sort((a, b) => a.x - b.x);
+
+    const maxBridgingRows = Math.floor(runRows.length * 0.2);
+    const separators = [];
+    let coverage = 0;
+    let openStart = null;
+    let i = 0;
+    while (i < events.length) {
+      const x = events[i].x;
+      while (i < events.length && events[i].x === x) {
+        coverage += events[i].delta;
+        i++;
+      }
+      if (coverage <= maxBridgingRows) {
+        if (openStart === null) {
+          openStart = x;
+        }
+      } else {
+        if (openStart !== null && x - openStart >= gapThreshold) {
+          separators.push((openStart + x) / 2);
+        }
+        openStart = null;
+      }
+    }
+    // A trailing open region past the last segment lies outside the table.
+
+    if (separators.length === 0) {
+      return null;
+    }
+
+    return runRows.map(row => {
+      const cells = new Array(separators.length + 1).fill('');
+      for (const segment of row.segments) {
+        const center = (segment.xStart + segment.xEnd) / 2;
+        let col = 0;
+        while (col < separators.length && center > separators[col]) {
+          col++;
+        }
+        cells[col] = cells[col] ? `${cells[col]} ${segment.text}` : segment.text;
+      }
+      return cells.map(cell => cell.trim());
+    });
+  }
+
   /**
    * Process multiple PDFs concurrently
    * @param {Array} sources - Array of PDF sources

package/src/tools/basic/extractMetadata.js

@@ -73,9 +73,10 @@ export async function extractMetadataHandler({ url }) {
     const $ = load(html);
 
     // Stronger title fallback: og:title → <title> → h1
+    // 'head > title' (not bare 'title') so inline <svg><title> elements are not matched
     const title =
       $('meta[property="og:title"]').attr('content') ||
-      $('title').text().trim() ||
+      $('head > title').first().text().trim() ||
       $('h1').first().text().trim() ||
       '';
 
@@ -86,8 +87,9 @@ export async function extractMetadataHandler({ url }) {
     const canonical = $('link[rel="canonical"]').attr('href') || '';
 
     const ogTags = {};
-    $('meta[property^="og:"]').each((_, el) => {
-      const property = $(el).attr('property');
+    // Some sites (e.g. MDN) emit OG tags with name= instead of the standard property=
+    $('meta[property^="og:"], meta[name^="og:"]').each((_, el) => {
+      const property = $(el).attr('property') || $(el).attr('name');
       const content = $(el).attr('content');
       if (property && content) ogTags[property.replace('og:', '')] = content;
     });

package/src/tools/extract/processDocument.js

@@ -20,6 +20,9 @@ const ProcessDocumentSchema = z.object({
     // PDF processing options
     extractText: z.boolean().default(true),
     extractMetadata: z.boolean().default(true),
+    // Detect grid tables in PDFs from the text layer; the result then always
+    // carries a top-level `tables` array (empty when none are found).
+    extractTables: z.boolean().default(false),
     maxPages: z.number().min(1).max(500).default(100),
     // C3: extract a specific 1-based, inclusive page range from a PDF
     pageRange: z.object({
@@ -54,6 +57,10 @@ const ProcessDocumentResult = z.object({
     html: z.string().optional(),
     extractedContent: z.string().optional()
   }),
+  tables: z.array(z.object({
+    page: z.number(),
+    rows: z.array(z.array(z.string()))
+  })).optional(),
   metadata: z.object({
     // Common metadata
     title: z.string().nullable(),
@@ -204,6 +211,7 @@ export class ProcessDocumentTool {
       options: {
         extractText: options.extractText,
         extractMetadata: options.extractMetadata,
+        extractTables: options.extractTables,
         maxPages: options.maxPages,
         ...(options.pageRange ? { pageRange: options.pageRange } : {})
       }
@@ -218,6 +226,12 @@ export class ProcessDocumentTool {
       text: pdfResult.text || ''
     };
 
+    // When table extraction was requested, always answer with a tables array —
+    // honestly empty when the detector found none.
+    if (options.extractTables) {
+      result.tables = pdfResult.tables || [];
+    }
+
     // Set title
     result.title = pdfResult.metadata?.title || null;
 

package/src/tools/extract/summarizeContent.js

@@ -106,10 +106,15 @@ export class SummarizeContentTool {
     
     try {
       const validated = SummarizeContentSchema.parse(params);
-      const { text, options } = validated;
+      const { text: rawText, options } = validated;
+
+      // Page text often opens with navigation chrome ("Jump to content",
+      // "From Wikipedia, the free encyclopedia") that otherwise leads the
+      // summary and key points — strip it before summarizing.
+      const text = this.stripLeadingBoilerplate(rawText);
 
       const result = {
-        originalText: text.substring(0, 500) + (text.length > 500 ? '...' : ''),
+        originalText: rawText.substring(0, 500) + (rawText.length > 500 ? '...' : ''),
         summarizedAt: new Date().toISOString(),
         success: false,
         processingTime: 0
@@ -254,6 +259,48 @@ export class SummarizeContentTool {
     }
   }
 
+  /**
+   * Strip leading navigation boilerplate from extracted page text.
+   *
+   * Conservative rule: scan only the leading lines, dropping each line that
+   * looks navigation-ish — short (≤ 60 chars), few words (≤ 8), and free of
+   * sentence-ending punctuation (.!?…。！？) — and stop at the first line of
+   * real prose (anything longer, wordier, or punctuated). Prose that starts
+   * mid-sentence survives: it is either punctuated, longer than the caps, or
+   * protected by the size guard — if the strip would remove more than
+   * min(600 chars, 20% of the text), nothing is stripped at all.
+   * @param {string} text - Text to clean
+   * @returns {string} - Text without leading navigation chrome
+   */
+  stripLeadingBoilerplate(text) {
+    const lines = text.split('\n');
+    const maxStrip = Math.min(600, Math.floor(text.length * 0.2));
+    let index = 0;
+    let strippedChars = 0;
+    let strippedLines = 0;
+
+    while (index < lines.length) {
+      const line = lines[index].trim();
+      if (line.length === 0) {
+        index++;
+        continue;
+      }
+      const wordCount = line.split(/\s+/).length;
+      const hasSentencePunctuation = /[.!?…。！？]/.test(line);
+      const isBoilerplate = line.length <= 60 && wordCount <= 8 && !hasSentencePunctuation;
+      if (!isBoilerplate) break;
+
+      strippedChars += line.length;
+      if (strippedChars > maxStrip) {
+        return text; // would eat too much — leave the text untouched
+      }
+      strippedLines++;
+      index++;
+    }
+
+    return strippedLines > 0 ? lines.slice(index).join('\n') : text;
+  }
+
   /**
    * Extract key points from original text and summary
    * @param {string} originalText - Original text

package/src/tools/research/deepResearch.js

@@ -499,6 +499,9 @@ export class DeepResearchTool {
         return {
           ...formatted,
           findings: results.findings,
+          // LLM synthesis (aiSummary/intelligentInsights) — without this the
+          // llmEnhanced flag was true but the LLM output never reached users.
+          insights: results.insights,
           supportingEvidence: results.supportingEvidence,
           consensus: results.consensus,
           conflicts: results.conflicts,
@@ -514,6 +517,7 @@ export class DeepResearchTool {
         return {
           ...formatted,
           keyFindings: results.findings.slice(0, 5),
+          aiSummary: results.insights?.aiSummary,
           topSources: results.supportingEvidence.slice(0, 5),
           mainConflicts: results.conflicts.slice(0, 3),
           primaryRecommendations: results.recommendations.slice(0, 3),

package/src/tools/search/adapters/searchProviderFactory.js

@@ -48,7 +48,8 @@ export class SearchProviderFactory {
 
     return new CrawlForgeSearchAdapter(
       apiKey,
-      options.apiBaseUrl || 'https://api.crawlforge.dev'
+      // www is the live API host — api.crawlforge.dev does not resolve.
+      options.apiBaseUrl || 'https://www.crawlforge.dev'
     );
   }
 

package/src/tools/search/redditSearch.js

@@ -101,12 +101,15 @@ export class RedditSearchTool {
     // and PullPush stopped serving automated clients in August 2026.
     this.searchAdapter = options.searchAdapter || null;
     this.searchApiKey = options.searchApiKey || null;
+    this.searchApiBaseUrl = options.searchApiBaseUrl || null;
   }
 
   /** The web-search adapter used to discover posts, built once on first use. */
   #search() {
     if (!this.searchAdapter) {
-      this.searchAdapter = SearchProviderFactory.createAdapter(this.searchApiKey);
+      this.searchAdapter = SearchProviderFactory.createAdapter(this.searchApiKey, {
+        apiBaseUrl: this.searchApiBaseUrl,
+      });
     }
     return this.searchAdapter;
   }

package/src/tools/tracking/trackChanges/index.js

@@ -281,6 +281,7 @@ export class TrackChangesTool extends EventEmitter {
       details: comparisonResult.details,
       metrics: comparisonResult.metrics,
       recommendations: comparisonResult.recommendations,
+      ...(comparisonResult.warnings ? { warnings: comparisonResult.warnings } : {}),
       snapshot: snapshotInfo, timestamp: Date.now()
     };
   }