crawlforge-mcp-server 4.10.0 → 5.0.1

package/src/core/analysis/ContentAnalyzer.js

@@ -3,7 +3,6 @@
  * Uses multiple NLP libraries for comprehensive content analysis
  */
 
-import { SummarizerManager } from 'node-summarizer';
 import { franc, francAll } from 'franc';
 import nlp from 'compromise';
 import { z } from 'zod';
@@ -179,7 +178,6 @@ const LANGUAGE_NAMES = {
 
 export class ContentAnalyzer {
   constructor() {
-    this.summarizer = new SummarizerManager();
     this.defaultOptions = {
       summarize: true,
       detectLanguage: true,
@@ -324,7 +322,7 @@ export class ContentAnalyzer {
       .map(([code, score]) => ({
         code,
         name: LANGUAGE_NAMES[code] || code,
-        confidence: Math.round((1 - score) * 100) / 100
+        confidence: Math.round(score * 100) / 100
       }));
 
       return {
@@ -379,11 +377,15 @@ export class ContentAnalyzer {
       targetSentences = Math.min(targetSentences, sentences.length);
 
       let summarySentences;
-      
+
       if (options.summaryType === 'extractive') {
-        // Use node-summarizer for extractive summarization
-        const summary = await this.summarizer.getSummaryByRanking(text, targetSentences);
-        summarySentences = splitSentences(summary);
+        // Extractive summarization via word-frequency + position scoring
+        // (Luhn-style salience), tokenized with compromise. Selects the
+        // top-N sentences and restores original document order.
+        summarySentences = this.createExtractiveSummary(sentences, targetSentences);
+        if (!summarySentences || summarySentences.length === 0) {
+          throw new Error('Extractive summarization returned no sentences');
+        }
       } else {
         // Simple abstractive approach (for demonstration)
         summarySentences = await this.createAbstractiveSummary(text, targetSentences);
@@ -444,6 +446,49 @@ export class ContentAnalyzer {
       .map(item => item.sentence.trim());
   }
 
+  /**
+   * Create extractive summary by scoring pre-split sentences via word
+   * frequency (Luhn-style salience), tokenized with compromise, plus a small
+   * positional bonus for leading/closing sentences. Selects the top N and
+   * restores original document order.
+   * @param {string[]} sentences - Sentences in original document order
+   * @param {number} targetSentences - Number of sentences to select
+   * @returns {string[]} - Selected sentences, restored to original order
+   */
+  createExtractiveSummary(sentences, targetSentences) {
+    if (targetSentences >= sentences.length) {
+      return sentences.slice();
+    }
+
+    // Word-frequency table (stop words excluded) drives sentence salience.
+    const freq = {};
+    const sentenceWords = sentences.map(sentence => {
+      const words = nlp(sentence).terms().out('array')
+        .map(w => w.toLowerCase().replace(/[^a-z0-9]/g, ''))
+        .filter(w => w.length > 2 && !this.isStopWord(w));
+      words.forEach(w => { freq[w] = (freq[w] || 0) + 1; });
+      return words;
+    });
+
+    const maxFreq = Math.max(1, ...Object.values(freq));
+
+    const scored = sentences.map((sentence, index) => {
+      const words = sentenceWords[index];
+      const wordScore = words.length > 0
+        ? words.reduce((sum, w) => sum + freq[w] / maxFreq, 0) / words.length
+        : 0;
+      // Leading/closing sentences tend to carry more salience in prose.
+      const positionScore = (index === 0 || index === sentences.length - 1) ? 0.15 : 0;
+      return { sentence, index, score: wordScore + positionScore };
+    });
+
+    return scored
+      .sort((a, b) => b.score - a.score)
+      .slice(0, targetSentences)
+      .sort((a, b) => a.index - b.index)
+      .map(item => item.sentence);
+  }
+
   /**
    * Extract topics from text
    * @param {string} text - Text to analyze

package/src/core/crawlers/BFSCrawler.js

@@ -168,7 +168,13 @@ export class BFSCrawler {
       }
     }
 
-    // Mark as visited
+    // Mark as visited. Re-check the cap and dedupe first: the checks at the
+    // top of this task ran before the awaited robots lookup, so concurrent
+    // queue tasks may have filled the budget (or claimed this URL) since.
+    // This block is synchronous, so the cap is exact.
+    if (this.visited.size >= this.maxPages || this.visited.has(normalizedUrl)) {
+      return;
+    }
     this.visited.add(normalizedUrl);
 
     try {
@@ -204,11 +210,15 @@ export class BFSCrawler {
 
       // Process links for analysis
       if (this.enableLinkAnalysis && this.linkAnalyzer && pageData.links) {
+        // Parse the page once and reuse for every link; re-parsing per link is
+        // O(links × page size) and starves the event loop on link-dense pages.
+        const $page = pageData.originalHtml ? load(pageData.originalHtml) : null;
+        const pageBodyText = $page ? $page('body').text() : '';
         for (const link of pageData.links) {
           const absoluteUrl = this.resolveUrl(link, normalizedUrl);
           if (absoluteUrl) {
             // Extract anchor text and context from link
-            const linkMetadata = this.extractLinkMetadata(link, pageData.originalHtml, normalizedUrl);
+            const linkMetadata = this.extractLinkMetadata(link, $page, pageBodyText);
             this.linkAnalyzer.addLink(normalizedUrl, absoluteUrl, linkMetadata);
           }
         }
@@ -238,7 +248,19 @@ export class BFSCrawler {
           
           const absoluteUrl = this.resolveUrl(link, normalizedUrl);
           if (absoluteUrl && !this.visited.has(absoluteUrl)) {
-            await this.queue.add(() => this.processUrl(absoluteUrl, depth + 1));
+            // Not awaited: this task already holds a queue slot, so awaiting a
+            // child would keep that slot pinned for the rest of the recursive
+            // crawl (starving other tasks when concurrency <= depth, and making
+            // the per-task queue timeout measure the whole crawl instead of one
+            // page). crawl() waits for everything via queue.onIdle() instead.
+            this.queue.add(() => this.processUrl(absoluteUrl, depth + 1)).catch(error => {
+              this.errors.push({
+                url: absoluteUrl,
+                depth: depth + 1,
+                error: error.message,
+                timestamp: new Date().toISOString()
+              });
+            });
           }
         }
       }
@@ -254,7 +276,7 @@ export class BFSCrawler {
 
   async fetchPage(url) {
     const controller = new AbortController();
-    const timeoutId = setTimeout(() => controller.abort(), this.timeout);
+    let timeoutId = setTimeout(() => controller.abort(), this.timeout);
 
     try {
       // Get domain-specific headers and timeout
@@ -282,7 +304,7 @@ export class BFSCrawler {
       // Update timeout if different
       if (effectiveTimeout !== this.timeout) {
         clearTimeout(timeoutId);
-        setTimeout(() => controller.abort(), effectiveTimeout);
+        timeoutId = setTimeout(() => controller.abort(), effectiveTimeout);
       }
 
       const response = await safeFetch(url, {
@@ -445,6 +467,18 @@ export class BFSCrawler {
     this.queue.pause();
   }
 
+  /**
+   * Release resources held by this crawler instance (cache cleanup/monitoring
+   * timers). Must be called by the owner once crawling is complete — unref()
+   * on the timers keeps the process from hanging, but the instance itself
+   * stays reachable (and its cached pages retained) until destroy() runs.
+   */
+  destroy() {
+    if (this.cache && typeof this.cache.destroy === 'function') {
+      this.cache.destroy();
+    }
+  }
+
   /**
    * Get the domain filter instance
    * @returns {DomainFilter} Current domain filter
@@ -524,19 +558,18 @@ export class BFSCrawler {
   }
 
   /**
-   * Extract link metadata from HTML
+   * Extract link metadata from a pre-parsed page
    * @param {string} href - The href attribute value
-   * @param {string} html - Original HTML content
-   * @param {string} baseUrl - Base URL for context
+   * @param {Object} $ - Cheerio document for the page (parsed once per page)
+   * @param {string} bodyText - Pre-computed body text of the page
    * @returns {Object} Link metadata
    */
-  extractLinkMetadata(href, html, baseUrl) {
-    if (!html) return {};
+  extractLinkMetadata(href, $, bodyText = '') {
+    if (!$) return {};
 
     try {
-      const $ = load(html);
       const linkElement = $(`a[href="${href}"]`).first();
-      
+
       if (linkElement.length === 0) {
         return { href };
       }
@@ -545,10 +578,8 @@ export class BFSCrawler {
       const title = linkElement.attr('title');
       const rel = linkElement.attr('rel');
       const className = linkElement.attr('class');
-      
+
       // Get surrounding context (up to 100 characters before and after)
-      const linkHtml = linkElement.prop('outerHTML');
-      const bodyText = $('body').text();
       const linkTextIndex = bodyText.indexOf(anchorText);
       let context = '';
       

package/src/core/processing/BrowserProcessor.js

@@ -201,8 +201,17 @@ export class BrowserProcessor {
         });
 
       } finally {
-        // Always close the page
+        // Always close the page. Also close the owning context for
+        // non-stealth pages — createPage() gives each call its own
+        // dedicated BrowserContext that is never tracked/closed elsewhere,
+        // so leaving it open here leaks it until server shutdown. Stealth
+        // contexts are pooled/reused (see StealthBrowserManager /
+        // activeContexts) and must not be closed here.
+        const ctx = !processingOptions.stealthMode?.enabled ? page.context() : null;
         await page.close();
+        if (ctx) {
+          await ctx.close();
+        }
       }
 
       return result;
@@ -864,6 +873,15 @@ export class BrowserProcessor {
       this.humanBehaviorSimulator.resetStats();
       this.humanBehaviorSimulator = null;
     }
+
+    // Tear down the LocalizationManager this processor created — its
+    // health-check setInterval timers would otherwise keep firing for the
+    // process lifetime after this processor is discarded. The instance is
+    // kept (not nulled): it's constructor-created with no lazy re-init, and
+    // localizeBrowserContext() must keep working if the processor is reused.
+    if (this.localizationManager) {
+      await this.localizationManager.cleanup();
+    }
   }
 
   /**

package/src/core/processing/PDFProcessor.js

@@ -7,6 +7,8 @@
 import { z } from 'zod';
 import fs from 'fs/promises';
 import path from 'path';
+import { safeFetch } from '../../utils/ssrfGuard.js';
+import { config } from '../../constants/config.js';
 
 const PDFProcessorSchema = z.object({
   source: z.string().min(1),
@@ -14,8 +16,9 @@ const PDFProcessorSchema = z.object({
   options: z.object({
     extractMetadata: z.boolean().default(true),
     extractText: z.boolean().default(true),
-    password: z.string().optional(),
     maxPages: z.number().min(1).max(1000).default(100),
+    // For decrypting password-protected PDFs (pdf-parse 2.x / pdfjs-dist honors this).
+    password: z.string().optional(),
     // C3: true page-range extraction (1-based, inclusive). When set, only the
     // text from pages [start..end] is returned.
     pageRange: z.object({
@@ -101,70 +104,79 @@ export class PDFProcessor {
         return result;
       }
 
-      // C3: when a page range is requested, capture per-page text so we can
-      // return exactly pages [start..end] (pdf-parse otherwise concatenates the
-      // whole document and its `max` option only caps the *upper* page bound).
+      // C3: page-range extraction (1-based, inclusive) — pdf-parse 2.x's
+      // getText({ partial: [...] }) parses and returns exactly the requested
+      // pages, so no manual per-page capture is needed here anymore.
       const pageRange = processingOptions.pageRange;
-      const capturedPages = [];
-
-      // Parse PDF with options
-      const parseOptions = {
-        ...processingOptions.parseOptions,
-        max: processingOptions.maxPages
-      };
-
-      // If extracting a range, raise `max` to at least the requested end page
-      // and install a pagerender that records each page's text.
-      if (pageRange) {
-        if (pageRange.end) {
-          parseOptions.max = Math.max(parseOptions.max, pageRange.end);
-        } else {
-          parseOptions.max = processingOptions.maxPages;
-        }
-        parseOptions.pagerender = (pageData) => this._renderPage(pageData, capturedPages);
-      }
 
-      if (processingOptions.password) {
-        parseOptions.password = processingOptions.password;
-      }
+      // Dynamic import to avoid initialization issues
+      const { PDFParse, PasswordException } = await import('pdf-parse');
+      const parser = new PDFParse({
+        data: pdfBuffer,
+        ...(processingOptions.password ? { password: processingOptions.password } : {})
+      });
 
-      let pdfData;
       try {
-        // Dynamic import to avoid initialization issues
-        const pdfParse = (await import('pdf-parse')).default;
-        pdfData = await pdfParse(pdfBuffer, parseOptions);
-      } catch (error) {
-        result.error = `PDF parsing failed: ${error.message}`;
-        result.processingTime = Date.now() - startTime;
-        return result;
-      }
+        let info;
+        try {
+          info = await parser.getInfo();
+        } catch (error) {
+          if (error instanceof PasswordException) {
+            result.error = `PDF parsing failed: password required or incorrect (${error.message})`;
+          } else {
+            result.error = `PDF parsing failed: ${error.message}`;
+          }
+          result.processingTime = Date.now() - startTime;
+          return result;
+        }
 
-      // Extract text content
-      if (processingOptions.extractText) {
-        if (pageRange) {
-          const start = pageRange.start || 1;
-          const end = pageRange.end || capturedPages.length;
-          const slice = capturedPages.slice(start - 1, end);
-          result.text = this.cleanPDFText(slice.join('\n\n'));
-          result.extractedPages = { start, end, count: slice.length };
-        } else {
-          result.text = this.cleanPDFText(pdfData.text);
+        const totalPages = info.total || 0;
+        // pdf-parse 2.x has no equivalent of v1's disableCombineTextItems; only
+        // normalizeWhitespace maps onto a v2 ParseParameters field (inverted).
+        const disableNormalization = !processingOptions.parseOptions?.normalizeWhitespace;
+
+        // Extract text content
+        if (processingOptions.extractText) {
+          if (pageRange) {
+            const start = pageRange.start || 1;
+            // C3: a start past the last page means the requested range
+            // doesn't exist in this PDF — report that explicitly instead of
+            // silently returning success:true with empty text.
+            if (start > totalPages) {
+              result.error = `Requested page range starts at page ${start}, but the PDF only has ${totalPages} page(s).`;
+              result.processingTime = Date.now() - startTime;
+              return result;
+            }
+            const end = Math.min(pageRange.end || processingOptions.maxPages, totalPages);
+            const pageNumbers = [];
+            for (let n = start; n <= end; n++) pageNumbers.push(n);
+
+            const textResult = await parser.getText({ partial: pageNumbers, disableNormalization });
+            const slice = textResult.pages.map(p => p.text);
+            result.text = this.cleanPDFText(slice.join('\n\n'));
+            result.extractedPages = { start, end, count: slice.length };
+          } else {
+            const textResult = await parser.getText({ first: processingOptions.maxPages, disableNormalization });
+            result.text = this.cleanPDFText(textResult.pages.map(p => p.text).join('\n\n'));
+          }
         }
-      }
 
-      // Extract metadata
-      if (processingOptions.extractMetadata) {
-        result.metadata = this.extractPDFMetadata(pdfData);
-      }
+        // Extract metadata
+        if (processingOptions.extractMetadata) {
+          result.metadata = this.extractPDFMetadata(info);
+        }
 
-      // Set page count
-      result.pageCount = pdfData.numpages || 0;
+        // Set page count
+        result.pageCount = totalPages;
 
-      // Calculate processing time
-      result.processingTime = Date.now() - startTime;
-      result.success = true;
+        // Calculate processing time
+        result.processingTime = Date.now() - startTime;
+        result.success = true;
 
-      return result;
+        return result;
+      } finally {
+        await parser.destroy().catch(() => {});
+      }
 
     } catch (error) {
       return {
@@ -205,11 +217,14 @@ export class PDFProcessor {
    */
   async downloadPDFFromURL(url) {
     try {
-      const response = await fetch(url, {
+      // `timeout` is not a fetch init option — undici/Node fetch silently
+      // ignores unknown properties, so only `signal` actually enforces a
+      // deadline here.
+      const response = await safeFetch(url, {
         headers: {
           'User-Agent': 'Mozilla/5.0 (compatible; MCP-WebScraper/2.0; PDF-Processor)'
         },
-        timeout: 30000
+        signal: AbortSignal.timeout(30000)
       });
 
       if (!response.ok) {
@@ -221,14 +236,61 @@ export class PDFProcessor {
         console.warn(`Warning: Content-Type is ${contentType}, expected PDF`);
       }
 
-      const arrayBuffer = await response.arrayBuffer();
-      return Buffer.from(arrayBuffer);
+      return await this.readBodyWithSizeCap(response);
 
     } catch (error) {
+      // AbortSignal.timeout() aborts with a TimeoutError-named DOMException
+      // (not AbortError — that name is only used for a plain controller.abort()).
+      if (error.name === 'TimeoutError' || error.name === 'AbortError') {
+        throw new Error('Failed to download PDF from URL: request timeout after 30000ms');
+      }
       throw new Error(`Failed to download PDF from URL: ${error.message}`);
     }
   }
 
+  /**
+   * Read a response body into a Buffer while enforcing config.fetch.maxBodySize
+   * (Content-Length pre-check, then a streaming byte-count backstop for
+   * servers that omit or lie about it), so a stalling or multi-GB response
+   * can't hang the request indefinitely or OOM the process before pdf-parse's
+   * own maxPages cap ever applies.
+   * @param {Response} response
+   * @returns {Promise<Buffer>}
+   */
+  async readBodyWithSizeCap(response) {
+    const maxBodySize = config.fetch.maxBodySize;
+
+    const contentLengthHeader = response.headers?.get?.('content-length') ?? null;
+    if (contentLengthHeader !== null) {
+      const declared = parseInt(contentLengthHeader, 10);
+      if (!isNaN(declared) && declared > maxBodySize) {
+        throw new Error(`PDF too large: Content-Length ${declared} exceeds limit of ${maxBodySize} bytes`);
+      }
+    }
+
+    if (!response.body || typeof response.body.getReader !== 'function') {
+      const arrayBuffer = await response.arrayBuffer();
+      return Buffer.from(arrayBuffer);
+    }
+
+    const reader = response.body.getReader();
+    const chunks = [];
+    let totalBytes = 0;
+
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      totalBytes += value.byteLength;
+      if (totalBytes > maxBodySize) {
+        reader.cancel();
+        throw new Error(`PDF too large: exceeded limit of ${maxBodySize} bytes`);
+      }
+      chunks.push(value);
+    }
+
+    return Buffer.concat(chunks, totalBytes);
+  }
+
   /**
    * Read PDF from local file
    * @param {string} filePath - Local file path
@@ -260,12 +322,12 @@ export class PDFProcessor {
 
   /**
    * Extract and format PDF metadata
-   * @param {Object} pdfData - Parsed PDF data from pdf-parse
+   * @param {Object} infoResult - InfoResult from pdf-parse's PDFParse#getInfo()
    * @returns {Object} - Formatted metadata
    */
-  extractPDFMetadata(pdfData) {
-    const info = pdfData.info || {};
-    const metadata = pdfData.metadata || {};
+  extractPDFMetadata(infoResult) {
+    const info = infoResult.info || {};
+    const metadata = infoResult.metadata || {};
 
     return {
       title: this.cleanMetadataValue(info.Title || metadata.title),
@@ -276,8 +338,10 @@ export class PDFProcessor {
       creationDate: this.formatPDFDate(info.CreationDate || metadata.creationDate),
       modificationDate: this.formatPDFDate(info.ModDate || metadata.modificationDate),
       format: this.cleanMetadataValue(info.Format || metadata.format),
-      pages: pdfData.numpages || null,
-      encrypted: info.IsEncrypted || false,
+      pages: infoResult.total || null,
+      // pdfjs-dist's Info dictionary has no `IsEncrypted` flag; it reports the
+      // security filter name (e.g. "Standard") when the doc is encrypted, null otherwise.
+      encrypted: !!info.EncryptFilterName,
       linearized: info.IsLinearized || false,
       pdfVersion: this.cleanMetadataValue(info.PDFFormatVersion || metadata.pdfVersion)
     };

package/src/core/queue/QueueManager.js

@@ -9,12 +9,13 @@ export class QueueManager {
       timeout = 30000
     } = options;
 
+    // p-queue v9 removed `throwOnTimeout` — timeouts always throw now
+    // (this was already the effective behavior since it was set to true).
     this.queue = new PQueue({
       concurrency,
       interval,
       intervalCap,
-      timeout,
-      throwOnTimeout: true
+      timeout
     });
 
     this.stats = {