crawlforge-mcp-server 4.9.0 → 5.0.0

package/src/tools/extract/_fetchAndParse.js

@@ -12,10 +12,90 @@
 
 import { load } from 'cheerio';
 import { safeFetch } from '../../utils/ssrfGuard.js';
+import { config } from '../../constants/config.js';
 
 const DEFAULT_USER_AGENT = 'Mozilla/5.0 (compatible; CrawlForge-MCP/3.0)';
 const DEFAULT_TIMEOUT_MS = 15000;
 
+/**
+ * Read a response body as text while enforcing config.fetch.maxBodySize —
+ * the same cap _fetch.js applies to basic tools, so a large/hostile response
+ * (multi-hundred-MB file, endpoint streaming zeros) can't be buffered whole
+ * into a JS string and handed to cheerio/JSDOM/Turndown downstream.
+ * Content-Length is checked up front; actual bytes read are counted as a
+ * backstop for servers that omit or lie about it.
+ * @param {Response} response
+ * @returns {Promise<string>}
+ */
+async function readTextWithSizeCap(response) {
+  const maxBodySize = config.fetch.maxBodySize;
+
+  const contentLengthHeader = response.headers?.get?.('content-length') ?? null;
+  if (contentLengthHeader !== null) {
+    const declared = parseInt(contentLengthHeader, 10);
+    if (!isNaN(declared) && declared > maxBodySize) {
+      throw new Error(
+        `Response body too large: Content-Length ${declared} exceeds limit of ${maxBodySize} bytes`
+      );
+    }
+  }
+
+  // Responses without a ReadableStream body (test mocks, already-buffered
+  // responses) fall back to the native .text() with no additional guard.
+  if (!response.body || typeof response.body.getReader !== 'function') {
+    return response.text();
+  }
+
+  const reader = response.body.getReader();
+  const chunks = [];
+  let totalBytes = 0;
+
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    totalBytes += value.byteLength;
+    if (totalBytes > maxBodySize) {
+      reader.cancel();
+      throw new Error(
+        `Response body too large: exceeded limit of ${maxBodySize} bytes`
+      );
+    }
+    chunks.push(value);
+  }
+
+  const mergedBytes = new Uint8Array(totalBytes);
+  let offset = 0;
+  for (const chunk of chunks) {
+    mergedBytes.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return new TextDecoder().decode(mergedBytes);
+}
+
+/**
+ * Classify a Content-Type header for the purposes of HTML parsing.
+ * Missing header is treated as 'html' (permissive default — many servers,
+ * and most test doubles, omit it for what is genuinely HTML).
+ * @param {string|null} contentType
+ * @returns {'html'|'text'|'binary'}
+ */
+function classifyContentType(contentType) {
+  if (!contentType) return 'html';
+  const type = contentType.split(';')[0].trim().toLowerCase();
+  if (
+    type === 'text/html' ||
+    type === 'application/xhtml+xml' ||
+    type === 'application/xml' ||
+    type === 'text/xml' ||
+    type.endsWith('+xml') ||
+    type.startsWith('text/')
+  ) {
+    return type === 'text/plain' ? 'text' : 'html';
+  }
+  if (type === 'application/json') return 'text';
+  return 'binary';
+}
+
 /**
  * Fetch a URL and return parsed HTML via Cheerio.
  *
@@ -45,7 +125,24 @@ export async function fetchAndParse(url, options = {}) {
     throw new Error(`HTTP ${response.status}: ${response.statusText}`);
   }
 
-  const html = await response.text();
+  const contentType = response.headers?.get?.('content-type') || null;
+  const classification = classifyContentType(contentType);
+
+  if (classification === 'binary') {
+    throw new Error(
+      `Unsupported content type "${contentType}" — this looks like binary content, not HTML/text. Use process_document for PDFs/documents/binary files.`
+    );
+  }
+
+  const html = await readTextWithSizeCap(response);
+
+  // text/plain and application/json aren't markup: running them through the
+  // HTML parser risks misinterpreting substrings (e.g. a "<script>" value
+  // inside a JSON string) as real tags and stripping/mangling content.
+  if (classification === 'text') {
+    return { html, $: load(''), textContent: html.trim(), finalUrl: response.url };
+  }
+
   const $ = load(html);
 
   if (stripTags.length > 0) {

package/src/tools/extract/extractContent.js

@@ -298,14 +298,17 @@ export class ExtractContentTool {
    * @returns {Promise<boolean>} - Whether JavaScript is needed
    */
   async shouldUseJavaScript(url) {
-    // Simple heuristics for determining if JavaScript is needed
+    // Simple heuristics for determining if JavaScript is needed. Strip the
+    // fragment first: an ordinary document anchor (e.g. #install) isn't a
+    // signal for client-side routing, and anchoring the path pattern to full
+    // segments avoids false positives like "/apple" or "/spaces".
+    const urlWithoutFragment = url.split('#')[0];
     const jsIndicators = [
-      /\/(app|spa|dashboard|admin)/,
-      /#/,
+      /\/(app|spa|dashboard|admin)(\/|$)/,
       /\.(js|jsx|ts|tsx)$/
     ];
 
-    return jsIndicators.some(pattern => pattern.test(url));
+    return jsIndicators.some(pattern => pattern.test(urlWithoutFragment));
   }
 
   /**

package/src/tools/extract/extractStructured.js

@@ -109,10 +109,13 @@ export class ExtractStructuredTool {
       // Step 3: Try LLM extraction first
       let extractionResult = null;
       let extractionMethod = 'llm';
+      let llmErrorMessage = null;
+      let llmAvailable = false;
 
       try {
         const llm = this._ensureLLMManager(llmConfig || {});
-        if (llm.isAvailable()) {
+        llmAvailable = llm.isAvailable();
+        if (llmAvailable) {
           extractionResult = await llm.extractStructured(textContent, schema, {
             prompt: prompt || '',
             maxContentLength: 6000
@@ -120,12 +123,36 @@ export class ExtractStructuredTool {
           extractionMethod = 'llm';
         }
       } catch (llmError) {
-        // LLM failed — will fall through to CSS fallback
+        // LLM failed — will fall through to CSS fallback. Keep the message so
+        // callers can tell "LLM broken" apart from "no LLM configured".
         extractionResult = null;
+        llmErrorMessage = llmError.message;
       }
 
       // Step 4: CSS selector fallback if LLM unavailable or failed
       if (!extractionResult && fallbackToSelectors !== false) {
+        // D1.4: no LLM configured and the schema demands more than 3 required
+        // fields — confirm before running the lower-fidelity CSS fallback.
+        const requiredCount = (schema.required || []).length;
+        if (!llmAvailable && requiredCount > 3) {
+          const proceed = await this._elicitation.confirm(
+            `No LLM provider is configured and the requested schema has ${requiredCount} required fields. ` +
+            `extract_structured will fall back to lower-fidelity CSS selector extraction, which may miss required fields.`,
+            { url, required_fields: requiredCount }
+          );
+          if (!proceed) {
+            return {
+              url,
+              data: {},
+              extraction_method: 'none',
+              confidence: 0,
+              schema_used: schema,
+              processingTime: Date.now() - startTime,
+              error: 'Extraction cancelled by user (elicitation declined).',
+              validation: { valid: false, errors: ['Extraction cancelled by user (elicitation declined).'] }
+            };
+          }
+        }
         extractionResult = this._cssExtraction($, schema, selectorHints || {});
         extractionMethod = 'css_fallback';
       }
@@ -140,6 +167,11 @@ export class ExtractStructuredTool {
       // Step 6: Calculate confidence
       const confidence = this._calculateConfidence(extractionResult, extractionMethod);
 
+      const extractionNotes = extractionResult.extractionNotes || [];
+      if (llmErrorMessage) {
+        extractionNotes.push(`LLM extraction failed: ${llmErrorMessage}`);
+      }
+
       return {
         url,
         data: extractionResult.data || {},
@@ -151,7 +183,7 @@ export class ExtractStructuredTool {
           valid: extractionResult.valid || false,
           errors: extractionResult.validationErrors || []
         },
-        extractionNotes: extractionResult.extractionNotes || []
+        extractionNotes
       };
 
     } catch (error) {
@@ -178,98 +210,77 @@ export class ExtractStructuredTool {
     let fieldsFound = 0;
 
     for (const [key, fieldSchema] of Object.entries(properties)) {
-      const isArrayField = fieldSchema.type === 'array';
-
-      // Use explicit selector hint if provided
-      const selector = selectorHints[key];
-      if (selector) {
-        const els = $(selector);
-        if (els.length > 0) {
-          if (isArrayField || els.length > 1) {
-            const values = els.map((_, el) => $(el).text().trim()).get().filter(Boolean);
-            if (values.length > 0) {
-              extracted[key] = values;
-              fieldsFound++;
-              continue;
-            }
-          } else {
-            const rawValue = els.first().text().trim();
-            if (rawValue) {
-              extracted[key] = this._coerceValue(rawValue, fieldSchema);
-              fieldsFound++;
-              continue;
+      // Schema keys become CSS selector fragments below (class/id/data-attr).
+      // A key with spaces/parens/quotes produces an invalid selector that
+      // throws ("Attribute selector didn't terminate") — catch that per-field
+      // so one bad key can't discard extraction results for every other field.
+      try {
+        const isArrayField = fieldSchema.type === 'array';
+
+        // Use explicit selector hint if provided
+        const selector = selectorHints[key];
+        if (selector) {
+          const els = $(selector);
+          if (els.length > 0) {
+            if (isArrayField || els.length > 1) {
+              const values = els.map((_, el) => $(el).text().trim()).get().filter(Boolean);
+              if (values.length > 0) {
+                extracted[key] = values;
+                fieldsFound++;
+                continue;
+              }
+            } else {
+              const rawValue = els.first().text().trim();
+              if (rawValue) {
+                extracted[key] = this._coerceValue(rawValue, fieldSchema);
+                fieldsFound++;
+                continue;
+              }
             }
           }
         }
-      }
 
-      // For array fields: detect ul/ol > li patterns before meta/common selectors
-      if (isArrayField) {
-        const listSelectors = [
-          `ul.${key} > li`, `ol.${key} > li`,
-          `#${key} > li`, `[data-${key}] > li`,
-          `ul[class*="${key}"] > li`, `ol[class*="${key}"] > li`
-        ];
-        let listValues = null;
-        for (const lsel of listSelectors) {
-          const items = $(lsel);
-          if (items.length > 0) {
-            listValues = items.map((_, el) => $(el).text().trim()).get().filter(Boolean);
-            break;
+        // For array fields: detect ul/ol > li patterns before meta/common selectors
+        if (isArrayField) {
+          const listSelectors = [
+            `ul.${key} > li`, `ol.${key} > li`,
+            `#${key} > li`, `[data-${key}] > li`,
+            `ul[class*="${key}"] > li`, `ol[class*="${key}"] > li`
+          ];
+          let listValues = null;
+          for (const lsel of listSelectors) {
+            const items = $(lsel);
+            if (items.length > 0) {
+              listValues = items.map((_, el) => $(el).text().trim()).get().filter(Boolean);
+              break;
+            }
+          }
+          if (listValues && listValues.length > 0) {
+            extracted[key] = listValues;
+            fieldsFound++;
+            continue;
           }
         }
-        if (listValues && listValues.length > 0) {
-          extracted[key] = listValues;
+
+        // Try common patterns: meta tags, headings, semantic elements
+        const metaContent = $(`meta[name="${key}"], meta[property="${key}"], meta[property="og:${key}"]`).attr('content');
+        if (metaContent) {
+          extracted[key] = this._coerceValue(metaContent, fieldSchema);
           fieldsFound++;
           continue;
         }
-      }
-
-      // Try common patterns: meta tags, headings, semantic elements
-      const metaContent = $(`meta[name="${key}"], meta[property="${key}"], meta[property="og:${key}"]`).attr('content');
-      if (metaContent) {
-        extracted[key] = this._coerceValue(metaContent, fieldSchema);
-        fieldsFound++;
-        continue;
-      }
 
-      // Try matching by common selectors based on field name
-      const commonSelectors = [
-        `[itemprop="${key}"]`,
-        `[data-${key}]`,
-        `.${key}`,
-        `#${key}`
-      ];
-
-      for (const sel of commonSelectors) {
-        const el = $(sel);
-        if (el.length > 0) {
-          if (isArrayField && el.length > 1) {
-            const values = el.map((_, item) => $(item).text().trim()).get().filter(Boolean);
-            if (values.length > 0) {
-              extracted[key] = values;
-              fieldsFound++;
-              break;
-            }
-          } else {
-            const rawValue = el.first().text().trim();
-            if (rawValue) {
-              extracted[key] = this._coerceValue(rawValue, fieldSchema);
-              fieldsFound++;
-              break;
-            }
-          }
-        }
-      }
+        // Try matching by common selectors based on field name
+        const commonSelectors = [
+          `[itemprop="${key}"]`,
+          `[data-${key}]`,
+          `.${key}`,
+          `#${key}`
+        ];
 
-      // Last resort: semantic element selectors for well-known field names
-      // (e.g. title -> <h1>/<title>) so common fields resolve without hints.
-      if (!(key in extracted)) {
-        const semanticSelectors = SEMANTIC_FIELD_SELECTORS[key.toLowerCase()];
-        if (semanticSelectors) {
-          for (const sel of semanticSelectors) {
-            const el = $(sel);
-            if (el.length === 0) continue;
+        for (const sel of commonSelectors) {
+          const el = $(sel);
+          if (el.length > 0) {
             if (isArrayField && el.length > 1) {
               const values = el.map((_, item) => $(item).text().trim()).get().filter(Boolean);
               if (values.length > 0) {
@@ -287,6 +298,36 @@ export class ExtractStructuredTool {
             }
           }
         }
+
+        // Last resort: semantic element selectors for well-known field names
+        // (e.g. title -> <h1>/<title>) so common fields resolve without hints.
+        if (!(key in extracted)) {
+          const semanticSelectors = SEMANTIC_FIELD_SELECTORS[key.toLowerCase()];
+          if (semanticSelectors) {
+            for (const sel of semanticSelectors) {
+              const el = $(sel);
+              if (el.length === 0) continue;
+              if (isArrayField && el.length > 1) {
+                const values = el.map((_, item) => $(item).text().trim()).get().filter(Boolean);
+                if (values.length > 0) {
+                  extracted[key] = values;
+                  fieldsFound++;
+                  break;
+                }
+              } else {
+                const rawValue = el.first().text().trim();
+                if (rawValue) {
+                  extracted[key] = this._coerceValue(rawValue, fieldSchema);
+                  fieldsFound++;
+                  break;
+                }
+              }
+            }
+          }
+        }
+      } catch (_fieldError) {
+        // Invalid selector for this key — skip the field, keep going.
+        continue;
       }
     }
 

package/src/tools/extract/extractWithLlm.js

@@ -340,7 +340,9 @@ async function callOllama({ model, systemMessage, userMessage, maxTokens, schema
     ],
     stream: false,
     options: { num_predict: maxTokens, temperature: 0 },
-    format: (schema && Object.keys(schema).length > 0) ? schema : 'json'
+    // Normalize the same way the Anthropic branch does — Ollama's `format`
+    // needs a valid JSON Schema, not a raw flat field->type-hint map.
+    format: (schema && Object.keys(schema).length > 0) ? buildInputSchema(schema) : 'json'
   };
 
   let response;
@@ -398,6 +400,12 @@ async function callLLM({ provider, apiKey, model, systemMessage, userMessage, ma
 export class ExtractWithLlm {
   constructor(config = {}) {
     this.config = config;
+    this._mcpServer = null;
+  }
+
+  /** D1.3: Wire MCP server so the sampling fallback can reach the client. */
+  setMcpServer(mcpServer) {
+    this._mcpServer = mcpServer;
   }
 
   /**
@@ -479,7 +487,7 @@ export class ExtractWithLlm {
       if (providerParam === 'auto' || providerParam === 'ollama') {
         try {
           const SamplingClient = await getSamplingClient();
-          const samplingClient = new SamplingClient();
+          const samplingClient = new SamplingClient({ mcpServer: this._mcpServer });
           const { text: sampledText } = await samplingClient.complete(
             `${systemMessage}\n\n${userMessage}`,
             { maxTokens }

package/src/tools/extract/processDocument.js

@@ -4,6 +4,8 @@
  */
 
 import { z } from 'zod';
+import fs from 'fs/promises';
+import path from 'path';
 import { PDFProcessor } from '../../core/processing/PDFProcessor.js';
 import { ContentProcessor } from '../../core/processing/ContentProcessor.js';
 import { BrowserProcessor } from '../../core/processing/BrowserProcessor.js';
@@ -18,7 +20,6 @@ const ProcessDocumentSchema = z.object({
     // PDF processing options
     extractText: z.boolean().default(true),
     extractMetadata: z.boolean().default(true),
-    password: z.string().optional(),
     maxPages: z.number().min(1).max(500).default(100),
     // C3: extract a specific 1-based, inclusive page range from a PDF
     pageRange: z.object({
@@ -148,6 +149,9 @@ export class ProcessDocumentTool {
       if (sourceType.includes('pdf')) {
         result.documentType = 'pdf';
         await this.processPDFDocument(result, source, sourceType, options);
+      } else if (sourceType === 'file') {
+        result.documentType = 'file';
+        await this.processLocalFileDocument(result, source, options);
       } else {
         result.documentType = 'web';
         await this.processWebDocument(result, source, options);
@@ -200,7 +204,6 @@ export class ProcessDocumentTool {
       options: {
         extractText: options.extractText,
         extractMetadata: options.extractMetadata,
-        password: options.password,
         maxPages: options.maxPages,
         ...(options.pageRange ? { pageRange: options.pageRange } : {})
       }
@@ -293,10 +296,52 @@ export class ProcessDocumentTool {
 
     result.title = pageTitle;
 
+    await this.processFetchedHtml(result, html, source, options);
+  }
+
+  /**
+   * Process a local non-PDF file (sourceType 'file'): read it from disk and
+   * run it through the same content-processing pipeline used for web pages.
+   * @param {Object} result - Result object to populate
+   * @param {string} source - Local file path
+   * @param {Object} options - Processing options
+   * @returns {Promise<void>}
+   */
+  async processLocalFileDocument(result, source, options) {
+    const resolvedPath = path.resolve(source);
+    let content;
+    try {
+      content = await fs.readFile(resolvedPath, 'utf-8');
+    } catch (error) {
+      throw new Error(`Failed to read local file: ${error.message}`);
+    }
+
+    const isHtml = /\.html?$/i.test(resolvedPath) || /<html[\s>]/i.test(content.slice(0, 1000));
+    const html = isHtml
+      ? content
+      : `<html><body><pre>${content.replace(/[&<>]/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;' }[c]))}</pre></body></html>`;
+
+    result.title = isHtml ? this.extractTitleFromHTML(content) : null;
+
+    // No `url` to pass here — ContentProcessor's url field is a validated
+    // absolute URL, and a filesystem path doesn't qualify.
+    await this.processFetchedHtml(result, html, undefined, options);
+  }
+
+  /**
+   * Run the shared content-processing pipeline (Readability/boilerplate
+   * fallback, metadata, structured data) over already-fetched HTML.
+   * @param {Object} result - Result object to populate
+   * @param {string} html - HTML content
+   * @param {string|undefined} url - Source URL (omitted for local files)
+   * @param {Object} options - Processing options
+   * @returns {Promise<void>}
+   */
+  async processFetchedHtml(result, html, url, options) {
     // Step 2: Process content with ContentProcessor
     const processingResult = await this.contentProcessor.processContent({
       html,
-      url: source,
+      url,
       options: {
         extractStructuredData: options.extractStructuredData,
         calculateReadabilityScore: true,
@@ -449,13 +494,16 @@ export class ProcessDocumentTool {
    * @returns {Promise<boolean>} - Whether JavaScript is needed
    */
   async shouldUseJavaScript(url) {
+    // Strip the fragment first: an ordinary document anchor (e.g. #install)
+    // isn't a signal for client-side routing, and anchoring the path pattern
+    // to full segments avoids false positives like "/apple" or "/spaces".
+    const urlWithoutFragment = url.split('#')[0];
     const jsIndicators = [
-      /\/(app|spa|dashboard|admin)/,
-      /#/,
+      /\/(app|spa|dashboard|admin)(\/|$)/,
       /\.(js|jsx|ts|tsx)$/
     ];
 
-    return jsIndicators.some(pattern => pattern.test(url));
+    return jsIndicators.some(pattern => pattern.test(urlWithoutFragment));
   }
 
   /**

package/src/tools/extract/summarizeContent.js

@@ -76,6 +76,12 @@ const SummarizeContentResult = z.object({
 export class SummarizeContentTool {
   constructor() {
     this.contentAnalyzer = new ContentAnalyzer();
+    this._mcpServer = null;
+  }
+
+  /** D1.3: Wire MCP server so the sampling fallback can reach the client. */
+  setMcpServer(mcpServer) {
+    this._mcpServer = mcpServer;
   }
 
   /**
@@ -211,7 +217,7 @@ export class SummarizeContentTool {
   async _abstractiveSummaryViaSampling(text, extractiveSummary, summaryLength) {
     try {
       const SamplingClient = await getSamplingClient();
-      const client = new SamplingClient();
+      const client = new SamplingClient({ mcpServer: this._mcpServer });
 
       const lengthGuide = {
         short: '1-2 sentences',

package/src/tools/llmstxt/generateLLMsTxt.js

@@ -53,11 +53,6 @@ export class GenerateLLMsTxtTool {
       userAgent: options.userAgent || 'LLMs.txt-Generator/1.0',
       ...options
     };
-
-    this.analyzer = new LLMsTxtAnalyzer({
-      timeout: this.options.timeout,
-      userAgent: this.options.userAgent
-    });
   }
 
   async execute(params) {
@@ -71,8 +66,15 @@ export class GenerateLLMsTxtTool {
       const baseUrl = getBaseUrl(url);
       
       // Step 1: Comprehensive Website Analysis
+      // A fresh analyzer is constructed per call — LLMsTxtAnalyzer keeps mutable
+      // per-analysis state on `this.analysis`, so a shared instance would let
+      // concurrent (or successive) calls cross-contaminate results.
       logger.info(`Analyzing website: ${baseUrl}`);
-      const analysis = await this.analyzer.analyzeWebsite(url, analysisOptions);
+      const analyzer = new LLMsTxtAnalyzer({
+        timeout: this.options.timeout,
+        userAgent: this.options.userAgent
+      });
+      const analysis = await analyzer.analyzeWebsite(url, analysisOptions);
 
       // Step 2: Generate LLMs.txt Content
       const llmsTxtContent = this.generateLLMsTxt(analysis, outputOptions, complianceLevel);