crawlforge-mcp-server 5.2.9 → 5.3.0

package/src/tools/extract/extractContent.js

@@ -9,6 +9,8 @@ import { BrowserProcessor } from '../../core/processing/BrowserProcessor.js';
 import { HTMLCleaner, ContentQualityAssessor } from '../../utils/contentUtils.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js'; // D3.1
 import { safeFetch } from '../../utils/ssrfGuard.js';
+import { preflightFetch } from '../../utils/robotsGate.js';
+import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
 
 const ExtractContentSchema = z.object({
   url: z.string().url(),
@@ -17,6 +19,8 @@ const ExtractContentSchema = z.object({
   // deep_research). Without this field Zod stripped it and the tool always
   // re-fetched the URL — silently defeating any pre-fetched-HTML caller.
   html: z.string().optional(),
+  respect_robots: z.boolean().optional(),
+  user_agent: z.string().optional(),
   options: z.object({
     // Content extraction options
     useReadability: z.boolean().default(true),
@@ -130,7 +134,7 @@ export class ExtractContentTool {
     
     try {
       const validated = ExtractContentSchema.parse(params);
-      const { url, html: providedHtml, options } = validated;
+      const { url, html: providedHtml, options, respect_robots, user_agent } = validated;
 
       const result = {
         url,
@@ -147,6 +151,15 @@ export class ExtractContentTool {
         html = providedHtml;
         pageTitle = this.extractTitleFromHTML(html);
       } else {
+      // Robots gate before either fetch path — the browser render is a request
+      // to the target too.
+      const gate = await preflightFetch(url, {
+        respectRobots: respect_robots,
+        userAgent: user_agent,
+        tool: 'extract_content'
+      });
+      if (gate.warnings.length > 0) result.warnings = gate.warnings;
+
       const shouldUseJavaScript = options.requiresJavaScript || await this.shouldUseJavaScript(url);
 
       if (shouldUseJavaScript) {
@@ -171,13 +184,14 @@ export class ExtractContentTool {
       } else {
         // Simple HTTP fetch
         const response = await safeFetch(url, {
-          headers: {
-            'User-Agent': 'Mozilla/5.0 (compatible; MCP-WebScraper/3.0; Enhanced-Content-Extractor)'
-          },
+          headers: { ...gate.headers },
           signal: AbortSignal.timeout(15000)
         });
 
         if (!response.ok) {
+          if (response.status === 429 || response.status === 503) {
+            noteRetryAfter(url, response.headers.get('retry-after'));
+          }
           throw new Error(`HTTP ${response.status}: ${response.statusText}`);
         }
 

package/src/tools/extract/extractStructured.js

@@ -8,12 +8,9 @@ import { z } from 'zod';
 import { ElicitationHelper } from '../../core/ElicitationHelper.js'; // D1.4
 import { load } from 'cheerio';
 import { LLMManager } from '../../core/llm/LLMManager.js';
-import { createRequire } from 'module';
-
-const _require = createRequire(import.meta.url);
-const _pkg = _require('../../../package.json');
-const CRAWLFORGE_UA = `CrawlForge/${_pkg.version} (+https://crawlforge.dev)`;
-import { fetchAndParse } from './_fetchAndParse.js';
+import { CRAWLFORGE_USER_AGENT } from '../../utils/fetchIdentity.js';
+import { fetchAndParse, flattenBodyText } from './_fetchAndParse.js';
+import { extractMainContent } from '../scrape/_mainContent.js';
 
 // Semantic element selectors for well-known field names, tried as a last
 // resort in the CSS fallback so common fields (e.g. "title") still resolve when
@@ -32,6 +29,41 @@ const SEMANTIC_FIELD_SELECTORS = {
   price: ['[itemprop="price"]', '[class*="price"]']
 };
 
+/**
+ * Main-content text for the LLM, or '' when there is none to fall back from.
+ *
+ * Whole-body text hands the model the page chrome, and it answers from the
+ * first heading-shaped string it sees: the Cloudflare blog post returned
+ * headline "Skip to content". This is the same Readability pass `scrape` runs.
+ *
+ * @param {import('cheerio').CheerioAPI} $ - parsed document from fetchAndParse
+ * @param {string} html
+ * @param {string} url
+ * @returns {string}
+ */
+function mainContentText($, html, url) {
+  // fetchAndParse returns an empty $ for text/plain and JSON bodies. Those are
+  // not markup and must not go through Readability.
+  if ($('body').children().length === 0) return '';
+  const { html: mainHtml, title } = extractMainContent(html, url);
+  if (!mainHtml) return '';
+  // Readability strips the article's own heading out of the content it
+  // returns, so the title has to be put back: without it the IANA page's main
+  // text never says "Example Domains" and the model answers from the body.
+  return [title, flattenBodyText(load(mainHtml))].filter(Boolean).join('\n');
+}
+
+/**
+ * Missing, null, blank string or empty array — what "the extraction did not
+ * fill this field in" looks like across every extraction method.
+ */
+function isEmptyValue(value) {
+  if (value === undefined || value === null) return true;
+  if (typeof value === 'string') return value.trim() === '';
+  if (Array.isArray(value)) return value.length === 0;
+  return false;
+}
+
 const ExtractStructuredSchema = z.object({
   url: z.string().url(),
   schema: z.object({
@@ -45,14 +77,16 @@ const ExtractStructuredSchema = z.object({
     apiKey: z.string().optional()
   }).optional(),
   fallbackToSelectors: z.boolean().optional().default(true),
-  selectorHints: z.record(z.string()).optional()
+  selectorHints: z.record(z.string()).optional(),
+  respect_robots: z.boolean().optional(),
+  user_agent: z.string().optional()
 });
 
 export class ExtractStructuredTool {
   constructor(options = {}) {
     this.llmManager = null;
     this.llmConfig = options.llmConfig || {};
-    this.userAgent = CRAWLFORGE_UA;
+    this.userAgent = CRAWLFORGE_USER_AGENT;
     // D1.4: Elicitation helper
     this._elicitation = new ElicitationHelper({});
   }
@@ -102,10 +136,14 @@ export class ExtractStructuredTool {
 
     try {
       const validated = ExtractStructuredSchema.parse(params);
-      const { url, schema, prompt, llmConfig, fallbackToSelectors, selectorHints } = validated;
+      const { url, schema, prompt, llmConfig, fallbackToSelectors, selectorHints, respect_robots, user_agent } = validated;
 
       // Step 1: Fetch and parse — shared helper strips scripts/styles/iframes/svgs
-      const { html, $, textContent } = await fetchAndParse(url, { userAgent: this.userAgent });
+      const { html, $, textContent, warnings } = await fetchAndParse(url, {
+        userAgent: user_agent || this.userAgent,
+        respectRobots: respect_robots,
+        tool: 'extract_structured'
+      });
 
       // Step 3: Try LLM extraction first
       let extractionResult = null;
@@ -120,7 +158,7 @@ export class ExtractStructuredTool {
         // local Ollama was never used.
         llmAvailable = await llm.ready();
         if (llmAvailable) {
-          const result = await llm.extractStructured(textContent, schema, {
+          const result = await llm.extractStructured(mainContentText($, html, url) || textContent, schema, {
             prompt: prompt || '',
             maxContentLength: 6000
           });
@@ -155,6 +193,7 @@ export class ExtractStructuredTool {
           );
           if (!proceed) {
             return {
+              success: false,
               url,
               data: {},
               extraction_method: 'none',
@@ -185,22 +224,37 @@ export class ExtractStructuredTool {
         extractionNotes.push(`LLM extraction failed: ${llmErrorMessage}`);
       }
 
+      // A required field that came back missing or empty is a failed
+      // extraction, not a successful one carrying a note: surface it at the
+      // top level so a caller checking `success` sees it without reaching
+      // into `validation`.
+      const missingRequired = (schema.required || []).filter(
+        (field) => isEmptyValue((extractionResult.data || {})[field])
+      );
+      const failedRequired = extractionResult.valid !== true && missingRequired.length > 0;
+
       return {
+        success: !failedRequired,
         url,
         data: extractionResult.data || {},
         extraction_method: extractionMethod,
         confidence,
         schema_used: schema,
         processingTime: Date.now() - startTime,
+        ...(failedRequired
+          ? { error: `Required field(s) missing or empty: ${missingRequired.join(', ')}` }
+          : {}),
         validation: {
           valid: extractionResult.valid || false,
           errors: extractionResult.validationErrors || []
         },
-        extractionNotes
+        extractionNotes,
+        ...(warnings?.length ? { warnings } : {})
       };
 
     } catch (error) {
       return {
+        success: false,
         url: params.url || 'unknown',
         data: {},
         extraction_method: 'none',

package/src/tools/extract/extractWithLlm.js

@@ -255,6 +255,27 @@ function looksLikeSchemaEcho(parsed, schema) {
   return declarations.length === returned.length;
 }
 
+/**
+ * Detect output that parsed cleanly but carries nothing taken from the page —
+ * `{}`, `[]`, or a structure whose every leaf is null/blank. A small model that
+ * loses the instruction on a long input answers with `{}` and 2 output tokens,
+ * which would otherwise reach the caller as a successful extraction of nothing.
+ *
+ * Recurses through objects and arrays. Numbers and booleans are data at any
+ * depth, so `{count: 0}` and `{found: false}` are real results, not emptiness.
+ *
+ * @param {*} parsed - Parsed LLM output (or any value inside it)
+ * @returns {boolean}
+ */
+function hasNoExtractableData(parsed) {
+  if (parsed === null || parsed === undefined) return true;
+  if (typeof parsed === 'string') return parsed.trim() === '';
+  if (typeof parsed !== 'object') return false;
+  // Object.values covers arrays too; an empty object/array vacuously satisfies
+  // every(), which is the answer we want.
+  return Object.values(parsed).every(hasNoExtractableData);
+}
+
 /**
  * Validate parsed output against the schema hint.
  * @returns {{ valid: boolean, errors: string[] }}
@@ -462,6 +483,8 @@ export class ExtractWithLlm {
    * @param {string}  [params.provider]  - 'openai' | 'anthropic' | 'auto'
    * @param {string}  [params.model]     - Override default model
    * @param {number}  [params.maxTokens] - Max output tokens (default 4096)
+   * @param {boolean} [params.respect_robots] - Per-request robots.txt override
+   * @param {string}  [params.user_agent]     - Per-request identity override
    * @returns {Promise<Object>}
    */
   async execute(params) {
@@ -472,7 +495,9 @@ export class ExtractWithLlm {
       schema,
       provider: providerParam = 'auto',
       model: modelParam,
-      maxTokens = 4096
+      maxTokens = 4096,
+      respect_robots,
+      user_agent
     } = params;
 
     // Validate: exactly one of url or content must be provided
@@ -505,10 +530,16 @@ export class ExtractWithLlm {
 
     // Step 1: Get text to extract from
     let text;
+    let fetchWarnings = [];
     try {
       if (url) {
-        const { textContent } = await fetchAndParse(url);
+        const { textContent, warnings } = await fetchAndParse(url, {
+          respectRobots: respect_robots,
+          userAgent: user_agent,
+          tool: 'extract_with_llm'
+        });
         text = textContent;
+        fetchWarnings = warnings || [];
       } else {
         text = content;
       }
@@ -550,8 +581,8 @@ export class ExtractWithLlm {
     }
 
     // Step 3: Parse JSON; retry once with a stricter prompt if the response is
-    // unusable. "Unusable" covers both unparseable output and a schema echo —
-    // the latter parses cleanly but contains no page data at all.
+    // unusable. "Unusable" covers unparseable output, a schema echo and an
+    // empty result — the latter two parse cleanly but contain no page data.
     let parsed = null;
     let unusableReason = null;
     try {
@@ -559,6 +590,9 @@ export class ExtractWithLlm {
       if (looksLikeSchemaEcho(parsed, schema)) {
         parsed = null;
         unusableReason = 'echoed the output schema instead of extracting data from the page';
+      } else if (hasNoExtractableData(parsed)) {
+        parsed = null;
+        unusableReason = 'contained no data from the page — every field was empty';
       }
     } catch (_parseErr) {
       unusableReason = 'was not valid JSON';
@@ -606,6 +640,18 @@ export class ExtractWithLlm {
           raw: JSON.stringify(parsed).slice(0, 500)
         };
       }
+
+      if (hasNoExtractableData(parsed)) {
+        // Same failure mode as the echo: well-formed JSON with nothing from the
+        // page in it. Returning it would read as a successful extraction.
+        return {
+          success: false,
+          error: `LLM (${model}) returned no data from the page after retry — every field was empty. ` +
+                 'The page text is likely too long or too noisy for this model — try a larger model ' +
+                 '(OLLAMA_DEFAULT_MODEL) or narrow the input with onlyMainContent.',
+          raw: JSON.stringify(parsed).slice(0, 500)
+        };
+      }
     }
 
     // C3: surface truncation metadata so callers know the input was clipped
@@ -620,6 +666,7 @@ export class ExtractWithLlm {
       result.truncated = true;
       result.original_length = original_length;
     }
+    if (fetchWarnings.length > 0) result.warnings = fetchWarnings;
     // C3: validate output against the schema hint (zod). Non-fatal — the data
     // is still returned; callers can inspect `valid`/`validationErrors`.
     if (schema && Object.keys(schema).length > 0) {

package/src/tools/extract/processDocument.js

@@ -12,10 +12,14 @@ import { BrowserProcessor } from '../../core/processing/BrowserProcessor.js';
 import { HTMLCleaner, ContentQualityAssessor } from '../../utils/contentUtils.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js'; // D3.1
 import { safeFetch } from '../../utils/ssrfGuard.js';
+import { preflightFetch } from '../../utils/robotsGate.js';
+import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
 
 const ProcessDocumentSchema = z.object({
   source: z.string().min(1),
   sourceType: z.enum(['url', 'pdf_url', 'file', 'pdf_file']).default('url'),
+  respect_robots: z.boolean().optional(),
+  user_agent: z.string().optional(),
   options: z.object({
     // PDF processing options
     extractText: z.boolean().default(true),
@@ -142,7 +146,7 @@ export class ProcessDocumentTool {
     
     try {
       const validated = ProcessDocumentSchema.parse(params);
-      const { source, sourceType, options } = validated;
+      const { source, sourceType, options, respect_robots, user_agent } = validated;
 
       const result = {
         source,
@@ -155,13 +159,13 @@ export class ProcessDocumentTool {
       // Determine document type and processing method
       if (sourceType.includes('pdf')) {
         result.documentType = 'pdf';
-        await this.processPDFDocument(result, source, sourceType, options);
+        await this.processPDFDocument(result, source, sourceType, options, { respect_robots, user_agent });
       } else if (sourceType === 'file') {
         result.documentType = 'file';
         await this.processLocalFileDocument(result, source, options);
       } else {
         result.documentType = 'web';
-        await this.processWebDocument(result, source, options);
+        await this.processWebDocument(result, source, options, { respect_robots, user_agent });
       }
 
       // Add statistics if requested
@@ -177,6 +181,15 @@ export class ProcessDocumentTool {
         );
       }
 
+      // Readability is populated once, here, for every source type, from the
+      // same Flesch implementation that backs
+      // qualityAssessment.metrics.readability — so the two fields on a
+      // response can never contradict each other.
+      if (result.content?.text) {
+        const { score, level, ...metrics } = ContentQualityAssessor.calculateSimpleReadability(result.content.text);
+        result.readabilityScore = { score, level, metrics };
+      }
+
       result.processingTime = Date.now() - startTime;
       result.success = true;
 
@@ -204,7 +217,17 @@ export class ProcessDocumentTool {
    * @param {Object} options - Processing options
    * @returns {Promise<void>}
    */
-  async processPDFDocument(result, source, sourceType, options) {
+  async processPDFDocument(result, source, sourceType, options, identity = {}) {
+    // A remote PDF is a fetch of the target like any other; a local file is not.
+    if (sourceType === 'pdf_url') {
+      const gate = await preflightFetch(source, {
+        respectRobots: identity.respect_robots,
+        userAgent: identity.user_agent,
+        tool: 'process_document'
+      });
+      if (gate.warnings.length > 0) result.warnings = gate.warnings;
+    }
+
     const pdfResult = await this.pdfProcessor.processPDF({
       source,
       sourceType: sourceType.replace('pdf_', ''),
@@ -251,14 +274,6 @@ export class ProcessDocumentTool {
         encrypted: pdfResult.metadata.encrypted
       };
     }
-
-    // Calculate readability score for text content
-    if (options.assessContentQuality && result.content.text) {
-      const readabilityScore = this.calculateReadabilityScore(result.content.text);
-      if (readabilityScore) {
-        result.readabilityScore = readabilityScore;
-      }
-    }
   }
 
   /**
@@ -268,11 +283,18 @@ export class ProcessDocumentTool {
    * @param {Object} options - Processing options
    * @returns {Promise<void>}
    */
-  async processWebDocument(result, source, options) {
-    // Step 1: Fetch content (with or without JavaScript rendering)
+  async processWebDocument(result, source, options, identity = {}) {
+    // Step 1: Fetch content (with or without JavaScript rendering).
+    // Robots gate before either path — the browser render is a request too.
     let html, pageTitle;
+    const gate = await preflightFetch(source, {
+      respectRobots: identity.respect_robots,
+      userAgent: identity.user_agent,
+      tool: 'process_document'
+    });
+    if (gate.warnings.length > 0) result.warnings = gate.warnings;
     const shouldUseJavaScript = options.requiresJavaScript || await this.shouldUseJavaScript(source);
-    
+
     if (shouldUseJavaScript) {
       console.error('Using browser rendering for JavaScript content...');
       const browserResult = await this.browserProcessor.processURL({
@@ -294,13 +316,14 @@ export class ProcessDocumentTool {
     } else {
       // Simple HTTP fetch
       const response = await safeFetch(source, {
-        headers: {
-          'User-Agent': 'Mozilla/5.0 (compatible; MCP-WebScraper/3.0; Document-Processor)'
-        },
+        headers: { ...gate.headers },
         signal: AbortSignal.timeout(15000)
       });
 
       if (!response.ok) {
+        if (response.status === 429 || response.status === 503) {
+          noteRetryAfter(source, response.headers.get('retry-after'));
+        }
         throw new Error(`HTTP ${response.status}: ${response.statusText}`);
       }
 
@@ -358,7 +381,9 @@ export class ProcessDocumentTool {
       url,
       options: {
         extractStructuredData: options.extractStructuredData,
-        calculateReadabilityScore: true,
+        // execute() computes readability once for every source type; asking
+        // ContentProcessor for its own score here would only duplicate it.
+        calculateReadabilityScore: false,
         removeBoilerplate: options.useReadability,
         preserveImageInfo: false,
         extractMetadata: true
@@ -413,12 +438,7 @@ export class ProcessDocumentTool {
       };
     }
 
-    // Step 5: Add readability score
-    if (processingResult.readability_score) {
-      result.readabilityScore = processingResult.readability_score;
-    }
-
-    // Step 6: Add structured data
+    // Step 5: Add structured data
     if (options.extractStructuredData && processingResult.structured_data) {
       result.structuredData = processingResult.structured_data;
     }
@@ -449,59 +469,6 @@ export class ProcessDocumentTool {
     };
   }
 
-  /**
-   * Calculate readability score
-   * @param {string} text - Text to analyze
-   * @returns {Object|null} - Readability score
-   */
-  calculateReadabilityScore(text) {
-    try {
-      const sentences = text.split(/[.!?]+/).filter(s => s.trim().length > 0);
-      const words = text.split(/\s+/).filter(w => w.length > 0);
-      
-      if (sentences.length === 0 || words.length === 0) {
-        return null;
-      }
-
-      const avgWordsPerSentence = words.length / sentences.length;
-      const avgCharsPerWord = text.replace(/\s/g, '').length / words.length;
-      
-      // Flesch Reading Ease Score approximation
-      const score = 206.835 - (1.015 * avgWordsPerSentence) - (84.6 * (avgCharsPerWord / 4.7));
-      const clampedScore = Math.max(0, Math.min(100, score));
-
-      return {
-        score: Math.round(clampedScore * 100) / 100,
-        level: this.getReadabilityLevel(clampedScore),
-        metrics: {
-          sentences: sentences.length,
-          words: words.length,
-          avgWordsPerSentence: Math.round(avgWordsPerSentence * 100) / 100,
-          avgCharsPerWord: Math.round(avgCharsPerWord * 100) / 100
-        }
-      };
-
-    } catch (error) {
-      console.warn('Readability calculation failed:', error.message);
-      return null;
-    }
-  }
-
-  /**
-   * Get readability level from score
-   * @param {number} score - Readability score
-   * @returns {string} - Readability level
-   */
-  getReadabilityLevel(score) {
-    if (score >= 90) return 'Very Easy';
-    if (score >= 80) return 'Easy';
-    if (score >= 70) return 'Fairly Easy';
-    if (score >= 60) return 'Standard';
-    if (score >= 50) return 'Fairly Difficult';
-    if (score >= 30) return 'Difficult';
-    return 'Very Difficult';
-  }
-
   /**
    * Determine if JavaScript rendering is needed
    * @param {string} url - URL to analyze

package/src/tools/extract/summarizeContent.js

@@ -73,6 +73,25 @@ const SummarizeContentResult = z.object({
   error: z.string().optional()
 });
 
+// Navigation chrome that some sites write with a trailing full stop, e.g. a
+// skip link rendered "Jump to content." Chrome without a terminator is already
+// caught by the punctuation test below; these are the only lines allowed to
+// pass it. Half were observed live in extract_text output (en/es Wikipedia,
+// MDN, BBC, GOV.UK); the rest are the same construction. Kept multilingual on
+// purpose so the exception does not quietly narrow a language-agnostic rule.
+const NAVIGATION_PHRASES = new Set([
+  'jump to content',
+  'jump to navigation',
+  'jump to search',
+  'skip to content',
+  'skip to main content',
+  'skip to navigation',
+  'skip to search',
+  'from wikipedia, the free encyclopedia',
+  'ir al contenido',
+  'de wikipedia, la enciclopedia libre'
+]);
+
 export class SummarizeContentTool {
   constructor() {
     this.contentAnalyzer = new ContentAnalyzer();
@@ -269,6 +288,17 @@ export class SummarizeContentTool {
    * mid-sentence survives: it is either punctuated, longer than the caps, or
    * protected by the size guard — if the strip would remove more than
    * min(600 chars, 20% of the text), nothing is stripped at all.
+   *
+   * One exception to the punctuation test: a line that matches NAVIGATION_PHRASES
+   * exactly — case-insensitively, ignoring trailing terminators — is stripped
+   * even though it is punctuated, so a skip link written "Jump to content." is
+   * caught. That exception has to be a list rather than a rule, because
+   * "Jump to content." and "It was cold." are identical on every feature this
+   * function can measure (≤ 60 chars, ≤ 8 words, a single terminator in final
+   * position, nothing punctuated in between). Anything general enough to catch
+   * the first eats the second, and dropping a summary's opening sentence is the
+   * worse failure. The cost is the usual one for a list: it catches the phrases
+   * it names and no others.
    * @param {string} text - Text to clean
    * @returns {string} - Text without leading navigation chrome
    */
@@ -287,7 +317,11 @@ export class SummarizeContentTool {
       }
       const wordCount = line.split(/\s+/).length;
       const hasSentencePunctuation = /[.!?…。！？]/.test(line);
-      const isBoilerplate = line.length <= 60 && wordCount <= 8 && !hasSentencePunctuation;
+      const isNavigationPhrase = NAVIGATION_PHRASES.has(
+        line.replace(/[.!?…。！？]+$/, '').trim().toLowerCase()
+      );
+      const isBoilerplate = line.length <= 60 && wordCount <= 8 &&
+        (!hasSentencePunctuation || isNavigationPhrase);
       if (!isBoilerplate) break;
 
       strippedChars += line.length;

package/src/tools/llmstxt/generateLLMsTxt.js

@@ -4,6 +4,8 @@ import { LLMsTxtAnalyzer } from '../../core/LLMsTxtAnalyzer.js';
 import { Logger } from '../../utils/Logger.js';
 import { getBaseUrl } from '../../utils/urlNormalizer.js';
 import { safeFetch } from '../../utils/ssrfGuard.js';
+import { resolveUserAgent } from '../../utils/fetchIdentity.js';
+import { preflightFetch } from '../../utils/robotsGate.js';
 
 const logger = new Logger('GenerateLLMsTxtTool');
 
@@ -52,7 +54,7 @@ export class GenerateLLMsTxtTool {
   constructor(options = {}) {
     this.options = {
       timeout: options.timeout || 30000,
-      userAgent: options.userAgent || 'LLMs.txt-Generator/1.0',
+      userAgent: resolveUserAgent(options.userAgent),
       ...options
     };
   }
@@ -86,7 +88,7 @@ export class GenerateLLMsTxtTool {
       // link (llmstxt.org) instead of boilerplate. Best-effort: null on
       // failure, in which case the generic fallbacks below apply.
       if (!outputOptions.robotsStyle) {
-        analysis.homePage = await this.fetchHomePageMetadata(baseUrl);
+        analysis.homePage = await this.fetchHomePageMetadata(baseUrl, analysisOptions.respectRobots);
       }
 
       // Step 2: Generate LLMs.txt Content
@@ -240,6 +242,14 @@ export class GenerateLLMsTxtTool {
     emitSection('Tools', flatten('tools'));
     emitSection('Navigation', flatten('navigation'));
 
+    // Pages the categorizer's keyword lists did not place — the site root and
+    // any section whose name it does not recognise. The analysis spent its
+    // page budget on these, so they belong in the guide; before this they were
+    // listed only when no named section matched at all, which silently dropped
+    // the homepage and whole sections (e.g. /specification) from sites that
+    // happen to also have a /docs.
+    emitSection('Pages', flatten('other'));
+
     // Fallback: if no categorized section produced output, list the raw
     // sitemap so llms.txt always carries a URL inventory. Must run BEFORE the
     // APIs section — an APIs entry alone used to set hasBody and suppress it.
@@ -355,13 +365,18 @@ export class GenerateLLMsTxtTool {
    * Fetch the site's homepage once and extract naming metadata for the spec
    * output. Best-effort: returns null on any failure.
    */
-  async fetchHomePageMetadata(baseUrl) {
+  async fetchHomePageMetadata(baseUrl, respectRobots) {
     const controller = new AbortController();
     const timeoutId = setTimeout(() => controller.abort(), Math.min(this.options.timeout, 10000));
     try {
+      const gate = await preflightFetch(baseUrl, {
+        respectRobots,
+        userAgent: this.options.userAgent,
+        tool: 'generate_llms_txt'
+      });
       const response = await safeFetch(baseUrl, {
         signal: controller.signal,
-        headers: { 'User-Agent': this.options.userAgent }
+        headers: { ...gate.headers }
       });
       if (!response.ok) return null;
       return this.extractHomePageMetadata(await response.text());

package/src/tools/research/deepResearch.js

@@ -5,6 +5,7 @@ import { ResearchOrchestrator } from '../../core/ResearchOrchestrator.js';
 import { getToolConfig } from '../../constants/config.js';
 import { Logger } from '../../utils/Logger.js';
 import { safeFetch } from '../../utils/ssrfGuard.js';
+import { identityHeaders } from '../../utils/fetchIdentity.js';
 
 /**
  * DeepResearchTool - MCP tool for conducting comprehensive multi-stage research
@@ -735,7 +736,7 @@ export class DeepResearchTool {
         method: 'POST',
         headers: {
           'Content-Type': 'application/json',
-          'User-Agent': 'MCP-WebScraper-DeepResearch/1.0',
+          ...identityHeaders({ role: 'webhook' }),
           ...webhook.headers
         },
         body: JSON.stringify(payload),