crawlforge-mcp-server 5.2.9 → 5.3.0

package/src/core/ActionExecutor.js

@@ -8,6 +8,7 @@ import BrowserProcessor from './processing/BrowserProcessor.js';
 import { EventEmitter } from 'events';
 import { createHash } from 'node:crypto';
 import { assertUrlAllowed } from '../utils/ssrfGuard.js';
+import { browserPreflight } from '../utils/robotsGate.js';
 
 // executeJavaScript hardening limits (only relevant when the deploy-time flag
 // ALLOW_JAVASCRIPT_EXECUTION=true is set; JS execution stays off by default).
@@ -104,6 +105,34 @@ const ScrollActionSchema = BaseActionSchema.extend({
   y: z.number().min(0).optional()
 });
 
+const SelectActionSchema = BaseActionSchema.extend({
+  type: z.literal('select'),
+  selector: z.string(),
+  // Playwright's string form of selectOption matches an <option> by its `value`
+  // OR its visible label, so one field covers both and the caller doesn't have
+  // to know which one the page uses. `values` selects several in a multi-select.
+  value: z.string().optional(),
+  values: z.array(z.string()).optional()
+}).refine(data => data.value !== undefined || (data.values && data.values.length > 0), {
+  message: 'Select action requires value or values'
+});
+
+const HoverActionSchema = BaseActionSchema.extend({
+  type: z.literal('hover'),
+  selector: z.string(),
+  force: z.boolean().default(false),
+  position: z.object({
+    x: z.number(),
+    y: z.number()
+  }).optional()
+});
+
+const NavigateActionSchema = BaseActionSchema.extend({
+  type: z.literal('navigate'),
+  url: z.string().url(),
+  waitUntil: z.enum(['load', 'domcontentloaded', 'networkidle', 'commit']).optional()
+});
+
 const ScreenshotActionSchema = BaseActionSchema.extend({
   type: z.literal('screenshot'),
   selector: z.string().optional(),
@@ -125,6 +154,9 @@ const ActionSchema = z.union([
   TypeActionSchema,
   PressActionSchema,
   ScrollActionSchema,
+  SelectActionSchema,
+  HoverActionSchema,
+  NavigateActionSchema,
   ScreenshotActionSchema,
   ExecuteJavaScriptActionSchema
 ]);
@@ -294,12 +326,14 @@ export class ActionExecutor extends EventEmitter {
         // context for non-stealth pages — createPage() gives each call its
         // own dedicated BrowserContext that is never tracked/closed
         // elsewhere, so leaving it open here leaks it until server shutdown.
-        // Stealth contexts are pooled/reused (see StealthBrowserManager) and
-        // must not be closed here.
+        // A stealth context belongs to StealthBrowserManager's pool, so it is
+        // released through the manager instead of closed directly here.
         if (page) {
-          const ctx = !browserOptions.stealthMode?.enabled ? page.context() : null;
-          try { await page.close(); } catch (_) { /* ignore close errors */ }
-          if (ctx) {
+          if (browserOptions.stealthMode?.enabled) {
+            await this.browserProcessor.releaseStealthPage(page);
+          } else {
+            const ctx = page.context();
+            try { await page.close(); } catch (_) { /* ignore close errors */ }
             try { await ctx.close(); } catch (_) { /* ignore close errors */ }
           }
         }
@@ -512,7 +546,7 @@ export class ActionExecutor extends EventEmitter {
       // "Action timeout".
       const backstopMs = timeout + ACTION_TIMEOUT_GRACE_MS;
       let backstopTimer;
-      const executionPromise = this.executeActionByType(page, action);
+      const executionPromise = this.executeActionByType(page, action, executionContext);
       const timeoutPromise = new Promise((_, reject) => {
         backstopTimer = setTimeout(
           () => reject(new Error(
@@ -665,9 +699,10 @@ export class ActionExecutor extends EventEmitter {
    * Execute action based on its type
    * @param {Page} page - Playwright page
    * @param {Object} action - Action configuration
+   * @param {Object} [executionContext] - Execution context (navigate reads its browserOptions)
    * @returns {Promise<any>} Action result
    */
-  async executeActionByType(page, action) {
+  async executeActionByType(page, action, executionContext) {
     switch (action.type) {
       case 'wait':
         return await this.executeWaitAction(page, action);
@@ -679,6 +714,12 @@ export class ActionExecutor extends EventEmitter {
         return await this.executePressAction(page, action);
       case 'scroll':
         return await this.executeScrollAction(page, action);
+      case 'select':
+        return await this.executeSelectAction(page, action);
+      case 'hover':
+        return await this.executeHoverAction(page, action);
+      case 'navigate':
+        return await this.executeNavigateAction(page, action, executionContext);
       case 'screenshot':
         return await this.executeScreenshotAction(page, action);
       case 'executeJavaScript':
@@ -923,6 +964,83 @@ export class ActionExecutor extends EventEmitter {
     };
   }
 
+  /**
+   * Execute select action on a <select> dropdown
+   * @param {Page} page - Playwright page
+   * @param {Object} action - Select action
+   * @returns {Promise<Object>} Select result
+   */
+  async executeSelectAction(page, action) {
+    const timeout = this.actionTimeout(action);
+    const values = action.values?.length ? action.values : [action.value];
+
+    const selected = await this.elementLocator(page, action.selector)
+      .selectOption(values, { timeout });
+
+    // Faceted-search dropdowns commonly submit the form on change, so let any
+    // navigation commit before the next action runs.
+    await this.settleAfterInteraction(page, timeout);
+
+    return {
+      selector: action.selector,
+      requested: values,
+      selected
+    };
+  }
+
+  /**
+   * Execute hover action
+   * @param {Page} page - Playwright page
+   * @param {Object} action - Hover action
+   * @returns {Promise<Object>} Hover result
+   */
+  async executeHoverAction(page, action) {
+    const timeout = this.actionTimeout(action);
+    const hoverOptions = { force: action.force, timeout };
+    if (action.position) {
+      hoverOptions.position = action.position;
+    }
+
+    await this.elementLocator(page, action.selector).hover(hoverOptions);
+
+    // No settle: a hover reveals a menu, it does not replace the document.
+    return {
+      selector: action.selector,
+      position: action.position
+    };
+  }
+
+  /**
+   * Execute navigate action - load a new URL in the running page.
+   *
+   * A navigate action is a fetch of a new URL, so it goes through the same gate
+   * order as the chain's initial load: SSRF, then blocklist/robots, then the
+   * navigation itself. Routing it here rather than straight at page.goto() is
+   * what stops a chain from using `navigate` to reach a URL the gate would have
+   * refused at initializePage.
+   * @param {Page} page - Playwright page
+   * @param {Object} action - Navigate action
+   * @param {Object} [executionContext] - Execution context (for browserOptions)
+   * @returns {Promise<Object>} Navigate result
+   */
+  async executeNavigateAction(page, action, executionContext) {
+    const timeout = this.actionTimeout(action);
+
+    await assertUrlAllowed(action.url, { resolveDns: true });
+    await this.assertRobotsAllowed(action.url, executionContext?.browserOptions);
+
+    await this.navigateToUrl(page, action.url, {
+      waitUntil: action.waitUntil,
+      timeout
+    });
+
+    return {
+      url: action.url,
+      finalUrl: page.url(),
+      waitUntil: action.waitUntil || 'domcontentloaded'
+    };
+  }
+
   /**
    * Execute screenshot action
    * @param {Page} page - Playwright page
@@ -1041,19 +1159,21 @@ export class ActionExecutor extends EventEmitter {
 
   /**
    * Navigate an existing page to a URL under the SSRF checks every load needs.
-   * Used for the initial load and again before each chain retry.
+   * Used for the initial load, again before each chain retry, and by the
+   * `navigate` action.
    * @param {Page} page - Playwright page
    * @param {string} url - URL to navigate to
+   * @param {{ waitUntil?: string, timeout?: number }} [options] - Navigation options
    * @returns {Promise<void>}
    */
-  async navigateToUrl(page, url) {
+  async navigateToUrl(page, url, options = {}) {
     // resolveDns:true because Playwright does its own DNS resolution, so
     // hostname-based checks alone would miss DNS-rebinding/private-IP targets.
     await assertUrlAllowed(url, { resolveDns: true });
 
     await page.goto(url, {
-      waitUntil: 'domcontentloaded',
-      timeout: 30000
+      waitUntil: options.waitUntil || 'domcontentloaded',
+      timeout: options.timeout || 30000
     });
 
     // Re-validate the landed URL: a redirect during navigation could have
@@ -1064,6 +1184,28 @@ export class ActionExecutor extends EventEmitter {
     }
   }
 
+  /**
+   * Platform blocklist (G7), robots.txt (G5) and politeness (G6) gate for a URL
+   * this executor is about to load.
+   *
+   * The gate is deliberately asked about the canonical CrawlForge product
+   * token: no `userAgent` is passed through, so a caller setting
+   * browserOptions.userAgent — or a stealth context presenting a randomized
+   * UA — still matches the same robots rules our own token is bound by.
+   * Matching robots as whatever identity the caller asked us to wear would let
+   * browser traffic slip our own rules, which is the hole this gate closes.
+   * @param {string} url - URL about to be loaded
+   * @param {Object} [browserOptions] - Browser options (`respectRobots` override)
+   * @returns {Promise<void>}
+   * @throws {BlockedHostError|RobotsDisallowedError}
+   */
+  async assertRobotsAllowed(url, browserOptions = {}) {
+    await browserPreflight(url, {
+      respectRobots: browserOptions?.respectRobots,
+      tool: 'scrape_with_actions'
+    });
+  }
+
   /**
    * Initialize page with browser options (supports stealth mode)
    * @param {string} url - URL to navigate to
@@ -1076,6 +1218,12 @@ export class ActionExecutor extends EventEmitter {
     // hostname-based checks alone would miss DNS-rebinding/private-IP targets.
     await assertUrlAllowed(url, { resolveDns: true });
 
+    // Then the compliance gate — before the browser launches, so a blocked host
+    // or a disallowed path never costs a Chromium process. preflightFetch is
+    // deliberately not used here: its identity/signature headers belong on an
+    // HTTP fetch, not on a browser context.
+    await this.assertRobotsAllowed(url, browserOptions);
+
     const isStealth = !!browserOptions.stealthMode?.enabled;
 
     // Use the enhanced BrowserProcessor initialization that supports stealth mode
@@ -1109,11 +1257,15 @@ export class ActionExecutor extends EventEmitter {
       // Any failure between page creation and return (navigation, SSRF
       // re-check, stealth challenge handling) must not leak the page it
       // already created — close it, and its dedicated context for
-      // non-stealth pages (stealth contexts are pooled/reused elsewhere),
-      // before rethrowing.
-      const ctx = !isStealth ? page.context() : null;
-      await page.close().catch(() => {});
-      if (ctx) await ctx.close().catch(() => {});
+      // non-stealth pages, before rethrowing. A stealth context goes back to
+      // the manager's pool the same way it does after a successful chain.
+      if (isStealth) {
+        await this.browserProcessor.releaseStealthPage(page);
+      } else {
+        const ctx = page.context();
+        await page.close().catch(() => {});
+        await ctx.close().catch(() => {});
+      }
       throw error;
     }
   }

package/src/core/AlertNotificationSystem.js

@@ -6,6 +6,7 @@
 import { EventEmitter } from 'events';
 // Using native fetch (Node.js 18+)
 import crypto from 'crypto';
+import { identityHeaders } from '../utils/fetchIdentity.js';
 
 export class AlertNotificationSystem extends EventEmitter {
   constructor(options = {}) {
@@ -125,7 +126,7 @@ export class AlertNotificationSystem extends EventEmitter {
       // Generate signature if secret provided
       const requestHeaders = {
         'Content-Type': 'application/json',
-        'User-Agent': 'CrawlForge-AlertSystem/3.0',
+        ...identityHeaders({ role: 'alerts' }),
         ...headers
       };
       

package/src/core/AuthManager.js

@@ -541,8 +541,10 @@ class AuthManager {
    * (crawlforge-website/src/lib/credits.ts TOOL_CREDIT_COSTS).
    *
    * @param {string} tool
+   * @param {object} [params] — the call's params; only consulted by the tools
+   *   whose cost depends on what was asked for (see stealth_mode below)
    */
-  getToolCost(tool) {
+  getToolCost(tool, params) {
     const costs = {
       // 1 credit
       fetch_url: 1,
@@ -598,6 +600,17 @@ class AuthManager {
       return 0;
     }
 
+    // stealth_mode bills per operation, and only some of its operations launch
+    // a browser. At a flat 5 the ordinary create_context → create_page →
+    // cleanup sequence cost 15 credits to render one page. Browser work keeps
+    // the published price; the bookkeeping operations, which touch no browser,
+    // cost 1. An unknown or absent operation falls through to the flat 5 — the
+    // published price is the ceiling, never the floor.
+    if (tool === 'stealth_mode') {
+      const bookkeepingOps = new Set(['configure', 'enable', 'disable', 'get_stats', 'cleanup']);
+      if (bookkeepingOps.has(params?.operation)) return 1;
+    }
+
     return costs[tool] ?? 1;
   }
 
@@ -642,6 +655,11 @@ class AuthManager {
       case 'extract_with_llm':
         note = 'External LLM API call billed by your LLM provider, separate from the credit cost.';
         break;
+      case 'stealth_mode':
+        note = projected === 1
+          ? 'Bookkeeping operation — launches no browser.'
+          : 'Browser operation. configure/enable/disable/get_stats/cleanup cost 1 credit each.';
+        break;
       case 'serp_rank':
         note = projected === 0
           ? 'DataForSEO not configured — no-op, no credits charged. Set DATAFORSEO_LOGIN/PASSWORD to enable.'

package/src/core/ChangeTracker.js

@@ -1212,27 +1212,55 @@ export class ChangeTracker extends EventEmitter {
     return 'text_change';
   }
   
+  /**
+   * Count text-level changes for the summary. word_diff and line_diff describe
+   * the same edit at two granularities, so summing them would double-count —
+   * take the word diff, falling back to the line diff (a whitespace-only edit
+   * produces only the latter). Entries capDiffPayload dropped still count, or a
+   * large diff would report the cap instead of the real number.
+   */
+  countTextChanges(textChanges = []) {
+    const diff = textChanges.find(c => c.type === 'word_diff')
+      || textChanges.find(c => c.type === 'line_diff');
+
+    if (!diff) return 0;
+
+    return diff.changes.reduce(
+      (count, part) => count + (part.added || part.removed ? 1 : part.omittedEntries || 0),
+      0
+    );
+  }
+
   generateChangeSummary(changeAnalysis, significance) {
     const { addedElements, removedElements, modifiedElements, similarity } = changeAnalysis;
-    
-    const total = addedElements.length + removedElements.length + modifiedElements.length;
-    
+
+    // Text-only changes were left out of the total, so a compare whose entire
+    // diff was textual (a rotated UUID, an edited paragraph) reported
+    // totalChanges:0 beside a populated details.textChanges, and anything
+    // alerting on the counters never fired. Counted raw, like the three
+    // element counters beside it: all four report what the diff holds, and
+    // hasChanges alone reports whether it cleared the significance threshold.
+    const textChanges = this.countTextChanges(changeAnalysis.textChanges);
+
+    const total = addedElements.length + removedElements.length + modifiedElements.length + textChanges;
+
     return {
       totalChanges: total,
       contentSimilarity: Math.round(similarity * 100),
       added: addedElements.length,
       removed: removedElements.length,
       modified: modifiedElements.length,
+      textChanges,
       // Sub-threshold text noise (a rotating session token, a base64 timestamp)
-      // still lands in textChanges, so the description read "Text content
-      // changed" on a compare that reported hasChanges:false and
+      // still lands in details.textChanges, so the description read "Text
+      // content changed" on a compare that reported hasChanges:false and
       // totalChanges:0. Defer to the verdict the caller is given.
       changeDescription: significance === 'none'
         ? 'No significant changes detected'
         : this.generateChangeDescription(changeAnalysis)
     };
   }
-  
+
   generateChangeDescription(changeAnalysis) {
     const { addedElements, removedElements, modifiedElements, textChanges } = changeAnalysis;
     

package/src/core/LLMsTxtAnalyzer.js

@@ -5,9 +5,17 @@ import { CrawlDeepTool } from '../tools/crawl/crawlDeep.js';
 import { normalizeUrl, getBaseUrl } from '../utils/urlNormalizer.js';
 import { Logger } from '../utils/Logger.js';
 import { safeFetch } from '../utils/ssrfGuard.js';
+import { resolveUserAgent } from '../utils/fetchIdentity.js';
+import { preflightFetch } from '../utils/robotsGate.js';
+import { noteRetryAfter } from '../utils/hostRateLimiter.js';
 
 const logger = new Logger('LLMsTxtAnalyzer');
 
+// How many URLs to gather before ranking them down to maxPages. Large enough
+// that a mid-size documentation site's top-level sections are all represented
+// in the pool, small enough not to walk a big sitemap index end to end.
+const CANDIDATE_POOL = 500;
+
 /**
  * LLMsTxtAnalyzer - Comprehensive website analysis for LLMs.txt generation
  * 
@@ -25,7 +33,7 @@ export class LLMsTxtAnalyzer {
       maxDepth: options.maxDepth || 3,
       maxPages: options.maxPages || 100,
       timeout: options.timeout || 30000,
-      userAgent: options.userAgent || 'LLMs.txt-Analyzer/1.0',
+      userAgent: resolveUserAgent(options.userAgent),
       respectRobots: options.respectRobots !== false,
       detectAPIs: options.detectAPIs !== false,
       analyzeContent: options.analyzeContent !== false,
@@ -136,11 +144,16 @@ export class LLMsTxtAnalyzer {
     logger.info('Analyzing site structure...');
 
     try {
-      // Get comprehensive site map
+      // Get comprehensive site map. map_site truncates in sitemap document
+      // order, so asking it for exactly maxPages returns whichever section the
+      // sitemap happens to list first — the homepage and the docs/spec entry
+      // points never arrive. Ask for a larger candidate pool instead (the
+      // sitemap is downloaded whole either way, so a site with one sitemap pays
+      // no extra request), then keep the maxPages that describe the site.
       const siteMap = await this.mapSiteTool.execute({
         url,
         include_sitemap: true,
-        max_urls: this.options.maxPages,
+        max_urls: Math.max(this.options.maxPages, CANDIDATE_POOL),
         group_by_path: true,
         include_metadata: true
       });
@@ -154,17 +167,23 @@ export class LLMsTxtAnalyzer {
         respect_robots: this.options.respectRobots
       });
 
+      // group_by_path:true returns {section: [...]}; flatten before ranking.
+      const candidates = Array.isArray(siteMap.urls) ? siteMap.urls :
+                         (siteMap.urls && typeof siteMap.urls === 'object' ? Object.values(siteMap.urls).flat() : []);
+      const pages = this.prioritizeUrls(candidates, getBaseUrl(url), this.options.maxPages);
+
       this.analysis.structure = {
-        siteMap: siteMap.site_map,
-        totalPages: siteMap.total_urls,
-        sections: this.categorizeSections(siteMap.urls),
+        // Every field describes the same selected pages, not the wider pool.
+        siteMap: this.mapSiteTool.generateSiteMap(pages),
+        totalPages: pages.length,
+        sections: this.categorizeSections(pages),
         navigation: this.analyzeNavigation(crawlResult.pages),
-        hierarchy: this.buildHierarchy(siteMap.urls),
+        hierarchy: this.buildHierarchy(pages),
         robotsTxt: await this.fetchRobotsTxt(url),
-        sitemap: siteMap.urls || []
+        sitemap: pages
       };
 
-      logger.info(`Analyzed ${siteMap.total_urls} pages in site structure`);
+      logger.info(`Selected ${pages.length} of ${siteMap.total_urls} discovered pages for the site structure`);
 
     } catch (error) {
       logger.error(`Site structure analysis failed: ${error.message}`);
@@ -441,18 +460,24 @@ export class LLMsTxtAnalyzer {
 
   async fetchWithTimeout(url, options = {}) {
     const { timeout = this.options.timeout } = options;
+    const gate = await preflightFetch(url, {
+      respectRobots: this.options.respectRobots,
+      userAgent: this.options.userAgent,
+      tool: 'generate_llms_txt'
+    });
     const controller = new AbortController();
     const timeoutId = setTimeout(() => controller.abort(), timeout);
 
     try {
       const response = await safeFetch(url, {
         signal: controller.signal,
-        headers: {
-          'User-Agent': this.options.userAgent
-        },
+        headers: { ...gate.headers },
         ...options
       });
       clearTimeout(timeoutId);
+      if (response.status === 429 || response.status === 503) {
+        noteRetryAfter(url, response.headers.get('retry-after'));
+      }
       return response;
     } catch (error) {
       clearTimeout(timeoutId);
@@ -473,6 +498,63 @@ export class LLMsTxtAnalyzer {
     return null;
   }
 
+  /**
+   * Order candidate URLs by how much they tell an LLM about the site, then
+   * keep the first `limit`.
+   *
+   * Truncating the crawl order instead returned whichever section the sitemap
+   * lists first: modelcontextprotocol.io with maxPages 15 produced 15
+   * /community/* pages and no homepage, docs or specification. The order here
+   * is the site root, then one entry point per top-level section the site
+   * declares, then the rest of each section a page at a time until the budget
+   * runs out. Sections are visited alphabetically and ties inside a section
+   * break on depth then alphabetically, so a given site always yields the same
+   * list.
+   */
+  prioritizeUrls(urls, baseUrl, limit) {
+    const sections = new Map();
+    let sawAny = false;
+
+    for (const url of urls) {
+      let segments;
+      try {
+        segments = new URL(url).pathname.split('/').filter(Boolean);
+      } catch {
+        continue;
+      }
+      sawAny = true;
+      // The root arrives as either "https://site.com" or "https://site.com/"
+      // (normalizeUrl strips a trailing slash from every path but "/"), so it
+      // is recognised by its path and re-added below in one canonical form.
+      if (segments.length === 0) continue;
+      const section = segments[0];
+      if (!sections.has(section)) sections.set(section, []);
+      sections.get(section).push({ url, depth: segments.length });
+    }
+
+    if (!sawAny) return [];
+
+    // The site root leads: it is always fetched, and a site guide that omits
+    // the homepage is a misleading one.
+    const ordered = [`${baseUrl}/`];
+    const queues = [...sections.keys()].sort().map((name) =>
+      sections.get(name).sort((a, b) => a.depth - b.depth || a.url.localeCompare(b.url)));
+
+    let placed = true;
+    while (ordered.length < limit && placed) {
+      placed = false;
+      for (const queue of queues) {
+        if (ordered.length >= limit) break;
+        const next = queue.shift();
+        if (!next) continue;
+        ordered.push(next.url);
+        placed = true;
+      }
+    }
+
+    return ordered;
+  }
+
   categorizeSections(urls) {
     const categories = {
       content: [],

package/src/core/LocalizationManager.js

@@ -6,6 +6,7 @@
 
 import { z } from 'zod';
 import { EventEmitter } from 'events';
+import { identityHeaders } from '../utils/fetchIdentity.js';
 
 // ISO 3166-1 alpha-2 country codes with associated settings (Expanded to 15+ countries)
 const SUPPORTED_COUNTRIES = {
@@ -1058,7 +1059,7 @@ export class LocalizationManager extends EventEmitter {
       const start = Date.now();
       const response = await fetch('http://httpbin.org/ip', {
         method: 'GET',
-        headers: { 'User-Agent': 'Health-Check/1.0' },
+        headers: identityHeaders({ role: 'health-check' }),
         // Proxy configuration would go here
         timeout: 10000
       });