crawlforge-mcp-server 3.4.0 → 4.2.1

package/src/core/SamplingClient.js

@@ -0,0 +1,191 @@
+/**
+ * SamplingClient — MCP Sampling wrapper for CrawlForge
+ *
+ * Allows tools to request LLM completions from the connected MCP client
+ * instead of holding server-side API keys.
+ *
+ * Fallback chain (applied in resolveCompletion):
+ *   1. Ollama (local, no API key needed)
+ *   2. Server-side API key (OPENAI_API_KEY / ANTHROPIC_API_KEY)
+ *   3. MCP sampling request to client
+ *   4. Error
+ */
+
+const OLLAMA_DEFAULT_MODEL = 'llama3.2';
+const OLLAMA_BASE_URL = () => (process.env.OLLAMA_BASE_URL || 'http://localhost:11434').replace(/\/$/, '');
+
+/**
+ * Attempt an Ollama completion.
+ * @param {string} prompt
+ * @param {object} options
+ * @returns {Promise<string>}
+ */
+async function tryOllama(prompt, { model, maxTokens } = {}) {
+  const ollamaModel = model || process.env.OLLAMA_DEFAULT_MODEL || OLLAMA_DEFAULT_MODEL;
+  const url = `${OLLAMA_BASE_URL()}/api/generate`;
+  const res = await fetch(url, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      model: ollamaModel,
+      prompt,
+      stream: false,
+      ...(maxTokens ? { options: { num_predict: maxTokens } } : {}),
+    }),
+    signal: AbortSignal.timeout(30_000),
+  });
+  if (!res.ok) throw new Error(`Ollama HTTP ${res.status}`);
+  const data = await res.json();
+  if (!data.response) throw new Error('Ollama returned empty response');
+  return data.response;
+}
+
+/**
+ * Attempt an OpenAI completion using server-side API key.
+ * @param {string} prompt
+ * @param {object} options
+ * @returns {Promise<string>}
+ */
+async function tryOpenAI(prompt, { model, maxTokens } = {}) {
+  const apiKey = process.env.OPENAI_API_KEY;
+  if (!apiKey) throw new Error('OPENAI_API_KEY not set');
+  const base = (process.env.OPENAI_BASE_URL || 'https://api.openai.com').replace(/\/$/, '');
+  const res = await fetch(`${base}/v1/chat/completions`, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      Authorization: `Bearer ${apiKey}`,
+    },
+    body: JSON.stringify({
+      model: model || 'gpt-4o-mini',
+      messages: [{ role: 'user', content: prompt }],
+      ...(maxTokens ? { max_tokens: maxTokens } : {}),
+    }),
+    signal: AbortSignal.timeout(30_000),
+  });
+  if (!res.ok) throw new Error(`OpenAI HTTP ${res.status}`);
+  const data = await res.json();
+  return data.choices?.[0]?.message?.content || '';
+}
+
+/**
+ * Attempt an Anthropic completion using server-side API key.
+ * @param {string} prompt
+ * @param {object} options
+ * @returns {Promise<string>}
+ */
+async function tryAnthropic(prompt, { model, maxTokens } = {}) {
+  const apiKey = process.env.ANTHROPIC_API_KEY;
+  if (!apiKey) throw new Error('ANTHROPIC_API_KEY not set');
+  const base = (process.env.ANTHROPIC_BASE_URL || 'https://api.anthropic.com').replace(/\/$/, '');
+  const res = await fetch(`${base}/v1/messages`, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      'x-api-key': apiKey,
+      'anthropic-version': '2023-06-01',
+    },
+    body: JSON.stringify({
+      model: model || 'claude-haiku-4-5-20251001',
+      max_tokens: maxTokens || 1024,
+      messages: [{ role: 'user', content: prompt }],
+    }),
+    signal: AbortSignal.timeout(30_000),
+  });
+  if (!res.ok) throw new Error(`Anthropic HTTP ${res.status}`);
+  const data = await res.json();
+  return data.content?.[0]?.text || '';
+}
+
+export class SamplingClient {
+  /**
+   * @param {object} options
+   * @param {object|null} options.mcpServer - McpServer instance (must have requestSampling method if sampling is desired)
+   */
+  constructor({ mcpServer } = {}) {
+    this._mcpServer = mcpServer || null;
+  }
+
+  /**
+   * Resolve an LLM completion using the fallback chain:
+   * Ollama → API key (OpenAI then Anthropic) → MCP sampling → error
+   *
+   * @param {string} prompt - The prompt to complete
+   * @param {object} options
+   * @param {string} [options.model] - Override model name
+   * @param {number} [options.maxTokens] - Max tokens for response
+   * @param {string} [options.systemPrompt] - Optional system-level instruction
+   * @returns {Promise<{ text: string, provider: string }>}
+   */
+  async complete(prompt, options = {}) {
+    const fullPrompt = options.systemPrompt
+      ? `${options.systemPrompt}\n\n${prompt}`
+      : prompt;
+
+    // 1. Try Ollama (local, no API key)
+    try {
+      const text = await tryOllama(fullPrompt, options);
+      return { text, provider: 'ollama' };
+    } catch (_ollamaErr) {
+      // Ollama unavailable — continue fallback chain
+    }
+
+    // 2. Try server-side API keys
+    if (process.env.OPENAI_API_KEY) {
+      try {
+        const text = await tryOpenAI(fullPrompt, options);
+        return { text, provider: 'openai' };
+      } catch (_openaiErr) {
+        // OpenAI failed — try Anthropic
+      }
+    }
+
+    if (process.env.ANTHROPIC_API_KEY) {
+      try {
+        const text = await tryAnthropic(fullPrompt, options);
+        return { text, provider: 'anthropic' };
+      } catch (_anthropicErr) {
+        // Anthropic failed — try sampling
+      }
+    }
+
+    // 3. Try MCP sampling (client-side LLM)
+    if (this._mcpServer?.server?.createMessage) {
+      try {
+        const samplingResult = await this._mcpServer.server.createMessage({
+          messages: [{ role: 'user', content: { type: 'text', text: fullPrompt } }],
+          maxTokens: options.maxTokens || 1024,
+          includeContext: 'none',
+        });
+        const text = samplingResult?.content?.text || '';
+        if (text) return { text, provider: 'sampling' };
+      } catch (_samplingErr) {
+        // Sampling not supported or failed
+      }
+    }
+
+    // 4. All fallbacks exhausted
+    throw new Error(
+      'No LLM available: Ollama is not running, no API keys set (OPENAI_API_KEY / ANTHROPIC_API_KEY), and the MCP client does not support sampling.'
+    );
+  }
+
+  /**
+   * Check which LLM providers are available without making a completion call.
+   * @returns {Promise<{ ollama: boolean, openai: boolean, anthropic: boolean, sampling: boolean }>}
+   */
+  async probe() {
+    const result = { ollama: false, openai: false, anthropic: false, sampling: false };
+
+    try {
+      const res = await fetch(`${OLLAMA_BASE_URL()}/api/tags`, { signal: AbortSignal.timeout(3000) });
+      result.ollama = res.ok;
+    } catch (_) { /* unavailable */ }
+
+    result.openai = !!process.env.OPENAI_API_KEY;
+    result.anthropic = !!process.env.ANTHROPIC_API_KEY;
+    result.sampling = !!(this._mcpServer?.server?.createMessage);
+
+    return result;
+  }
+}

package/src/core/StealthBrowserManager.js

@@ -60,8 +60,9 @@ const StealthConfigSchema = z.object({
 export class StealthBrowserManager {
   constructor(options = {}) {
     this.browser = null;
+    this._maxContexts = parseInt(process.env.MAX_BROWSER_CONTEXTS || '10', 10);
     this.contexts = new BrowserContextPool({
-      maxContexts: parseInt(process.env.MAX_BROWSER_CONTEXTS || '10', 10),
+      maxContexts: this._maxContexts,
       periodicRefreshAfter: 200,
       closeIdleAfterMs: 30 * 60 * 1000,
       waitTimeoutMs: 10_000,
@@ -69,6 +70,8 @@ export class StealthBrowserManager {
         this.fingerprints.delete(contextId);
       }
     });
+    // D2.2: fingerprints Map is capped at _maxContexts to prevent unbounded growth.
+    // Oldest entries are evicted when the cap is exceeded (insertion order via Map).
     this.fingerprints = new Map();
     
     // Enhanced stealth components
@@ -377,7 +380,8 @@ export class StealthBrowserManager {
     await this.applyAdvancedStealthConfigurations(context, validatedConfig, fingerprint);
     
     await this.contexts.set(contextId, { context, fingerprint, config: validatedConfig });
-    this.fingerprints.set(contextId, fingerprint);
+    // D2.2: enforce LRU cap on fingerprints Map
+    this._setFingerprint(contextId, fingerprint);
 
     return { context, contextId, fingerprint };
   }
@@ -1702,6 +1706,18 @@ export class StealthBrowserManager {
     }
   }
 
+  /**
+   * D2.2: LRU-capped fingerprint setter.
+   * Evicts the oldest entry when the Map exceeds _maxContexts to prevent unbounded growth.
+   */
+  _setFingerprint(contextId, fingerprint) {
+    if (this.fingerprints.size >= this._maxContexts) {
+      const oldestKey = this.fingerprints.keys().next().value;
+      this.fingerprints.delete(oldestKey);
+    }
+    this.fingerprints.set(contextId, fingerprint);
+  }
+
   /**
    * Close all contexts and browser
    */
@@ -1800,4 +1816,234 @@ export class StealthBrowserManager {
   }
 }
 
+
+
+// ─── D3.2: BrowserEngine interface + CamoufoxAdapter ──────────────────────────
+//
+// Camoufox licensing note:
+//   camoufox (github.com/daijro/camoufox) is MIT-licensed.
+//   python-camoufox launcher is MPL-2.0. The JS bindings
+//   (@camoufox/jsapi) are MIT. There are no AGPL forks in the
+//   main distribution chain as of 2026-05. Always re-verify before
+//   distributing: https://github.com/daijro/camoufox/blob/main/LICENSE
+//
+// Engine-selection criteria:
+//   playwright — Chromium-based, fastest, best Playwright ecosystem support.
+//               Good default for most sites.
+//   camoufox  — Firefox-based, patches browser internals to hide automation
+//               markers at the C++ level, not via JS injection. Scores
+//               significantly higher on CreepJS and Datadome than any
+//               Playwright+stealth combination. Use when Playwright is
+//               detected and blocked.
+//
+// Benchmark methodology (not run here — network-dependent):
+//   1. Open https://bot.sannysoft.com with each engine — count red indicators.
+//   2. Open https://nowsecure.nl with each engine — check "You are not a bot".
+//   3. Run https://abrahamjuliot.github.io/creepjs/ — compare trust score %.
+//   4. Use Datadome test page — verify challenge is not triggered.
+//   All tests must be run with a clean incognito context and no extensions.
+
+/**
+ * BrowserEngine interface (D3.2).
+ * Implementors must provide:
+ *   launch(config)  → Promise<Browser-like>
+ *   name()          → string
+ *   isAvailable()   → Promise<boolean>
+ */
+export class BrowserEngine {
+  /** @returns {string} */
+  name() { throw new Error('BrowserEngine.name() must be implemented'); }
+
+  /** @returns {Promise<boolean>} */
+  async isAvailable() { return false; }
+
+  /**
+   * @param {object} config
+   * @returns {Promise<object>} browser-like handle
+   */
+  async launch(_config) { throw new Error('BrowserEngine.launch() must be implemented'); }
+}
+
+/**
+ * CamoufoxAdapter — Firefox-based engine using the camoufox package.
+ * Falls back gracefully when camoufox is not installed.
+ *
+ * Install: npm install camoufox  (MIT license)
+ */
+export class CamoufoxAdapter extends BrowserEngine {
+  name() { return 'camoufox'; }
+
+  async isAvailable() {
+    try {
+      await import('camoufox');
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  async launch(config = {}) {
+    let camoufox;
+    try {
+      camoufox = await import('camoufox');
+    } catch {
+      throw new Error(
+        'camoufox is not installed. Run: npm install camoufox. Note: camoufox is MIT-licensed and requires Firefox to be installed.'
+      );
+    }
+
+    // camoufox API mirrors playwright — returns a Browser object
+    const browser = await (camoufox.launch || camoufox.default?.launch)({
+      headless: config.headless !== false,
+      ...config.launchOptions
+    });
+
+    return browser;
+  }
+}
+
+// ─── D3.4: BrowserBackend interface + backends ────────────────────────────────
+//
+// CRAWLFORGE_BROWSER_BACKEND=local  → LocalPlaywrightBackend (default, current behavior)
+// CRAWLFORGE_BROWSER_BACKEND=browserbase → BrowserBaseBackend via CDP
+//
+// Graceful fallback: if BrowserBaseBackend fails to connect (no API key, network error,
+// quota exceeded), StealthBrowserManager.getBrowserBackend() falls back to local.
+
+/**
+ * BrowserBackend interface (D3.4).
+ * Implementors must provide:
+ *   connect(config)    → Promise<Browser-like>
+ *   disconnect()       → Promise<void>
+ *   name()             → string
+ *   isConfigured()     → boolean
+ */
+export class BrowserBackend {
+  name() { throw new Error('BrowserBackend.name() must be implemented'); }
+  isConfigured() { return false; }
+  async connect(_config) { throw new Error('BrowserBackend.connect() must be implemented'); }
+  async disconnect() {}
+}
+
+/**
+ * LocalPlaywrightBackend — wraps existing Playwright Chromium behavior.
+ * This is the default backend (preserves all pre-D3.4 behavior).
+ */
+export class LocalPlaywrightBackend extends BrowserBackend {
+  name() { return 'local'; }
+  isConfigured() { return true; }
+
+  async connect(config = {}) {
+    const { chromium } = await import('playwright');
+    return chromium.launch({
+      headless: config.headless !== false,
+      ...config.launchOptions
+    });
+  }
+
+  async disconnect() {}
+}
+
+/**
+ * BrowserBaseBackend — connects to BrowserBase cloud browser via CDP.
+ *
+ * Requirements:
+ *   BROWSERBASE_API_KEY — your BrowserBase API key
+ *   CRAWLFORGE_BROWSER_BACKEND=browserbase
+ *
+ * The backend creates a BrowserBase session, gets the CDP endpoint, and
+ * connects Playwright over it.  All stealth fingerprint injection still
+ * runs through CrawlForge's existing page-level scripts.
+ *
+ * Docs: https://docs.browserbase.com/integrations/playwright
+ */
+export class BrowserBaseBackend extends BrowserBackend {
+  constructor() {
+    super();
+    this._sessionId = null;
+  }
+
+  name() { return 'browserbase'; }
+
+  isConfigured() {
+    return Boolean(process.env.BROWSERBASE_API_KEY);
+  }
+
+  async connect(config = {}) {
+    const apiKey = process.env.BROWSERBASE_API_KEY;
+    if (!apiKey) {
+      throw new Error(
+        'BrowserBase requires BROWSERBASE_API_KEY environment variable. ' +
+        'Get your key at https://browserbase.com'
+      );
+    }
+
+    // Create a BrowserBase session
+    const sessionRes = await fetch('https://www.browserbase.com/v1/sessions', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'X-BB-API-Key': apiKey
+      },
+      body: JSON.stringify({
+        projectId: process.env.BROWSERBASE_PROJECT_ID,
+        ...config.sessionOptions
+      })
+    });
+
+    if (!sessionRes.ok) {
+      const err = await sessionRes.text().catch(() => '');
+      throw new Error();
+    }
+
+    const session = await sessionRes.json();
+    this._sessionId = session.id;
+
+    // Connect Playwright over CDP
+    const { chromium } = await import('playwright');
+    const browser = await chromium.connectOverCDP(session.connectUrl, {
+      timeout: config.timeout || 30000
+    });
+
+    return browser;
+  }
+
+  async disconnect() {
+    if (!this._sessionId) return;
+    const apiKey = process.env.BROWSERBASE_API_KEY;
+    if (!apiKey) return;
+
+    try {
+      await fetch(`https://www.browserbase.com/v1/sessions/${this._sessionId}`, {
+        method: 'DELETE',
+        headers: { 'X-BB-API-Key': apiKey }
+      });
+    } catch {
+      // Non-fatal — session will expire on BrowserBase's side
+    } finally {
+      this._sessionId = null;
+    }
+  }
+}
+
+/**
+ * Factory: resolve which BrowserBackend to use based on env config.
+ * Falls back to local on any error.
+ *
+ * @param {object} [options]
+ * @returns {BrowserBackend}
+ */
+export function resolveBrowserBackend(options = {}) {
+  const requested = (process.env.CRAWLFORGE_BROWSER_BACKEND || 'local').toLowerCase();
+
+  if (requested === 'browserbase') {
+    const bb = new BrowserBaseBackend();
+    if (bb.isConfigured()) return bb;
+    // BROWSERBASE_API_KEY not set — fall through to local
+    console.error('[StealthBrowserManager] CRAWLFORGE_BROWSER_BACKEND=browserbase but BROWSERBASE_API_KEY is not set. Falling back to local Playwright.');
+  }
+
+  return new LocalPlaywrightBackend();
+}
+
 export default StealthBrowserManager;

package/src/core/WebhookDispatcher.js

@@ -287,7 +287,9 @@ export class WebhookDispatcher extends EventEmitter {
     this.processing = true;
     
     try {
-      const batchSize = this.enableBatching ? this.batchSize : 1;
+      // D2.5: cap retry batch size to prevent a flood of retries overwhelming targets
+      const rawBatchSize = this.enableBatching ? this.batchSize : 1;
+      const batchSize = Math.min(rawBatchSize, 10); // never process more than 10 at once
       const batch = this.queue.splice(0, batchSize);
 
       if (this.enableBatching && batch.length > 1) {
@@ -328,14 +330,13 @@ export class WebhookDispatcher extends EventEmitter {
       
       // Check if we should retry
       if (event.attempts < this.maxRetries) {
-        // Re-queue for retry with exponential backoff
-        const delay = Math.min(
-          this.retryDelay * Math.pow(2, event.attempts - 1),
-          60000 // Max 1 minute delay
-        );
+        // D2.5: per-webhook exponential backoff with jitter to prevent retry storms
+        const baseDelay = this.retryDelay * Math.pow(2, event.attempts - 1);
+        const jitter = Math.random() * Math.min(baseDelay * 0.25, 5000); // up to 25% or 5s
+        const delay = Math.min(baseDelay + jitter, 60000); // cap at 1 minute
         
         setTimeout(() => {
-          this.queue.unshift(event); // Add to front for priority
+          this.queue.push(event); // push to back (not front) to avoid head-of-line blocking
         }, delay);
 
         this.emit('webhookRetry', event, error, delay);
@@ -503,9 +504,16 @@ export class WebhookDispatcher extends EventEmitter {
       clearInterval(this.healthMonitoringTimer);
     }
 
-    this.healthMonitoringTimer = setInterval(() => {
-      this.performHealthChecks();
-    }, this.healthCheckInterval);
+    // D2.5: add jitter to health check interval to prevent synchronized storms
+    const scheduleNextHealthCheck = () => {
+      const jitter = Math.floor(Math.random() * Math.min(this.healthCheckInterval * 0.1, 10000));
+      this.healthMonitoringTimer = setTimeout(() => {
+        this.performHealthChecks().finally(() => {
+          if (this.healthMonitoringTimer !== null) scheduleNextHealthCheck();
+        });
+      }, this.healthCheckInterval + jitter);
+    };
+    scheduleNextHealthCheck();
   }
 
   /**