crawlforge-mcp-server 3.0.18 → 3.4.0

package/src/core/StealthBrowserManager.js

@@ -12,6 +12,7 @@ import { chromium } from 'playwright';
 import { z } from 'zod';
 import crypto from 'crypto';
 import HumanBehaviorSimulator from '../utils/HumanBehaviorSimulator.js';
+import { BrowserContextPool } from './BrowserContextPool.js';
 
 const StealthConfigSchema = z.object({
   level: z.enum(['basic', 'medium', 'advanced']).default('medium'),
@@ -59,7 +60,15 @@ const StealthConfigSchema = z.object({
 export class StealthBrowserManager {
   constructor(options = {}) {
     this.browser = null;
-    this.contexts = new Map();
+    this.contexts = new BrowserContextPool({
+      maxContexts: parseInt(process.env.MAX_BROWSER_CONTEXTS || '10', 10),
+      periodicRefreshAfter: 200,
+      closeIdleAfterMs: 30 * 60 * 1000,
+      waitTimeoutMs: 10_000,
+      onContextExpired: (contextId) => {
+        this.fingerprints.delete(contextId);
+      }
+    });
     this.fingerprints = new Map();
     
     // Enhanced stealth components
@@ -367,7 +376,7 @@ export class StealthBrowserManager {
     // Apply stealth scripts and configurations
     await this.applyAdvancedStealthConfigurations(context, validatedConfig, fingerprint);
     
-    this.contexts.set(contextId, { context, fingerprint, config: validatedConfig });
+    await this.contexts.set(contextId, { context, fingerprint, config: validatedConfig });
     this.fingerprints.set(contextId, fingerprint);
 
     return { context, contextId, fingerprint };
@@ -1493,11 +1502,20 @@ export class StealthBrowserManager {
       throw new Error('Context not found');
     }
 
+    // Record use and check if context needs periodic refresh
+    const needsRefresh = this.contexts.recordUse(contextId);
+    if (needsRefresh) {
+      // Dispose old context; caller should create a fresh one
+      await this.contexts.dispose(contextId);
+      this.fingerprints.delete(contextId);
+      throw new Error(`StealthBrowserManager: context ${contextId} has reached its use limit and was recycled. Create a new context.`);
+    }
+
     const page = await contextData.context.newPage();
-    
+
     // Apply additional page-level stealth measures
     await this.applyPageStealthMeasures(page, contextData.config, contextData.fingerprint);
-    
+
     return page;
   }
 
@@ -1678,10 +1696,8 @@ export class StealthBrowserManager {
    * Close specific context
    */
   async closeContext(contextId) {
-    const contextData = this.contexts.get(contextId);
-    if (contextData) {
-      await contextData.context.close();
-      this.contexts.delete(contextId);
+    if (this.contexts.has(contextId)) {
+      await this.contexts.dispose(contextId);
       this.fingerprints.delete(contextId);
     }
   }
@@ -1690,16 +1706,8 @@ export class StealthBrowserManager {
    * Close all contexts and browser
    */
   async cleanup() {
-    // Close all contexts
-    for (const [contextId, contextData] of this.contexts.entries()) {
-      try {
-        await contextData.context.close();
-      } catch (error) {
-        console.warn(`Failed to close context ${contextId}:`, error.message);
-      }
-    }
-
-    this.contexts.clear();
+    // Close all contexts via pool (handles idle timer cleanup + wait queue drain)
+    await this.contexts.destroy();
     this.fingerprints.clear();
 
     // Reset human behavior simulator

package/src/core/cache/CacheManager.js

@@ -82,10 +82,12 @@ export class CacheManager extends EventEmitter {
       this.startMonitoring(monitoringInterval);
     }
     
-    // Initialize auto cleanup
+    // Initialize auto cleanup. .unref() so the timer never blocks process exit
+    // — short-lived CLI invocations and tests don't need an explicit destroy().
     this.cleanupTimer = setInterval(() => {
       this.cleanupExpired();
     }, autoCleanupInterval);
+    if (typeof this.cleanupTimer.unref === 'function') this.cleanupTimer.unref();
     
     // Eviction tracking is handled in the LRU cache dispose callback above
   }
@@ -546,6 +548,7 @@ export class CacheManager extends EventEmitter {
       this.updateStats();
       this.emit('monitoring', this.getDetailedStats());
     }, interval);
+    if (typeof this.monitoringTimer.unref === 'function') this.monitoringTimer.unref();
   }
 
   /**

package/src/core/crawlers/BFSCrawler.js

@@ -19,7 +19,8 @@ export class BFSCrawler {
       concurrency = 10,
       domainFilter = null,
       enableLinkAnalysis = true,
-      linkAnalyzerOptions = {}
+      linkAnalyzerOptions = {},
+      sessionContext = null
     } = options;
 
     this.maxDepth = maxDepth;
@@ -28,6 +29,8 @@ export class BFSCrawler {
     this.respectRobots = respectRobots;
     this.userAgent = userAgent;
     this.timeout = timeout;
+    // Session context for cookie jar + persistent headers (null = stateless)
+    this.sessionContext = sessionContext;
 
     this.visited = new Set();
     this.results = [];
@@ -254,21 +257,32 @@ export class BFSCrawler {
         'Connection': 'keep-alive',
         'Upgrade-Insecure-Requests': '1'
       };
-      
-      const headers = { ...defaultHeaders, ...domainRules.customHeaders };
+
+      let headers = { ...defaultHeaders, ...domainRules.customHeaders };
+
+      // If a session is active, layer in session headers + cookie jar
+      if (this.sessionContext) {
+        headers = this.sessionContext.applyToHeaders(url, headers);
+      }
+
       const effectiveTimeout = domainRules.timeout || this.timeout;
-      
+
       // Update timeout if different
       if (effectiveTimeout !== this.timeout) {
         clearTimeout(timeoutId);
         setTimeout(() => controller.abort(), effectiveTimeout);
       }
-      
+
       const response = await fetch(url, {
         signal: controller.signal,
         headers
       });
 
+      // Capture any cookies the server sets during the crawl
+      if (this.sessionContext) {
+        this.sessionContext.recordCookies(response, url);
+      }
+
       clearTimeout(timeoutId);
 
       if (!response.ok) {

package/src/observability/metrics.js

@@ -0,0 +1,137 @@
+/**
+ * Prometheus metrics — dependency-free implementation.
+ *
+ * Why no prom-client? CrawlForge is shipped via npm and runs in stdio mode
+ * by default. Pulling in prom-client (and its dependency tree) just to
+ * expose four counters/gauges is overkill. This 150 LOC implementation
+ * conforms to the Prometheus exposition format 0.0.4.
+ *
+ * Disabled by default. Enable via `CRAWLFORGE_METRICS=true` in HTTP mode.
+ *
+ * Counters/gauges exposed:
+ *   - crawlforge_tool_requests_total{tool,outcome}
+ *   - crawlforge_tool_errors_total{tool,error_class}
+ *   - crawlforge_tool_duration_ms{tool}    (histogram, summed)
+ *   - crawlforge_credits_consumed_total{tool}
+ *   - crawlforge_browser_pool_in_use       (gauge)
+ *   - crawlforge_browser_pool_capacity     (gauge)
+ */
+
+const CONTENT_TYPE = 'text/plain; version=0.0.4; charset=utf-8';
+
+export function createMetricsRegistry() {
+  const counters = new Map();   // name|labels -> number
+  const gauges = new Map();     // name|labels -> number
+  const histograms = new Map(); // name|labels -> { count, sum, buckets:{le->count} }
+
+  const HISTOGRAM_BUCKETS_MS = [10, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000];
+
+  function key(name, labels) {
+    const labelStr = Object.entries(labels ?? {})
+      .sort(([a], [b]) => a.localeCompare(b))
+      .map(([k, v]) => `${k}="${escapeLabel(String(v))}"`)
+      .join(',');
+    return labelStr ? `${name}{${labelStr}}` : name;
+  }
+
+  return {
+    contentType: CONTENT_TYPE,
+
+    incCounter(name, labels, by = 1) {
+      const k = key(name, labels);
+      counters.set(k, (counters.get(k) ?? 0) + by);
+    },
+
+    setGauge(name, labels, value) {
+      gauges.set(key(name, labels), value);
+    },
+
+    observeHistogram(name, labels, valueMs) {
+      const k = key(name, labels);
+      let h = histograms.get(k);
+      if (!h) {
+        h = { count: 0, sum: 0, buckets: Object.fromEntries(HISTOGRAM_BUCKETS_MS.map(b => [b, 0])) };
+        histograms.set(k, h);
+      }
+      h.count += 1;
+      h.sum += valueMs;
+      for (const b of HISTOGRAM_BUCKETS_MS) {
+        if (valueMs <= b) h.buckets[b] += 1;
+      }
+    },
+
+    async render() {
+      const lines = [];
+
+      // Counters
+      const counterNames = new Set();
+      for (const k of counters.keys()) counterNames.add(k.split('{')[0]);
+      for (const name of counterNames) {
+        lines.push(`# HELP ${name} ${describe(name)}`);
+        lines.push(`# TYPE ${name} counter`);
+        for (const [k, v] of counters.entries()) {
+          if (k.split('{')[0] === name) lines.push(`${k} ${v}`);
+        }
+      }
+
+      // Gauges
+      const gaugeNames = new Set();
+      for (const k of gauges.keys()) gaugeNames.add(k.split('{')[0]);
+      for (const name of gaugeNames) {
+        lines.push(`# HELP ${name} ${describe(name)}`);
+        lines.push(`# TYPE ${name} gauge`);
+        for (const [k, v] of gauges.entries()) {
+          if (k.split('{')[0] === name) lines.push(`${k} ${v}`);
+        }
+      }
+
+      // Histograms
+      const histNames = new Set();
+      for (const k of histograms.keys()) histNames.add(k.split('{')[0]);
+      for (const name of histNames) {
+        lines.push(`# HELP ${name} ${describe(name)}`);
+        lines.push(`# TYPE ${name} histogram`);
+        for (const [k, h] of histograms.entries()) {
+          if (k.split('{')[0] !== name) continue;
+          // Reconstruct base labels (everything inside { })
+          const baseLabels = k.includes('{') ? k.slice(k.indexOf('{') + 1, -1) : '';
+          const sep = baseLabels ? ',' : '';
+          for (const b of HISTOGRAM_BUCKETS_MS) {
+            lines.push(`${name}_bucket{${baseLabels}${sep}le="${b}"} ${h.buckets[b]}`);
+          }
+          lines.push(`${name}_bucket{${baseLabels}${sep}le="+Inf"} ${h.count}`);
+          lines.push(`${name}_sum${baseLabels ? `{${baseLabels}}` : ''} ${h.sum}`);
+          lines.push(`${name}_count${baseLabels ? `{${baseLabels}}` : ''} ${h.count}`);
+        }
+      }
+
+      return lines.join('\n') + '\n';
+    },
+
+    // Snapshot for tests
+    _snapshot() {
+      return {
+        counters: Object.fromEntries(counters.entries()),
+        gauges: Object.fromEntries(gauges.entries()),
+        histograms: Object.fromEntries(histograms.entries())
+      };
+    }
+  };
+}
+
+const HELP = {
+  crawlforge_tool_requests_total: 'Total number of MCP tool invocations',
+  crawlforge_tool_errors_total: 'Total number of MCP tool errors',
+  crawlforge_tool_duration_ms: 'MCP tool invocation duration in milliseconds',
+  crawlforge_credits_consumed_total: 'Total CrawlForge credits consumed',
+  crawlforge_browser_pool_in_use: 'Number of browser contexts currently leased from the pool',
+  crawlforge_browser_pool_capacity: 'Maximum browser context pool capacity'
+};
+
+function describe(name) {
+  return HELP[name] ?? name;
+}
+
+function escapeLabel(v) {
+  return v.replace(/\\/g, '\\\\').replace(/\n/g, '\\n').replace(/"/g, '\\"');
+}

package/src/observability/tracing.js

@@ -0,0 +1,74 @@
+/**
+ * OpenTelemetry-style tracing facade.
+ *
+ * Disabled by default. When `OTEL_SDK_DISABLED !== 'false'`, all calls are
+ * no-ops with zero overhead (no SDK loaded). To enable, install
+ * `@opentelemetry/api` + `@opentelemetry/sdk-node` in the host application
+ * and set `OTEL_SDK_DISABLED=false`.
+ *
+ * Why a facade instead of importing `@opentelemetry/api` directly?
+ *   - CrawlForge ships via npm; we don't want to force the OTel runtime
+ *     on every user. The facade pattern matches `@opentelemetry/api`'s
+ *     no-op-by-default design but doesn't add the dependency to package.json.
+ *   - Operators who want tracing install the SDK themselves and configure
+ *     OTEL_* env vars. We call into `globalThis.__otelTracer` if present.
+ *
+ * Span attributes set on every tool invocation:
+ *   - mcp.tool.name
+ *   - mcp.tool.duration_ms
+ *   - mcp.tool.outcome
+ *   - mcp.credit.cost
+ */
+
+const NOOP_SPAN = {
+  setAttribute() { return this; },
+  setAttributes() { return this; },
+  setStatus() { return this; },
+  recordException() { return this; },
+  end() {}
+};
+
+export function isTracingEnabled() {
+  return process.env.OTEL_SDK_DISABLED === 'false' && Boolean(globalThis.__otelTracer);
+}
+
+export function startToolSpan(toolName) {
+  if (!isTracingEnabled()) return NOOP_SPAN;
+  try {
+    const tracer = globalThis.__otelTracer;
+    const span = tracer.startSpan(`mcp.tool.${toolName}`, {
+      attributes: { 'mcp.tool.name': toolName }
+    });
+    return span;
+  } catch {
+    return NOOP_SPAN;
+  }
+}
+
+/**
+ * Record a complete tool invocation. Convenience wrapper used by withAuth.
+ *
+ * @param {string} toolName
+ * @param {object} attrs   — { duration_ms, outcome, credit_cost, creator_mode }
+ * @param {Error}  [error]
+ */
+export function recordToolInvocation(toolName, attrs, error) {
+  if (!isTracingEnabled()) return;
+  try {
+    const span = startToolSpan(toolName);
+    span.setAttributes({
+      'mcp.tool.duration_ms': attrs.duration_ms,
+      'mcp.tool.outcome': attrs.outcome,
+      'mcp.credit.cost': attrs.credit_cost,
+      'mcp.credit.outcome': attrs.outcome,
+      'mcp.creator_mode': Boolean(attrs.creator_mode)
+    });
+    if (error) {
+      span.recordException(error);
+      span.setStatus({ code: 2, message: error.message });
+    }
+    span.end();
+  } catch {
+    // tracing must never break the request path
+  }
+}