crawlforge-mcp-server 3.0.17 → 3.3.1

package/src/constants/config.js

@@ -1,6 +1,7 @@
 import dotenv from 'dotenv';
 import { fileURLToPath } from 'url';
 import { dirname, join } from 'path';
+import { resolveApiEndpoint } from '../core/endpointGuard.js';
 
 // Load environment variables
 const __filename = fileURLToPath(import.meta.url);
@@ -11,7 +12,7 @@ export const config = {
   // CrawlForge API Configuration
   crawlforge: {
     apiKey: process.env.CRAWLFORGE_API_KEY || '',
-    apiBaseUrl: process.env.CRAWLFORGE_API_URL || 'https://www.crawlforge.dev'
+    apiBaseUrl: resolveApiEndpoint(process.env.CRAWLFORGE_API_URL || 'https://www.crawlforge.dev')
   },
 
   // Performance

package/src/core/ActionExecutor.js

@@ -171,49 +171,8 @@ export class ActionExecutor extends EventEmitter {
         actualChainConfig = chainConfig;
       }
 
-      // For testing purposes, provide mock execution for example.com
-      if (url === 'http://example.com') {
-        const actions = Array.isArray(chainConfig) ? chainConfig : actualChainConfig.actions;
-        const mockResults = actions.map((action, index) => {
-          const baseResult = {
-            id: `mock_action_${index}`,
-            type: action.type,
-            success: true,
-            executionTime: 10,
-            timestamp: Date.now(),
-            description: `Mock ${action.type} action`
-          };
-
-          if (action.type === 'wait') {
-            const waitTime = action.duration || action.milliseconds || 100;
-            baseResult.result = { waited: waitTime };
-          } else if (action.type === 'click') {
-            baseResult.result = { selector: action.selector, button: 'left' };
-          } else {
-            baseResult.result = { mockResult: true };
-          }
-
-          return baseResult;
-        });
-
-        return {
-          success: true,
-          chainId,
-          url,
-          executionTime: Date.now() - startTime,
-          results: mockResults,
-          screenshots: [],
-          metadata: {
-            userAgent: 'mock-agent',
-            viewport: { width: 1280, height: 720 }
-          },
-          stats: {
-            totalActions: mockResults.length,
-            successfulActions: mockResults.filter(r => r.success).length,
-            failedActions: mockResults.filter(r => !r.success).length
-          }
-        };
-      }
+      // (v3.0.19 cleanup) The legacy example.com mock branch was removed — no
+      // test depended on it and it short-circuited real validation. See §A3.
 
       // Validate chain configuration
       const validatedChain = ActionChainSchema.parse(actualChainConfig);

package/src/core/AuthManager.js

@@ -6,16 +6,21 @@
 // Using native fetch (Node.js 18+)
 import fs from 'fs/promises';
 import path from 'path';
+import { randomUUID } from 'crypto';
 import { isCreatorModeVerified } from './creatorMode.js';
+import { resolveApiEndpoint } from './endpointGuard.js';
+import { logger } from '../utils/Logger.js';
 
 class AuthManager {
   constructor() {
-    this.apiEndpoint = process.env.CRAWLFORGE_API_URL || 'https://www.crawlforge.dev';
+    this.apiEndpoint = resolveApiEndpoint(process.env.CRAWLFORGE_API_URL || 'https://www.crawlforge.dev');
     this.configPath = path.join(process.env.HOME || process.env.USERPROFILE, '.crawlforge', 'config.json');
+    this.pendingUsagePath = path.join(process.env.HOME || process.env.USERPROFILE, '.crawlforge', 'pending-usage.json');
     this.config = null;
     this.creditCache = new Map();
     this.lastCreditCheck = null;
-    this.CREDIT_CHECK_INTERVAL = 60000; // Check credits every minute max
+    this.lastSuccessfulCreditCheck = new Map();
+    this.CREDIT_CHECK_INTERVAL = 15000;
     this.initialized = false;
     // NOTE: Don't read creator mode in constructor - it's set dynamically in server.js
   }
@@ -30,17 +35,23 @@ class AuthManager {
 
   /**
    * Initialize the auth manager and load stored config
+   *
+   * Audit phase 5: re-validate the stored API key against the backend at startup.
+   * If the backend explicitly reports the key as revoked/invalid, we throw —
+   * the server must refuse to start rather than silently run with a dead key.
+   * Network failures are tolerated (we already have a cached config and the
+   * fail-closed credit check from audit phase 2 handles runtime revocation).
    */
   async initialize() {
     if (this.initialized) return;
-    
+
     // Skip config loading in creator mode
     if (this.isCreatorMode()) {
       console.log('🚀 Creator Mode Active - Unlimited Access Enabled');
       this.initialized = true;
       return;
     }
-    
+
     try {
       await this.loadConfig();
       this.initialized = true;
@@ -48,6 +59,45 @@ class AuthManager {
       console.log('No existing CrawlForge configuration found. Run setup to configure.');
       this.initialized = true;
     }
+
+    // Phase 5: re-validate cached API key with backend. Refuse to start if revoked.
+    if (this.config?.apiKey && process.env.CRAWLFORGE_SKIP_STARTUP_VALIDATION !== 'true') {
+      const validation = await this.validateApiKey(this.config.apiKey);
+      if (!validation.valid) {
+        const lower = (validation.error || '').toLowerCase();
+        const isExplicitReject =
+          lower.includes('invalid') ||
+          lower.includes('revoked') ||
+          lower.includes('not found') ||
+          lower.includes('expired') ||
+          lower.includes('unauthorized');
+        if (isExplicitReject) {
+          const rejectErr = new Error(
+            `CrawlForge API key rejected by backend at startup: ${validation.error}. ` +
+            `Run \`npm run setup\` with a current key, or set CRAWLFORGE_SKIP_STARTUP_VALIDATION=true to bypass.`
+          );
+          logger.error('Startup API key validation rejected by backend', rejectErr, {
+            backendError: validation.error
+          });
+          throw rejectErr;
+        }
+        // Connection error — tolerate, log, continue. Runtime credit check will fail closed.
+        logger.warn('Startup API key validation skipped (backend unreachable)', {
+          error: validation.error
+        });
+      } else {
+        logger.info('Startup API key validation OK', {
+          userId: validation.userId,
+          creditsRemaining: validation.creditsRemaining
+        });
+      }
+    }
+
+    try {
+      await this._flushPendingUsage();
+    } catch {
+      // Best-effort flush — do not block startup
+    }
   }
 
   /**
@@ -192,20 +242,16 @@ class AuthManager {
         const data = await response.json();
         this.creditCache.set(this.config.userId, data.creditsRemaining);
         this.lastCreditCheck = now;
+        this.lastSuccessfulCreditCheck.set(this.config.userId, now);
         return data.creditsRemaining >= estimatedCredits;
       }
     } catch (error) {
       console.error('Failed to check credits:', error.message);
 
-      // Grace period: allow stale cached credits during transient network failures
-      // This prevents outages from blocking authenticated users while still
-      // failing closed when there's no cached data (no free usage bypass)
+      const lastOk = this.lastSuccessfulCreditCheck.get(this.config.userId) ?? 0;
+      const withinGrace = Date.now() - lastOk < 30_000;
       const cached = this.creditCache.get(this.config.userId);
-      if (cached !== undefined && cached >= estimatedCredits) {
-        console.warn('Using cached credits due to network error — will re-verify on next call');
-        return true;
-      }
-
+      if (withinGrace && cached !== undefined && cached >= estimatedCredits) return true;
       throw new Error('Unable to verify credits. Please check your connection and try again.');
     }
   }
@@ -218,39 +264,188 @@ class AuthManager {
     if (this.isCreatorMode()) {
       return;
     }
-    
+
     if (!this.config) {
       return; // Silently skip if not configured
     }
 
-    try {
-      const payload = {
-        tool,
-        creditsUsed,
-        requestData,
-        responseStatus,
-        processingTime,
-        timestamp: new Date().toISOString(),
-        version: '3.0.3'
-      };
+    const userId = this.config.userId;
 
+    // Pre-decrement cache before fetch so network failures still deplete credits
+    const cached = this.creditCache.get(userId);
+    if (cached !== undefined) {
+      this.creditCache.set(userId, Math.max(0, cached - creditsUsed));
+    }
+
+    // Audit phase A2: every usage report gets a request ID and idempotency key
+    // so retries (in-memory or via pending-usage.json) are safe to replay.
+    const requestId = randomUUID();
+    const idempotencyKey = randomUUID();
+
+    const payload = {
+      tool,
+      creditsUsed,
+      requestData,
+      responseStatus,
+      processingTime,
+      timestamp: new Date().toISOString(),
+      requestId,
+      idempotencyKey,
+      version: '3.0.3'
+    };
+
+    try {
       await fetch(`${this.apiEndpoint}/api/v1/usage`, {
         method: 'POST',
         headers: {
           'Content-Type': 'application/json',
-          'X-API-Key': this.config.apiKey
+          'X-API-Key': this.config.apiKey,
+          'Idempotency-Key': idempotencyKey
         },
-        body: JSON.stringify(payload)
+        body: JSON.stringify(payload),
+        signal: AbortSignal.timeout(5000)
       });
 
-      // Update cached credits
-      const cached = this.creditCache.get(this.config.userId);
-      if (cached !== undefined) {
-        this.creditCache.set(this.config.userId, Math.max(0, cached - creditsUsed));
-      }
+      await this._flushPendingUsage();
     } catch (error) {
       // Log but don't throw - usage reporting should not break tool execution
-      console.error('Failed to report usage:', error.message);
+      logger.error('Failed to report usage; queued for retry', error, {
+        tool,
+        creditsUsed,
+        requestId,
+        idempotencyKey
+      });
+      await this._appendPendingUsage({
+        toolName: tool,
+        creditsUsed,
+        userId,
+        timestamp: payload.timestamp,
+        requestId,
+        idempotencyKey
+      });
+    }
+  }
+
+  async _appendPendingUsage(entry) {
+    try {
+      const configDir = path.dirname(this.pendingUsagePath);
+      await fs.mkdir(configDir, { recursive: true });
+
+      let entries = [];
+      try {
+        const raw = await fs.readFile(this.pendingUsagePath, 'utf-8');
+        entries = JSON.parse(raw);
+      } catch {
+        // File absent or corrupt — start fresh
+      }
+
+      // Audit phase A2: stamp every pending entry with a request ID and idempotency key
+      // so the backend (when it ships support) can dedupe, and so we can log dropped
+      // entries by ID when the flush retry path fails permanently.
+      const stamped = {
+        requestId: entry.requestId || randomUUID(),
+        idempotencyKey: entry.idempotencyKey || randomUUID(),
+        ...entry
+      };
+
+      entries.push(stamped);
+
+      // Cap at 1 MB — drop oldest entries until serialized size fits
+      let serialized = JSON.stringify(entries);
+      const dropped = [];
+      while (serialized.length > 1_048_576 && entries.length > 1) {
+        dropped.push(entries.shift());
+        serialized = JSON.stringify(entries);
+      }
+      if (dropped.length > 0) {
+        logger.warn('Pending usage queue truncated to 1 MB cap', {
+          droppedCount: dropped.length,
+          droppedIds: dropped.map(d => d.requestId).filter(Boolean)
+        });
+      }
+
+      await fs.writeFile(this.pendingUsagePath, serialized, { mode: 0o600 });
+    } catch (error) {
+      logger.error('Failed to append pending usage', error, {
+        toolName: entry?.toolName,
+        requestId: entry?.requestId
+      });
+    }
+  }
+
+  async _flushPendingUsage() {
+    if (!this.config) return;
+
+    let entries;
+    try {
+      const raw = await fs.readFile(this.pendingUsagePath, 'utf-8');
+      entries = JSON.parse(raw);
+    } catch (err) {
+      // ENOENT is normal (nothing pending). Anything else is corruption — log it.
+      if (err && err.code !== 'ENOENT') {
+        logger.warn('Pending usage file unreadable; treating as empty', {
+          error: err.message,
+          path: this.pendingUsagePath
+        });
+      }
+      return;
+    }
+
+    if (!Array.isArray(entries) || entries.length === 0) return;
+
+    const remaining = [];
+    const flushedIds = [];
+    const failedIds = [];
+    for (const entry of entries) {
+      try {
+        const idempotencyKey = entry.idempotencyKey || randomUUID();
+        await fetch(`${this.apiEndpoint}/api/v1/usage`, {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'X-API-Key': this.config.apiKey,
+            'Idempotency-Key': idempotencyKey
+          },
+          body: JSON.stringify({
+            tool: entry.toolName,
+            creditsUsed: entry.creditsUsed,
+            timestamp: entry.timestamp,
+            requestId: entry.requestId,
+            idempotencyKey,
+            version: '3.0.3'
+          }),
+          signal: AbortSignal.timeout(5000)
+        });
+        flushedIds.push(entry.requestId);
+      } catch (err) {
+        failedIds.push(entry.requestId);
+        remaining.push(entry);
+      }
+    }
+
+    if (flushedIds.length > 0) {
+      logger.info('Flushed pending usage entries', {
+        count: flushedIds.length,
+        requestIds: flushedIds.filter(Boolean)
+      });
+    }
+    if (failedIds.length > 0) {
+      logger.warn('Pending usage entries failed to flush; retained for next attempt', {
+        count: failedIds.length,
+        requestIds: failedIds.filter(Boolean)
+      });
+    }
+
+    try {
+      if (remaining.length === 0) {
+        await fs.unlink(this.pendingUsagePath);
+      } else {
+        await fs.writeFile(this.pendingUsagePath, JSON.stringify(remaining), { mode: 0o600 });
+      }
+    } catch (error) {
+      logger.error('Failed to update pending usage file', error, {
+        path: this.pendingUsagePath
+      });
     }
   }
 
@@ -287,7 +482,10 @@ class AuthManager {
       track_changes: 3,
       
       // Phase 1: LLM-Powered Structured Extraction
-      extract_structured: 4
+      extract_structured: 4,
+
+      // Phase C5: Natural-language LLM extraction (external paid API call per invocation)
+      extract_with_llm: 5
     };
 
     return costs[tool] || 1;

package/src/core/BrowserContextPool.js

@@ -0,0 +1,187 @@
+/**
+ * BrowserContextPool — bounded Playwright browser-context pool.
+ *
+ * Replaces the unbounded `this.contexts = new Map()` in StealthBrowserManager
+ * with a pool that:
+ *   - enforces a hard cap (MAX_BROWSER_CONTEXTS, default 10)
+ *   - disposes contexts after N uses (periodicRefreshAfter, default 200)
+ *   - closes idle contexts after a configurable timeout
+ *   - maintains a concurrency wait-queue so excess callers fail fast
+ *     (timeout: waitTimeoutMs, default 10 000 ms) rather than accumulating
+ *
+ * The Map-compatible surface (get/set/delete/entries/clear/size) lets
+ * StealthBrowserManager adopt it with minimal changes.
+ */
+
+const DEFAULT_MAX_CONTEXTS = parseInt(process.env.MAX_BROWSER_CONTEXTS || '10', 10);
+const DEFAULT_PERIODIC_REFRESH_AFTER = 200;   // context uses before forced close+relaunch
+const DEFAULT_CLOSE_IDLE_AFTER_MS = 5 * 60 * 1000; // 5 minutes
+const DEFAULT_WAIT_TIMEOUT_MS = 10_000;
+
+export class BrowserContextPool {
+  /**
+   * @param {Object} [opts]
+   * @param {number}   [opts.maxContexts]
+   * @param {number}   [opts.periodicRefreshAfter] — max uses per context before disposal
+   * @param {number}   [opts.closeIdleAfterMs]
+   * @param {number}   [opts.waitTimeoutMs] — max wait for a free slot; fails fast after
+   * @param {Function} [opts.onContextExpired] — async (contextId, contextData) => void
+   */
+  constructor(opts = {}) {
+    this._maxContexts = opts.maxContexts ?? DEFAULT_MAX_CONTEXTS;
+    this._periodicRefreshAfter = opts.periodicRefreshAfter ?? DEFAULT_PERIODIC_REFRESH_AFTER;
+    this._closeIdleAfterMs = opts.closeIdleAfterMs ?? DEFAULT_CLOSE_IDLE_AFTER_MS;
+    this._waitTimeoutMs = opts.waitTimeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS;
+    this._onContextExpired = opts.onContextExpired || null;
+
+    /** @type {Map<string, { context: any, fingerprint: any, config: any, uses: number, lastUsed: number, created: number }>} */
+    this._contexts = new Map();
+
+    /** Pending callers waiting for a free slot */
+    this._waitQueue = [];
+
+    /** Periodic idle-checker timer */
+    this._idleTimer = setInterval(() => this._closeIdleContexts(), Math.min(this._closeIdleAfterMs, 60_000));
+    this._idleTimer.unref?.(); // don't prevent process exit
+  }
+
+  // ── Map-compatible surface ──────────────────────────────────────────────────
+
+  get size() { return this._contexts.size; }
+
+  get(contextId) { return this._contexts.get(contextId) ?? undefined; }
+
+  has(contextId) { return this._contexts.has(contextId); }
+
+  entries() { return this._contexts.entries(); }
+
+  keys() { return this._contexts.keys(); }
+
+  values() { return this._contexts.values(); }
+
+  /**
+   * Register a context.  Throws if the pool is full and no slot becomes
+   * available within waitTimeoutMs.
+   */
+  async set(contextId, contextData) {
+    if (this._contexts.size >= this._maxContexts) {
+      await this._waitForSlot();
+    }
+    this._contexts.set(contextId, {
+      ...contextData,
+      uses: 0,
+      lastUsed: Date.now(),
+      created: Date.now()
+    });
+    return this;
+  }
+
+  /**
+   * Synchronous set — for callers that already verified there is capacity.
+   * Throws immediately if pool is at capacity.
+   */
+  setSync(contextId, contextData) {
+    if (this._contexts.size >= this._maxContexts) {
+      throw new Error(`BrowserContextPool is at capacity (${this._maxContexts} contexts). Use await pool.set() to wait for a free slot.`);
+    }
+    this._contexts.set(contextId, {
+      ...contextData,
+      uses: 0,
+      lastUsed: Date.now(),
+      created: Date.now()
+    });
+    return this;
+  }
+
+  delete(contextId) {
+    const deleted = this._contexts.delete(contextId);
+    if (deleted) this._notifyWaiter();
+    return deleted;
+  }
+
+  clear() {
+    this._contexts.clear();
+    // Drain any waiters with rejections so they don't hang
+    const waiters = this._waitQueue.splice(0);
+    waiters.forEach(({ reject }) => reject(new Error('BrowserContextPool cleared')));
+  }
+
+  // ── Pool-specific API ───────────────────────────────────────────────────────
+
+  /**
+   * Record a use for the given context.
+   * Returns true if the context should be closed and re-created (refresh needed).
+   * @param {string} contextId
+   */
+  recordUse(contextId) {
+    const entry = this._contexts.get(contextId);
+    if (!entry) return false;
+    entry.uses++;
+    entry.lastUsed = Date.now();
+    return entry.uses >= this._periodicRefreshAfter;
+  }
+
+  /**
+   * Dispose a context (close it + remove from pool).
+   * @param {string} contextId
+   */
+  async dispose(contextId) {
+    const entry = this._contexts.get(contextId);
+    if (!entry) return;
+    this._contexts.delete(contextId);
+    this._notifyWaiter();
+    try {
+      await entry.context?.close();
+    } catch {
+      // ignore close errors
+    }
+    this._onContextExpired?.(contextId, entry);
+  }
+
+  /**
+   * Close all idle contexts (lastUsed > closeIdleAfterMs ago).
+   */
+  async _closeIdleContexts() {
+    const now = Date.now();
+    for (const [contextId, entry] of this._contexts.entries()) {
+      if (now - entry.lastUsed > this._closeIdleAfterMs) {
+        await this.dispose(contextId);
+      }
+    }
+  }
+
+  /**
+   * Wait until a slot becomes available (or time out).
+   */
+  _waitForSlot() {
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        const idx = this._waitQueue.findIndex(w => w.resolve === resolve);
+        if (idx !== -1) this._waitQueue.splice(idx, 1);
+        reject(new Error(`BrowserContextPool: timed out waiting for a free context slot after ${this._waitTimeoutMs}ms`));
+      }, this._waitTimeoutMs);
+
+      this._waitQueue.push({ resolve, reject, timer });
+    });
+  }
+
+  /** Notify the oldest pending waiter that a slot is now free. */
+  _notifyWaiter() {
+    if (this._waitQueue.length === 0) return;
+    const { resolve, timer } = this._waitQueue.shift();
+    clearTimeout(timer);
+    resolve();
+  }
+
+  /** Destroy the pool — closes all contexts and clears the idle timer. */
+  async destroy() {
+    clearInterval(this._idleTimer);
+    for (const contextId of Array.from(this._contexts.keys())) {
+      await this.dispose(contextId);
+    }
+    const waiters = this._waitQueue.splice(0);
+    waiters.forEach(({ reject }) => reject(new Error('BrowserContextPool destroyed')));
+  }
+}
+
+export default BrowserContextPool;

package/src/core/JobManager.js

@@ -573,11 +573,13 @@ export class JobManager extends EventEmitter {
    * @returns {boolean} Whether job is valid
    */
   validateJob(job) {
-    return job &&
-           typeof job.id === 'string' &&
-           typeof job.type === 'string' &&
-           typeof job.status === 'string' &&
-           Object.values(this.JOB_STATES).includes(job.status);
+    return Boolean(
+      job &&
+      typeof job.id === 'string' &&
+      typeof job.type === 'string' &&
+      typeof job.status === 'string' &&
+      Object.values(this.JOB_STATES).includes(job.status)
+    );
   }
 
   /**