crawlforge-mcp-server 4.9.0 → 5.0.0

package/src/core/ActionExecutor.js

@@ -7,6 +7,7 @@ import { z } from 'zod';
 import BrowserProcessor from './processing/BrowserProcessor.js';
 import { EventEmitter } from 'events';
 import { createHash } from 'node:crypto';
+import { assertUrlAllowed } from '../utils/ssrfGuard.js';
 
 // executeJavaScript hardening limits (only relevant when the deploy-time flag
 // ALLOW_JAVASCRIPT_EXECUTION=true is set; JS execution stays off by default).
@@ -19,7 +20,11 @@ const BaseActionSchema = z.object({
   timeout: z.number().optional(),
   description: z.string().optional(),
   continueOnError: z.boolean().default(false),
-  retries: z.number().min(0).max(5).default(0)
+  retries: z.number().min(0).max(5).default(0),
+  // When true, capture page state (page.content()/page.url()) natively right
+  // after this action executes. Does not use in-page JS execution, so it
+  // works regardless of the ALLOW_JAVASCRIPT_EXECUTION flag.
+  captureAfter: z.boolean().default(false)
 });
 
 const WaitActionSchema = BaseActionSchema.extend({
@@ -162,7 +167,10 @@ export class ActionExecutor extends EventEmitter {
   async executeActionChain(url, chainConfig, browserOptions = {}) {
     const startTime = Date.now();
     const chainId = this.generateChainId();
-    
+    // Declared here (not inside the try block below) so the outer catch can
+    // still report partial results/screenshots/capturedStates on failure.
+    let executionContext = null;
+
     try {
       // Handle simplified signature: executeActionChain(url, actionsArray)
       let actualChainConfig;
@@ -186,7 +194,7 @@ export class ActionExecutor extends EventEmitter {
       this.stats.totalChains++;
       
       // Create execution context
-      const executionContext = {
+      executionContext = {
         id: chainId,
         url,
         chain: validatedChain,
@@ -253,11 +261,20 @@ export class ActionExecutor extends EventEmitter {
 
         throw error;
       } finally {
-        // D2.4: always close page to prevent leaks
+        // D2.4: always close page to prevent leaks. Also close the owning
+        // context for non-stealth pages — createPage() gives each call its
+        // own dedicated BrowserContext that is never tracked/closed
+        // elsewhere, so leaving it open here leaks it until server shutdown.
+        // Stealth contexts are pooled/reused (see StealthBrowserManager) and
+        // must not be closed here.
         if (page) {
+          const ctx = !browserOptions.stealthMode?.enabled ? page.context() : null;
           try { await page.close(); } catch (_) { /* ignore close errors */ }
+          if (ctx) {
+            try { await ctx.close(); } catch (_) { /* ignore close errors */ }
+          }
         }
-        
+
         // Update execution time
         const executionTime = Date.now() - startTime;
         executionContext.executionTime = executionTime;
@@ -266,10 +283,25 @@ export class ActionExecutor extends EventEmitter {
         // Remove from active chains
         this.activeChains.delete(chainId);
         
-        // Add to execution history
+        // Add to execution history. Strip finalHtml (full post-action page
+        // HTML, often 100KB-2MB), screenshots (base64 PNGs), capturedStates
+        // (full intermediate-page HTML), and each screenshot action's base64
+        // payload inside results — getExecutionHistory() only reads scalar
+        // fields and results[].success, so retaining the heavy payloads just
+        // pins them in memory for the life of the 100-entry history.
         this.executionHistory.push({
           ...executionContext,
-          page: undefined // Don't store page in history
+          page: undefined, // Don't store page in history
+          finalHtml: undefined,
+          screenshots: undefined,
+          screenshotCount: executionContext.screenshots.length,
+          capturedStates: undefined,
+          capturedStateCount: (executionContext.capturedStates || []).length,
+          results: executionContext.results.map(r => (
+            r?.result?.data !== undefined
+              ? { ...r, result: { ...r.result, data: undefined, dataBytes: typeof r.result.data === 'string' ? r.result.data.length : undefined } }
+              : r
+          ))
         });
         
         // Keep only last 100 executions in history
@@ -289,6 +321,7 @@ export class ActionExecutor extends EventEmitter {
         executionTime: Date.now() - startTime,
         results: executionContext.results,
         screenshots: executionContext.screenshots,
+        capturedStates: executionContext.capturedStates || [],
         metadata: executionContext.metadata,
         stats: {
           totalActions: executionContext.results.length,
@@ -305,8 +338,12 @@ export class ActionExecutor extends EventEmitter {
         url,
         executionTime: Date.now() - startTime,
         error: error.message,
-        results: [],
-        screenshots: []
+        // Preserve whatever was captured before the failure (per-action
+        // results, the error screenshot, and any intermediate-state
+        // captures) instead of discarding them.
+        results: executionContext?.results || [],
+        screenshots: executionContext?.screenshots || [],
+        capturedStates: executionContext?.capturedStates || []
       };
     }
   }
@@ -325,6 +362,7 @@ export class ActionExecutor extends EventEmitter {
         if (attempt > 0) {
           this.log('info', 'Retrying chain execution, attempt ' + (attempt + 1));
           executionContext.results = []; // Clear previous results on retry
+          executionContext.capturedStates = []; // Clear previous captures on retry
         }
 
         // Execute actions in sequence
@@ -358,6 +396,26 @@ export class ActionExecutor extends EventEmitter {
             }
           }
 
+          // Native intermediate-state capture: page.content()/page.url()
+          // directly (no in-page JS execution), so it works regardless of
+          // the ALLOW_JAVASCRIPT_EXECUTION flag and doesn't add phantom
+          // actions to the chain's failure/success counts.
+          if (action.captureAfter) {
+            try {
+              const capturedHtml = await page.content();
+              executionContext.capturedStates = executionContext.capturedStates || [];
+              executionContext.capturedStates.push({
+                afterActionIndex: i,
+                afterActionId: actionResult.id,
+                url: page.url(),
+                html: capturedHtml,
+                timestamp: Date.now()
+              });
+            } catch (captureErr) {
+              this.log('warn', 'Failed to capture intermediate state: ' + captureErr.message);
+            }
+          }
+
           // Add delay between actions
           if (i < chain.actions.length - 1 && this.actionDelay > 0) {
             await this.delay(this.actionDelay);
@@ -814,33 +872,59 @@ export class ActionExecutor extends EventEmitter {
    * @returns {Promise<Page>} Playwright page
    */
   async initializePage(url, browserOptions) {
+    // SSRF guard: validate before any page/context creation or navigation.
+    // resolveDns:true because Playwright does its own DNS resolution, so
+    // hostname-based checks alone would miss DNS-rebinding/private-IP targets.
+    await assertUrlAllowed(url, { resolveDns: true });
+
+    const isStealth = !!browserOptions.stealthMode?.enabled;
+
     // Use the enhanced BrowserProcessor initialization that supports stealth mode
     const page = await this.browserProcessor.initializePage(browserOptions);
-    
-    // Apply CloudFlare and reCAPTCHA detection if stealth mode is enabled
-    if (browserOptions.stealthMode?.enabled && this.browserProcessor.stealthManager) {
-      // Initialize human behavior simulator for the page
-      await this.browserProcessor.stealthManager.initializeHumanBehaviorSimulator();
-    }
-    
-    // Navigate to URL
-    await page.goto(url, {
-      waitUntil: 'domcontentloaded',
-      timeout: 30000
-    });
-    
-    // Handle CloudFlare challenges and reCAPTCHA if stealth mode is enabled
-    if (browserOptions.stealthMode?.enabled && this.browserProcessor.stealthManager) {
-      await this.browserProcessor.stealthManager.bypassCloudflareChallenge(page);
-      await this.browserProcessor.stealthManager.handleRecaptcha(page);
-      
-      // Simulate initial human behavior on page load
-      if (browserOptions.humanBehavior?.enabled) {
-        await this.simulateInitialPageInteraction(page);
+
+    try {
+      // Apply CloudFlare and reCAPTCHA detection if stealth mode is enabled
+      if (isStealth && this.browserProcessor.stealthManager) {
+        // Initialize human behavior simulator for the page
+        await this.browserProcessor.stealthManager.initializeHumanBehaviorSimulator();
       }
-    }
 
-    return page;
+      // Navigate to URL
+      await page.goto(url, {
+        waitUntil: 'domcontentloaded',
+        timeout: 30000
+      });
+
+      // Re-validate the landed URL: a redirect during navigation could have
+      // taken us into a blocked range even though the original URL was safe.
+      const landedUrl = page.url();
+      if (/^https?:\/\//i.test(landedUrl)) {
+        await assertUrlAllowed(landedUrl, { resolveDns: true });
+      }
+
+      // Handle CloudFlare challenges and reCAPTCHA if stealth mode is enabled
+      if (isStealth && this.browserProcessor.stealthManager) {
+        await this.browserProcessor.stealthManager.bypassCloudflareChallenge(page);
+        await this.browserProcessor.stealthManager.handleRecaptcha(page);
+
+        // Simulate initial human behavior on page load
+        if (browserOptions.humanBehavior?.enabled) {
+          await this.simulateInitialPageInteraction(page);
+        }
+      }
+
+      return page;
+    } catch (error) {
+      // Any failure between page creation and return (navigation, SSRF
+      // re-check, stealth challenge handling) must not leak the page it
+      // already created — close it, and its dedicated context for
+      // non-stealth pages (stealth contexts are pooled/reused elsewhere),
+      // before rethrowing.
+      const ctx = !isStealth ? page.context() : null;
+      await page.close().catch(() => {});
+      if (ctx) await ctx.close().catch(() => {});
+      throw error;
+    }
   }
   
   /**
@@ -1142,7 +1226,7 @@ export class ActionExecutor extends EventEmitter {
     // Cancel active chains
     for (const context of this.activeChains.values()) {
       if (context.page) {
-        await context.page.close();
+        try { await context.page.close(); } catch (_) { /* ignore close errors */ }
       }
     }
 

package/src/core/AgentOrchestrator.js

@@ -209,18 +209,24 @@ export class AgentOrchestrator {
     }
 
     // ── ACT loop ──────────────────────────────────────────────────────────────
+    // urlsFetched (capUrls) and step (capSteps) are deliberately decoupled:
+    // urlsFetched counts every fetch attempt (gates how many URLs we try),
+    // while step counts only attempts that yielded usable evidence (gates
+    // genuine progress). Coupling them 1:1 made capSteps the always-binding
+    // cap whenever it was smaller than capUrls, leaving maxUrls unreachable
+    // at its default. Both caps remain fully enforced (neither is weakened).
     const evidence = [];
     let urlsFetched = 0;
     let step = 0;
 
     for (const url of urlQueue) {
-      if (step >= capSteps || urlsFetched >= capUrls || deadline()) break;
-      step++;
+      if (urlsFetched >= capUrls || step >= capSteps || deadline()) break;
       urlsFetched++;
 
       try {
         const { textContent, finalUrl } = await fetchAndParse(url, { timeoutMs: 10000 });
         if (!isRelevant(textContent, prompt)) continue;
+        step++;
         evidence.push({
           url: finalUrl,
           text: truncate(textContent),

package/src/core/AuthManager.js

@@ -10,6 +10,7 @@ import { randomUUID } from 'crypto';
 import { isCreatorModeVerified } from './creatorMode.js';
 import { resolveApiEndpoint } from './endpointGuard.js';
 import { logger } from '../utils/Logger.js';
+import { maskSecrets } from '../utils/secretMask.js';
 // D1.4: Elicitation for low-credit warnings (lazy import to avoid circular dep)
 let _ElicitationHelper = null;
 function getElicitationHelper() {
@@ -166,30 +167,30 @@ class AuthManager {
    * Setup wizard for first-time users
    */
   async runSetup(apiKey) {
-    console.log('🔧 Setting up CrawlForge MCP Server...\n');
-    
+    console.error('🔧 Setting up CrawlForge MCP Server...\n');
+
     if (!apiKey) {
-      console.log('❌ API key is required for setup');
-      console.log('Get your API key from: https://www.crawlforge.dev/dashboard/api-keys');
+      console.error('❌ API key is required for setup');
+      console.error('Get your API key from: https://www.crawlforge.dev/dashboard/api-keys');
       return false;
     }
 
     // Validate API key with backend
     const validation = await this.validateApiKey(apiKey);
-    
+
     if (!validation.valid) {
-      console.log(`❌ Invalid API key: ${validation.error}`);
+      console.error(`❌ Invalid API key: ${validation.error}`);
       return false;
     }
 
     // Save configuration
     await this.saveConfig(apiKey, validation.userId, validation.email);
-    
-    console.log('✅ Setup complete!');
-    console.log(`📧 Account: ${validation.email}`);
-    console.log(`💳 Credits remaining: ${validation.creditsRemaining}`);
-    console.log(`📦 Plan: ${validation.planId}`);
-    
+
+    console.error('✅ Setup complete!');
+    console.error(`📧 Account: ${validation.email}`);
+    console.error(`💳 Credits remaining: ${validation.creditsRemaining}`);
+    console.error(`📦 Plan: ${validation.planId}`);
+
     return true;
   }
 
@@ -286,7 +287,31 @@ class AuthManager {
 
         return data.creditsRemaining >= estimatedCredits;
       }
+
+      // Non-OK response: distinguish an invalid/revoked key from a transient
+      // backend problem so callers don't tell a locked-out user to buy credits.
+      if (response.status === 401 || response.status === 403) {
+        let message = 'API key is invalid or has been revoked.';
+        try {
+          const errBody = await response.json();
+          if (errBody?.message) message = errBody.message;
+        } catch {
+          // body not JSON / unreadable — keep default message
+        }
+        const invalidKeyError = new Error(
+          `CrawlForge API key rejected (${response.status}): ${message} Run \`npm run setup\` with a current key.`
+        );
+        invalidKeyError.code = 'CRAWLFORGE_KEY_INVALID';
+        throw invalidKeyError;
+      }
+
+      // Other non-OK statuses (5xx etc.) — treat like a network failure below.
+      throw new Error(`CrawlForge backend returned ${response.status} while checking credits.`);
     } catch (error) {
+      if (error?.code === 'CRAWLFORGE_KEY_INVALID') {
+        throw error;
+      }
+
       console.error('Failed to check credits:', error.message);
 
       const lastOk = this.lastSuccessfulCreditCheck.get(this.config.userId) ?? 0;
@@ -335,7 +360,9 @@ class AuthManager {
     const payload = {
       tool,
       creditsUsed,
-      requestData,
+      // Never send raw tool params to the backend — they can carry third-party
+      // API keys, auth headers, or webhook secrets. Mask before it leaves the process.
+      requestData: maskSecrets(requestData),
       responseStatus,
       processingTime,
       timestamp: new Date().toISOString(),
@@ -345,7 +372,7 @@ class AuthManager {
     };
 
     try {
-      await fetch(`${this.apiEndpoint}/api/v1/usage`, {
+      const response = await fetch(`${this.apiEndpoint}/api/v1/usage`, {
         method: 'POST',
         headers: {
           'Content-Type': 'application/json',
@@ -356,6 +383,10 @@ class AuthManager {
         signal: AbortSignal.timeout(5000)
       });
 
+      if (!response.ok) {
+        throw new Error(`Usage report rejected by backend: HTTP ${response.status}`);
+      }
+
       await this._flushPendingUsage();
     } catch (error) {
       // Log but don't throw - usage reporting should not break tool execution
@@ -449,7 +480,7 @@ class AuthManager {
     for (const entry of entries) {
       try {
         const idempotencyKey = entry.idempotencyKey || randomUUID();
-        await fetch(`${this.apiEndpoint}/api/v1/usage`, {
+        const response = await fetch(`${this.apiEndpoint}/api/v1/usage`, {
           method: 'POST',
           headers: {
             'Content-Type': 'application/json',
@@ -466,6 +497,9 @@ class AuthManager {
           }),
           signal: AbortSignal.timeout(5000)
         });
+        if (!response.ok) {
+          throw new Error(`Usage report rejected by backend: HTTP ${response.status}`);
+        }
         flushedIds.push(entry.requestId);
       } catch (err) {
         failedIds.push(entry.requestId);
@@ -599,7 +633,7 @@ class AuthManager {
         break;
       }
       case 'crawl_deep': {
-        const maxPages = params?.maxPages || params?.options?.maxPages || 10;
+        const maxPages = params?.max_pages || params?.maxPages || params?.options?.maxPages || 10;
         projected = Math.max(base, Math.ceil(maxPages / 20) * base);
         note = `Lower-bound estimate. crawl_deep cost grows with page count (${maxPages} max).`;
         break;
@@ -657,7 +691,7 @@ class AuthManager {
       await fs.unlink(this.configPath);
       this.config = null;
       this.creditCache.clear();
-      console.log('Configuration cleared.');
+      console.error('Configuration cleared.');
     } catch (error) {
       console.error('Failed to clear configuration:', error.message);
     }

package/src/core/ChangeTracker.js

@@ -11,6 +11,7 @@ import { z } from 'zod';
 import { EventEmitter } from 'events';
 import { load } from 'cheerio';
 import { diffWords, diffLines, diffChars } from 'diff';
+import { calculateSimilarity as calculateContentSimilarity } from '../tools/tracking/trackChanges/differ.js';
 
 const ChangeTrackingSchema = z.object({
   url: z.string().url(),
@@ -169,9 +170,13 @@ export class ChangeTracker extends EventEmitter {
    * @param {string} url - URL to compare
    * @param {string} currentContent - Current content
    * @param {Object} options - Comparison options
+   * @param {Object} storageOptions - History retention overrides for this
+   *   call: retainHistory (boolean) and maxHistoryEntries (number), as
+   *   forwarded from TrackChangesSchema.storageOptions. Both optional —
+   *   falls back to this.options.maxHistoryLength when omitted.
    * @returns {Object} - Change analysis results
    */
-  async compareWithBaseline(url, currentContent, options = {}) {
+  async compareWithBaseline(url, currentContent, options = {}, storageOptions = {}) {
     const startTime = Date.now();
 
     // Expected no-baseline case: return a clean error WITHOUT emitting an
@@ -226,10 +231,19 @@ export class ChangeTracker extends EventEmitter {
       
       changeRecord.processingTime = Date.now() - startTime;
       
-      // Store change record
-      const changeHistory = this.changeHistory.get(url);
-      changeHistory.push(changeRecord);
-      
+      // Store change record, trimmed to maxHistoryEntries/maxHistoryLength so
+      // a long-running monitor doesn't accumulate a full diff record
+      // (word/line-level diff arrays included) per check for the life of the
+      // process. retainHistory:false skips storage entirely.
+      if (storageOptions.retainHistory !== false) {
+        const changeHistory = this.changeHistory.get(url);
+        changeHistory.push(changeRecord);
+        const maxHistoryLength = storageOptions.maxHistoryEntries ?? this.options.maxHistoryLength;
+        if (maxHistoryLength && changeHistory.length > maxHistoryLength) {
+          changeHistory.splice(0, changeHistory.length - maxHistoryLength);
+        }
+      }
+
       // Update statistics
       this.updateStats(changeRecord);
       
@@ -345,11 +359,13 @@ export class ChangeTracker extends EventEmitter {
       linkChanges: []
     };
     
-    // Calculate overall content similarity
-    changes.similarity = this.calculateSimilarity(
-      baseline.hashes.page,
-      current.hashes.page
-    );
+    // Calculate overall content similarity. Hash equality is used only as the
+    // fast identical/changed test — the hashes themselves are not comparable
+    // (a single-character edit changes ~every hex digit), so an actual
+    // similarity score is computed against the original content.
+    changes.similarity = baseline.hashes.page === current.hashes.page
+      ? 1
+      : calculateContentSimilarity(baseline.originalContent, current.originalContent);
     
     // Detect structural changes
     if (options.trackStructure) {

package/src/core/JobManager.js

@@ -317,8 +317,16 @@ export class JobManager extends EventEmitter {
 
     try {
       const result = await executor(job);
+
+      // A cancelJob() call can flip status to 'cancelled' while the executor
+      // await above was still in flight (cancelJob can't interrupt it
+      // directly). Don't clobber that terminal state back to 'completed'.
+      if (job.status === this.JOB_STATES.CANCELLED) {
+        return result;
+      }
+
       await this.updateJobStatus(jobId, this.JOB_STATES.COMPLETED, { result });
-      
+
       // Calculate execution time
       const executionTime = job.completedAt - job.startedAt;
       this.updateExecutionTime(executionTime);

package/src/core/LocalizationManager.js

@@ -984,19 +984,31 @@ export class LocalizationManager extends EventEmitter {
    * Setup periodic health checks for proxies and services
    */
   setupHealthChecks() {
-    // Proxy health checks every 5 minutes
-    setInterval(async () => {
+    // Proxy health checks every 5 minutes. Handles are recorded so cleanup()
+    // can actually clear them — previously they were never stored, so the
+    // `clearInterval` in cleanup() had nothing to clear and both intervals
+    // kept firing (and kept the process alive) after cleanup().
+    const proxyInterval = setInterval(async () => {
       if (this.proxyManager.activeProxies.size > 0) {
         await this.performProxyHealthChecks();
       }
     }, 300000);
-    
+
     // Translation service health checks every 10 minutes
-    setInterval(async () => {
+    const translationInterval = setInterval(async () => {
       if (this.translationProviders.size > 0) {
         await this.checkTranslationServiceHealth();
       }
     }, 600000);
+
+    this.healthCheckIntervals = [proxyInterval, translationInterval];
+
+    // Background health checks must never be the only thing keeping the
+    // process alive — unref() so an otherwise-idle process (tests, CLI
+    // one-shots that construct a tool without calling cleanup()) can exit.
+    for (const interval of this.healthCheckIntervals) {
+      if (typeof interval.unref === 'function') interval.unref();
+    }
   }
 
   /**
@@ -1497,8 +1509,9 @@ export class LocalizationManager extends EventEmitter {
       this.resetStats();
       
       // Clear all health check intervals
-      if (this.healthCheckInterval) {
-        clearInterval(this.healthCheckInterval);
+      if (this.healthCheckIntervals) {
+        this.healthCheckIntervals.forEach(clearInterval);
+        this.healthCheckIntervals = null;
       }
       
       // Reset proxy manager