crawlforge-mcp-server 4.9.0 → 5.0.0

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -392,7 +392,7 @@ export class ScrapeWithActionsTool extends EventEmitter {
     // Process action results
     const actionResults = this.processActionResults(chainResult.results);
     const intermediateStates = params.captureIntermediateStates ?
-      await this.extractIntermediateStates(actionResults, params) : [];
+      await this.extractIntermediateStates(chainResult.capturedStates || [], params) : [];
 
     // Get final page content after all actions (reads the post-action live page
     // captured by ActionExecutor, falling back to a fresh fetch only if missing).
@@ -440,7 +440,7 @@ export class ScrapeWithActionsTool extends EventEmitter {
         formAutoFillApplied: !!params.formAutoFill,
         intermediateStatesCount: intermediateStates.length,
         screenshotsCount: sessionContext.screenshots.length,
-        finalUrl: chainResult.metadata?.finalUrl,
+        finalUrl: chainResult?.finalUrl,
         timestamp: Date.now()
       },
 
@@ -523,23 +523,17 @@ export class ScrapeWithActionsTool extends EventEmitter {
   }
 
   insertCaptureActions(actions) {
-    const modifiedActions = [];
-
-    actions.forEach((action, index) => {
-      modifiedActions.push(action);
-
+    // Mark actions for native capture (page.content()/page.url() in
+    // ActionExecutor) instead of injecting synthetic executeJavaScript
+    // actions — avoids depending on ALLOW_JAVASCRIPT_EXECUTION (off by
+    // default) and keeps action/failure counts matching the user's own
+    // actions rather than being inflated by injected steps.
+    return actions.map(action => {
       if (this.shouldCaptureAfterAction(action) || action.captureAfter) {
-        modifiedActions.push({
-          type: 'executeJavaScript',
-          script: `return {url: window.location.href, title: document.title, html: document.documentElement.outerHTML, timestamp: Date.now(), capturePoint: ${index + 1}};`,
-          description: `Capture state after action ${index + 1}`,
-          returnResult: true,
-          continueOnError: true
-        });
+        return { ...action, captureAfter: true };
       }
+      return action;
     });
-
-    return modifiedActions;
   }
 
   shouldCaptureAfterAction(action) {
@@ -563,47 +557,44 @@ export class ScrapeWithActionsTool extends EventEmitter {
     }));
   }
 
-  async extractIntermediateStates(actionResults, params) {
+  async extractIntermediateStates(capturedStates, params) {
     const states = [];
 
-    for (const result of actionResults) {
-      if (result.type === 'executeJavaScript' && result.jsResult && result.jsResult.html) {
-        try {
-          const stateData = result.jsResult;
-          const $ = load(stateData.html);
-          
-          const state = {
-            capturePoint: stateData.capturePoint,
-            url: stateData.url,
-            title: stateData.title,
-            timestamp: stateData.timestamp,
-            content: {}
+    for (const stateData of capturedStates) {
+      try {
+        const $ = load(stateData.html);
+
+        const state = {
+          capturePoint: stateData.afterActionIndex + 1,
+          url: stateData.url,
+          title: $('title').text().trim(),
+          timestamp: stateData.timestamp,
+          content: {}
+        };
+
+        if (params.formats.includes('text')) {
+          state.content.text = $('body').text().replace(/\s+/g, ' ').trim();
+        }
+
+        if (params.formats.includes('html')) {
+          state.content.html = stateData.html;
+        }
+
+        if (params.formats.includes('json')) {
+          state.content.json = {
+            title: state.title,
+            headings: this.extractHeadings($),
+            links: this.extractLinks($)
           };
+        }
 
-          if (params.formats.includes('text')) {
-            state.content.text = $('body').text().replace(/\s+/g, ' ').trim();
-          }
-
-          if (params.formats.includes('html')) {
-            state.content.html = stateData.html;
-          }
-
-          if (params.formats.includes('json')) {
-            state.content.json = {
-              title: stateData.title,
-              headings: this.extractHeadings($),
-              links: this.extractLinks($)
-            };
-          }
-
-          if (params.extractionOptions?.selectors) {
-            state.content.extracted = this.extractWithSelectors($, params.extractionOptions.selectors);
-          }
-
-          states.push(state);
-        } catch (error) {
-          this.log('warn', `Failed to process intermediate state: ${error.message}`);
+        if (params.extractionOptions?.selectors) {
+          state.content.extracted = this.extractWithSelectors($, params.extractionOptions.selectors);
         }
+
+        states.push(state);
+      } catch (error) {
+        this.log('warn', `Failed to process intermediate state: ${error.message}`);
       }
     }
 

package/src/tools/advanced/batchScrape/index.js

@@ -34,7 +34,8 @@ export class BatchScrapeTool extends EventEmitter {
       defaultTimeout = 15000,
       maxBatchSize = 50,
       enableResultCaching = true,
-      enableLogging = true
+      enableLogging = true,
+      maxCachedBatches = 20
     } = options;
 
     this.jobManager = jobManager || new JobManager({
@@ -54,9 +55,20 @@ export class BatchScrapeTool extends EventEmitter {
 
     this.activeBatches = new Map();
     this.batchResults = new Map();
+    this.maxCachedBatches = maxCachedBatches;
+    this.resultCacheTtl = 3600000; // 1 hour — matches the ttl stored per cache entry
     // D1.4: Elicitation helper (set mcpServer after instantiation if desired)
     this._elicitation = new ElicitationHelper({});
 
+    // Bound batchResults' lifetime: sweep expired entries periodically (in
+    // addition to the on-read eviction in getBatchResults) so cached results —
+    // including full HTML bodies when formats includes 'html' — don't
+    // accumulate in memory for the life of the process.
+    // .unref() so this timer never blocks process exit on its own — matches
+    // SnapshotManager's cleanupTimer.
+    this._resultsSweepTimer = setInterval(() => this._sweepBatchResults(), 10 * 60 * 1000);
+    if (typeof this._resultsSweepTimer.unref === 'function') this._resultsSweepTimer.unref();
+
     this.stats = {
       totalBatches: 0,
       completedBatches: 0,
@@ -128,15 +140,27 @@ export class BatchScrapeTool extends EventEmitter {
 
   async _processBatchSync(batchId, urlConfigs, validated, webhookConfig, startTime) {
     try {
-      this.activeBatches.set(batchId, { id: batchId, mode: 'sync', startTime, total: urlConfigs.length, completed: 0 });
+      // Process in maxConcurrency-sized chunks (rather than one call covering
+      // all URLs) so `completed` can be updated for progress polling and so
+      // cancelBatch's `cancelled` flag is actually observed between chunks.
+      const activeEntry = { id: batchId, mode: 'sync', startTime, total: urlConfigs.length, completed: 0, cancelled: false };
+      this.activeBatches.set(batchId, activeEntry);
+
+      const rawResults = [];
+      for (let i = 0; i < urlConfigs.length; i += validated.maxConcurrency) {
+        if (activeEntry.cancelled) break;
+        const chunk = urlConfigs.slice(i, i + validated.maxConcurrency);
+        rawResults.push(...await scrapeUrlsBatch(chunk, validated, this.defaultTimeout));
+        activeEntry.completed = rawResults.length;
+      }
+      const wasCancelled = activeEntry.cancelled;
 
-      const rawResults = await scrapeUrlsBatch(urlConfigs, validated, this.defaultTimeout);
       const processedResults = processResults(rawResults, validated);
       const executionTime = Date.now() - startTime;
       this._updateAverageBatchTime(executionTime);
 
       const batchResult = {
-        batchId, mode: 'sync', success: true, executionTime,
+        batchId, mode: 'sync', success: true, cancelled: wasCancelled || undefined, executionTime,
         totalUrls: urlConfigs.length,
         successfulUrls: processedResults.filter(r => r.success).length,
         failedUrls: processedResults.filter(r => !r.success).length,
@@ -151,20 +175,23 @@ export class BatchScrapeTool extends EventEmitter {
       };
 
       if (this.enableResultCaching) {
-        this.batchResults.set(batchId, { results: processedResults, timestamp: Date.now(), ttl: 3600000 });
+        this._cacheBatchResult(batchId, processedResults);
       }
 
       this.stats.completedBatches++;
-      this.stats.totalUrls += urlConfigs.length;
+      this.stats.totalUrls += rawResults.length;
       this.stats.successfulUrls += batchResult.successfulUrls;
       this.stats.failedUrls += batchResult.failedUrls;
       this.stats.lastUpdated = Date.now();
       this.activeBatches.delete(batchId);
 
-      // C3: include webhook delivery status in the result
-      const webhookStatus = await sendWebhookNotification('batch_completed', batchResult, webhookConfig, this.webhookDispatcher, this.enableWebhookNotifications);
-      if (webhookStatus) batchResult.webhookDelivery = webhookStatus;
-      this.emit('batchCompleted', batchResult);
+      // C3: include webhook delivery status in the result (skip when cancelled
+      // early — a 'batch_completed' notification would be misleading).
+      if (!wasCancelled) {
+        const webhookStatus = await sendWebhookNotification('batch_completed', batchResult, webhookConfig, this.webhookDispatcher, this.enableWebhookNotifications);
+        if (webhookStatus) batchResult.webhookDelivery = webhookStatus;
+        this.emit('batchCompleted', batchResult);
+      }
       return batchResult;
     } catch (error) {
       this.stats.failedBatches++;
@@ -195,7 +222,7 @@ export class BatchScrapeTool extends EventEmitter {
         batchId, mode: 'async', jobId: job.id, status: 'queued',
         totalUrls: urlConfigs.length, createdAt: job.createdAt,
         estimatedCompletion: new Date(job.createdAt + (urlConfigs.length * 2000)),
-        statusCheckUrl: `batch_scrape_status?jobId=${job.id}`,
+        statusCheckUrl: `get_batch_results({batchId: "${batchId}"})`,
         webhook: webhookConfig ? { url: webhookConfig.url, events: webhookConfig.events } : null
       };
     } catch (error) {
@@ -206,14 +233,19 @@ export class BatchScrapeTool extends EventEmitter {
 
   async getBatchResults(batchId, page = 1, pageSize = 25) {
     const cached = this.batchResults.get(batchId);
-    if (cached && Date.now() - cached.timestamp < cached.ttl) {
-      const offset = (page - 1) * pageSize;
-      return {
-        batchId, success: true,
-        results: paginateResults(cached.results, offset, pageSize),
-        pagination: { page, pageSize, totalResults: cached.results.length, totalPages: Math.ceil(cached.results.length / pageSize) },
-        cached: true, timestamp: cached.timestamp
-      };
+    if (cached) {
+      if (Date.now() - cached.timestamp < cached.ttl) {
+        const offset = (page - 1) * pageSize;
+        return {
+          batchId, success: true,
+          results: paginateResults(cached.results, offset, pageSize),
+          pagination: { page, pageSize, totalResults: cached.results.length, totalPages: Math.ceil(cached.results.length / pageSize) },
+          cached: true, timestamp: cached.timestamp
+        };
+      }
+      // Expired — evict now instead of waiting for the periodic sweep, and
+      // fall through to the activeBatches/jobManager lookups below.
+      this.batchResults.delete(batchId);
     }
 
     const active = this.activeBatches.get(batchId);
@@ -225,6 +257,33 @@ export class BatchScrapeTool extends EventEmitter {
       };
     }
 
+    // Async batches are never added to activeBatches; look the underlying job
+    // up by its batchId tag so pending/running (and completed-but-uncached)
+    // batches are still reported instead of a misleading "not found".
+    const jobs = this.jobManager.getJobsByTag(batchId);
+    if (jobs.length > 0) {
+      const job = jobs[0];
+
+      if (job.status === 'completed' && job.result) {
+        const results = job.result.results || [];
+        const offset = (page - 1) * pageSize;
+        return {
+          batchId, success: true, jobId: job.id,
+          results: paginateResults(results, offset, pageSize),
+          pagination: { page, pageSize, totalResults: results.length, totalPages: Math.ceil(results.length / pageSize) },
+          cached: false, timestamp: job.completedAt
+        };
+      }
+
+      return {
+        batchId, jobId: job.id, status: job.status, mode: 'async',
+        progress: { percentage: job.progress, total: job.metadata?.urlCount },
+        startTime: job.startedAt || job.createdAt,
+        runningTime: Date.now() - (job.startedAt || job.createdAt),
+        error: job.error || undefined
+      };
+    }
+
     throw new Error(`Batch ${batchId} not found`);
   }
 
@@ -237,9 +296,14 @@ export class BatchScrapeTool extends EventEmitter {
   }
 
   async cancelBatch(batchId) {
-    if (this.activeBatches.has(batchId)) {
-      this.activeBatches.delete(batchId);
-      return { success: true, message: `Active batch ${batchId} cancelled` };
+    const active = this.activeBatches.get(batchId);
+    if (active) {
+      // Signal the running _processBatchSync loop to stop dispatching further
+      // chunks (the in-flight chunk still finishes — there is no per-request
+      // abort). Deleting the entry outright, as before, only hid the batch
+      // from getBatchResults; the scrape loop kept running to completion.
+      active.cancelled = true;
+      return { success: true, message: `Batch ${batchId} cancellation requested; processing stops after the in-flight chunk completes` };
     }
     const jobs = this.jobManager.getJobsByTag(batchId);
     if (jobs.length > 0) {
@@ -266,6 +330,7 @@ export class BatchScrapeTool extends EventEmitter {
     }
     this.activeBatches.clear();
     this.batchResults.clear();
+    if (this._resultsSweepTimer) clearInterval(this._resultsSweepTimer);
     this.jobManager?.destroy();
     this.webhookDispatcher?.destroy();
     this.removeAllListeners();
@@ -297,6 +362,25 @@ export class BatchScrapeTool extends EventEmitter {
     return this.webhookDispatcher.registerWebhook(webhookConfig.url, config);
   }
 
+  /**
+   * Cache a batch's results, capped to maxCachedBatches. Map preserves
+   * insertion order, so the oldest entry is evicted first (LRU by write time).
+   */
+  _cacheBatchResult(batchId, results) {
+    this.batchResults.set(batchId, { results, timestamp: Date.now(), ttl: this.resultCacheTtl });
+    while (this.batchResults.size > this.maxCachedBatches) {
+      const oldestKey = this.batchResults.keys().next().value;
+      this.batchResults.delete(oldestKey);
+    }
+  }
+
+  _sweepBatchResults() {
+    const now = Date.now();
+    for (const [id, entry] of this.batchResults) {
+      if (now - entry.timestamp >= entry.ttl) this.batchResults.delete(id);
+    }
+  }
+
   _updateAverageBatchTime(batchTime) {
     const n = this.stats.completedBatches;
     this.stats.averageBatchTime = n === 1 ? batchTime : ((this.stats.averageBatchTime * (n - 1)) + batchTime) / n;
@@ -315,8 +399,18 @@ export class BatchScrapeTool extends EventEmitter {
 
         const results = [];
         const total = urlConfigs.length;
+        let wasCancelled = false;
 
         for (let i = 0; i < total; i += validated.maxConcurrency) {
+          // Check job.status between slices so JobManager.cancelJob (which
+          // only flips status — it can't interrupt an in-flight await) is
+          // actually honored instead of the loop running to completion anyway.
+          const currentJob = this.jobManager.getJob(job.id);
+          if (currentJob?.status === 'cancelled') {
+            wasCancelled = true;
+            this._log('info', `Batch job ${job.id} cancelled; stopping after ${i}/${total} URLs`);
+            break;
+          }
           const batch = urlConfigs.slice(i, i + validated.maxConcurrency);
           results.push(...await scrapeUrlsBatch(batch, validated, this.defaultTimeout));
           const progress = Math.round(((i + batch.length) / total) * 100);
@@ -327,7 +421,7 @@ export class BatchScrapeTool extends EventEmitter {
         const executionTime = Date.now() - startTime;
 
         const batchResult = {
-          batchId, mode: 'async', success: true, executionTime,
+          batchId, mode: 'async', success: true, cancelled: wasCancelled || undefined, executionTime,
           totalUrls: urlConfigs.length,
           successfulUrls: processedResults.filter(r => r.success).length,
           failedUrls: processedResults.filter(r => !r.success).length,
@@ -336,16 +430,23 @@ export class BatchScrapeTool extends EventEmitter {
         };
 
         if (this.enableResultCaching) {
-          this.batchResults.set(batchId, { results: processedResults, timestamp: Date.now(), ttl: 3600000 });
+          this._cacheBatchResult(batchId, processedResults);
         }
 
-        this.stats.completedBatches++;
-        this.stats.totalUrls += urlConfigs.length;
+        this.stats.totalUrls += results.length;
         this.stats.successfulUrls += batchResult.successfulUrls;
         this.stats.failedUrls += batchResult.failedUrls;
-        this._updateAverageBatchTime(executionTime);
         this.stats.lastUpdated = Date.now();
 
+        if (wasCancelled) {
+          // Job status is already 'cancelled' (set by cancelJob); returning
+          // normally here would otherwise let JobManager.executeJob overwrite
+          // it back to 'completed'. See the JobManager.js guard below.
+          return batchResult;
+        }
+
+        this.stats.completedBatches++;
+        this._updateAverageBatchTime(executionTime);
         await sendWebhookNotification('batch_completed', batchResult, webhookConfig, this.webhookDispatcher, this.enableWebhookNotifications);
         this.emit('batchCompleted', batchResult);
         return batchResult;

package/src/tools/advanced/batchScrape/worker.js

@@ -5,6 +5,7 @@
  */
 
 import { load } from 'cheerio';
+import { config as appConfig } from '../../../constants/config.js';
 import { ssrfGuard, isSsrfError } from '../../../utils/ssrfGuard.js';
 import { throttleHost } from '../../../utils/hostRateLimiter.js';
 
@@ -12,9 +13,14 @@ const USER_AGENT = 'MCP-WebScraper-BatchTool/1.0.0';
 
 /**
  * Fetch a URL with AbortController timeout (SSRF-guarded + per-host throttled).
+ * The timeout stays live through the body read (not just until headers
+ * arrive), and the body is size-capped, so a server that sends headers then
+ * drips the body can't hold a semaphore slot indefinitely or exhaust memory.
+ * Returns { response, html } rather than the raw Response.
  */
 export async function fetchUrl(url, options = {}) {
   const { timeout = 15000, headers = {} } = options;
+  const maxBodySize = appConfig.fetch.maxBodySize;
   const guard = ssrfGuard(url); // SSRF pre-flight (throws before connecting)
   await throttleHost(url);
   const controller = new AbortController();
@@ -25,14 +31,59 @@ export async function fetchUrl(url, options = {}) {
       headers: { 'User-Agent': USER_AGENT, ...headers },
       ...guard
     });
-    clearTimeout(timeoutId);
-    return response;
+    const html = await readBodyCapped(response, maxBodySize);
+    return { response, html };
   } catch (error) {
-    clearTimeout(timeoutId);
     if (isSsrfError(error)) throw new Error(error.cause?.message || error.message);
     if (error.name === 'AbortError') throw new Error(`Request timeout after ${timeout}ms`);
     throw error;
+  } finally {
+    // Cleared only after the body read finishes (or the fetch itself fails),
+    // so the abort timer keeps protecting against a trickling body, not just
+    // the initial headers.
+    clearTimeout(timeoutId);
+  }
+}
+
+/**
+ * Read a Response body as text, enforcing maxBodySize (Content-Length
+ * pre-check, then a running byte count while streaming). Falls back to plain
+ * .text() for responses without a streamable body (e.g. test mocks).
+ */
+async function readBodyCapped(response, maxBodySize) {
+  const contentLengthHeader = response.headers?.get?.('content-length') ?? null;
+  if (contentLengthHeader !== null) {
+    const declared = parseInt(contentLengthHeader, 10);
+    if (!isNaN(declared) && declared > maxBodySize) {
+      throw new Error(`Response body too large: Content-Length ${declared} exceeds limit of ${maxBodySize} bytes`);
+    }
+  }
+
+  if (!response.body || typeof response.body.getReader !== 'function') {
+    return response.text();
+  }
+
+  const reader = response.body.getReader();
+  const chunks = [];
+  let totalBytes = 0;
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    totalBytes += value.byteLength;
+    if (totalBytes > maxBodySize) {
+      reader.cancel();
+      throw new Error(`Response body too large: exceeded limit of ${maxBodySize} bytes`);
+    }
+    chunks.push(value);
+  }
+
+  const merged = new Uint8Array(totalBytes);
+  let offset = 0;
+  for (const chunk of chunks) {
+    merged.set(chunk, offset);
+    offset += chunk.byteLength;
   }
+  return new TextDecoder().decode(merged);
 }
 
 /**
@@ -41,14 +92,13 @@ export async function fetchUrl(url, options = {}) {
 export async function scrapeUrl(config, options, defaultTimeout) {
   const startTime = Date.now();
   try {
-    const response = await fetchUrl(config.url, {
+    const { response, html } = await fetchUrl(config.url, {
       headers: config.headers,
       timeout: config.timeout || defaultTimeout
     });
 
     if (!response.ok) throw new Error(`HTTP ${response.status}: ${response.statusText}`);
 
-    const html = await response.text();
     const $ = load(html);
 
     const result = {

package/src/tools/advanced/scrapeWithActions/recorder.js

@@ -171,6 +171,9 @@ export function buildRecordedEntry(action, timestampMsSinceStart) {
   if (action.direction !== undefined) entry.direction = action.direction;
   if (action.distance !== undefined) entry.distance = action.distance;
   if (action.description !== undefined) entry.description = action.description;
+  // executeJavaScript actions require `script` to replay (ActionExecutor's
+  // ActionChainSchema rejects the entry otherwise) — preserve it.
+  if (action.script !== undefined) entry.script = action.script;
 
   return entry;
 }