crawlforge-mcp-server 6.4.0 → 6.6.0

package/src/core/ActionExecutor.js

@@ -9,6 +9,7 @@ import { EventEmitter } from 'events';
 import { createHash } from 'node:crypto';
 import { assertUrlAllowed } from '../utils/ssrfGuard.js';
 import { browserPreflight } from '../utils/robotsGate.js';
+import { isRef, resolveRef, captureSnapshot } from './browser/snapshot.js';
 
 // executeJavaScript hardening limits (only relevant when the deploy-time flag
 // ALLOW_JAVASCRIPT_EXECUTION=true is set; JS execution stays off by default).
@@ -148,6 +149,14 @@ const ExecuteJavaScriptActionSchema = BaseActionSchema.extend({
   returnResult: z.boolean().default(true)
 });
 
+// camelCase (interactiveOnly/maxNodes) to match every other field in the same
+// action object: clickCount, fullPage, toElement, captureAfter.
+const SnapshotActionSchema = BaseActionSchema.extend({
+  type: z.literal('snapshot'),
+  interactiveOnly: z.boolean().default(true),
+  maxNodes: z.number().min(1).max(1000).optional()
+});
+
 const ActionSchema = z.union([
   WaitActionSchema,
   ClickActionSchema,
@@ -158,7 +167,8 @@ const ActionSchema = z.union([
   HoverActionSchema,
   NavigateActionSchema,
   ScreenshotActionSchema,
-  ExecuteJavaScriptActionSchema
+  ExecuteJavaScriptActionSchema,
+  SnapshotActionSchema
 ]);
 
 const ActionChainSchema = z.object({
@@ -314,7 +324,15 @@ export class ActionExecutor extends EventEmitter {
               fullPage: true,
               description: 'Error screenshot'
             });
-            executionContext.screenshots.push(errorScreenshot);
+            // An actionId is what lets the server publish the shot as a
+            // crawlforge://screenshot/{actionId} resource and drop the base64
+            // from the result; without one a failed chain shipped 1.7 MB of
+            // PNG inline (R21, 2026-09-09).
+            executionContext.screenshots.push({
+              ...errorScreenshot,
+              actionId: this.generateActionId(),
+              error: true
+            });
           } catch (screenshotError) {
             this.log('warn', 'Failed to capture error screenshot: ' + screenshotError.message);
           }
@@ -507,6 +525,123 @@ export class ActionExecutor extends EventEmitter {
     throw lastError;
   }
 
+  /**
+   * Run actions against a page this executor does NOT own.
+   *
+   * executeActionChain() is the one-shot path and owns its page: it creates it,
+   * closes it in `finally`, and re-navigates to the chain's starting URL before
+   * each retry. None of that is right for a browser session, where the page
+   * outlives the call and "the starting URL" is wherever the last call left it —
+   * re-navigating there would throw away the very state the session exists to
+   * keep. So this is the same per-action loop with the lifecycle removed: it
+   * opens nothing, closes nothing, never re-navigates, and deliberately does not
+   * register in `activeChains` (destroy() closes the pages of everything in
+   * there, which would take a live session's page out from under its store).
+   *
+   * Gating comes for free and must not be duplicated by callers: a `navigate`
+   * action goes through executeNavigateAction, which re-runs the SSRF guard and
+   * the blocklist/robots gate on every hop (see the comment at its definition).
+   * `browserOptions.respectRobots` is what reaches that gate.
+   *
+   * No `finalHtml` here, unlike the chain: the page is still open afterwards, so
+   * content is read from it when it is asked for rather than on every call.
+   *
+   * @param {Page} page - a live page owned by the caller
+   * @param {Array} actions - actions to run, in order
+   * @param {Object} [options]
+   * @param {boolean} [options.continueOnError=false] - keep going past a failed action
+   * @param {number} [options.timeout] - per-action deadline for actions that name none
+   * @param {Object} [options.browserOptions] - carries `respectRobots` to the navigate gate
+   * @returns {Promise<Object>} Same results/screenshots/capturedStates shape the chain returns
+   */
+  async executeActionsOnPage(page, actions, { continueOnError = false, timeout, browserOptions = {} } = {}) {
+    const startTime = Date.now();
+    // executeActionInternal reads `id` for its events and `browserOptions` for
+    // the navigate action's robots gate; nothing else on a chain context is
+    // consulted from there.
+    const executionContext = {
+      id: this.generateChainId(),
+      url: page.url(),
+      browserOptions,
+      startTime,
+      results: [],
+      screenshots: [],
+      capturedStates: []
+    };
+
+    let success = true;
+    let error;
+
+    for (let i = 0; i < actions.length; i++) {
+      // actionTimeout() reads the deadline off the action itself, so a caller's
+      // per-call timeout is applied as the default for actions that set none.
+      const action = timeout && !actions[i].timeout ? { ...actions[i], timeout } : actions[i];
+
+      const actionResult = await this.executeActionInternal(page, action, executionContext);
+      executionContext.results.push(actionResult);
+      this.stats.totalActions++;
+
+      if (actionResult.success) {
+        this.stats.successfulActions++;
+      } else {
+        this.stats.failedActions++;
+      }
+
+      // Collection kept in step with executeChainWithRetries — a screenshot
+      // action's payload and a captureAfter snapshot of the page reach the
+      // caller in the same fields either way.
+      if (actionResult.success && action.type === 'screenshot' && actionResult.result?.data) {
+        executionContext.screenshots.push({
+          actionId: actionResult.id,
+          data: actionResult.result.data,
+          format: actionResult.result.format,
+          fullPage: actionResult.result.fullPage,
+          timestamp: actionResult.timestamp
+        });
+      }
+
+      if (action.captureAfter) {
+        try {
+          const capturedHtml = await page.content();
+          executionContext.capturedStates.push({
+            afterActionIndex: i,
+            afterActionId: actionResult.id,
+            url: page.url(),
+            html: capturedHtml,
+            timestamp: Date.now()
+          });
+        } catch (captureErr) {
+          this.log('warn', 'Failed to capture intermediate state: ' + captureErr.message);
+        }
+      }
+
+      if (!actionResult.success && !action.continueOnError && !continueOnError) {
+        success = false;
+        error = actionResult.error;
+        break;
+      }
+
+      if (i < actions.length - 1 && this.actionDelay > 0) {
+        await this.delay(this.actionDelay);
+      }
+    }
+
+    return {
+      success,
+      error,
+      finalUrl: page.url(),
+      executionTime: Date.now() - startTime,
+      results: executionContext.results,
+      screenshots: executionContext.screenshots,
+      capturedStates: executionContext.capturedStates,
+      stats: {
+        totalActions: executionContext.results.length,
+        successfulActions: executionContext.results.filter(r => r.success).length,
+        failedActions: executionContext.results.filter(r => !r.success).length
+      }
+    };
+  }
+
   /**
    * Execute individual action (original internal method)
    * @param {Page} page - Playwright page
@@ -627,6 +762,23 @@ export class ActionExecutor extends EventEmitter {
     return Math.min(this.actionTimeout(action), RECOVERY_TIMEOUT_MS);
   }
 
+  /**
+   * Resolve a caller-supplied selector.
+   *
+   * A selector starting with `@` names a ref a prior snapshot action assigned,
+   * and resolves to the attribute selector that snapshot stamped on the
+   * element; anything else is already a CSS selector. Every selector a caller
+   * writes goes through here, so refs work in every action type without any
+   * schema change — and, because a ref resolves to an ordinary CSS selector,
+   * in the stealth human-behaviour paths too.
+   * @param {Page} page - Playwright page
+   * @param {string} selector - CSS selector, or an `@e1` snapshot ref
+   * @returns {string} CSS selector
+   */
+  resolveSelector(page, selector) {
+    return isRef(selector) ? resolveRef(page, selector) : selector;
+  }
+
   /**
    * Locator for an action's selector.
    *
@@ -635,11 +787,11 @@ export class ActionExecutor extends EventEmitter {
    * selector matching more than one element, which would break action chains
    * that work today.
    * @param {Page} page - Playwright page
-   * @param {string} selector - CSS/text selector
+   * @param {string} selector - CSS/text selector, or an `@e1` snapshot ref
    * @returns {Locator} Playwright locator
    */
   elementLocator(page, selector) {
-    return page.locator(selector).first();
+    return page.locator(this.resolveSelector(page, selector)).first();
   }
 
   /**
@@ -725,6 +877,8 @@ export class ActionExecutor extends EventEmitter {
         return await this.executeScreenshotAction(page, action);
       case 'executeJavaScript':
         return await this.executeJavaScriptAction(page, action);
+      case 'snapshot':
+        return await this.executeSnapshotAction(page, action);
       default:
         throw new Error('Unknown action type: ' + action.type);
     }
@@ -784,7 +938,9 @@ export class ActionExecutor extends EventEmitter {
       // be there before it starts (locator.click() would have waited for it).
       await locator.waitFor({ state: 'visible', timeout });
       // Use human-like clicking behavior
-      await humanBehaviorSimulator.simulateClick(page, action.selector, {
+      // The simulator takes a raw selector string rather than a locator, so
+      // it needs the resolved form — it never goes through elementLocator.
+      await humanBehaviorSimulator.simulateClick(page, this.resolveSelector(page, action.selector), {
         button: action.button,
         clickCount: action.clickCount,
         delay: action.delay,
@@ -841,7 +997,7 @@ export class ActionExecutor extends EventEmitter {
       // Same as click: the simulator works from the selector, so wait first.
       await locator.waitFor({ state: 'visible', timeout });
       // Use human-like typing behavior
-      await humanBehaviorSimulator.simulateTyping(page, action.selector, action.text);
+      await humanBehaviorSimulator.simulateTyping(page, this.resolveSelector(page, action.selector), action.text);
     } else {
       // Standard typing behavior
       await locator.pressSequentially(action.text, { delay: action.delay, timeout });
@@ -899,7 +1055,7 @@ export class ActionExecutor extends EventEmitter {
       if (humanBehaviorSimulator) {
         // Use human-like scrolling to element
         await humanBehaviorSimulator.simulateScroll(page, {
-          target: action.toElement
+          target: this.resolveSelector(page, action.toElement)
         });
       } else {
         // scrollIntoViewIfNeeded, not scrollIntoView — the latter is a DOM API
@@ -1122,6 +1278,21 @@ export class ActionExecutor extends EventEmitter {
       result: action.returnResult ? result : undefined
     };
   }
+
+  /**
+   * Execute snapshot action - the page's interactive elements, each stamped
+   * with a ref later actions can target instead of a guessed CSS selector.
+   * @param {Page} page - Playwright page
+   * @param {Object} action - Snapshot action
+   * @returns {Promise<Object>} Snapshot tree with refs
+   */
+  async executeSnapshotAction(page, action) {
+    return await captureSnapshot(page, {
+      interactiveOnly: action.interactiveOnly,
+      maxNodes: action.maxNodes
+    });
+  }
+
   /**
    * Capture screenshot
    * @param {Page} page - Playwright page
@@ -1313,6 +1484,13 @@ export class ActionExecutor extends EventEmitter {
    * @returns {Promise<Object>} Recovery result
    */
   async attemptErrorRecovery(page, action, error, executionContext) {
+    // A stale ref names an element that went away with the old document, so no
+    // strategy here can find it again. Re-snapshotting is the caller's job:
+    // fail the action now rather than spend the recovery budget on it.
+    if (error?.name === 'StaleRefError') {
+      return { success: false };
+    }
+
     const strategies = this.errorRecoveryStrategies.get(action.type) || [];
     // `retries` caps how many strategies get a turn. Walking all of them
     // unconditionally would add a second full round of timeouts to every action

package/src/core/AuthManager.js

@@ -593,6 +593,7 @@ class AuthManager {
       analyze_content: 3,
       extract_structured: 3,
       extract_with_llm: 3,
+      browser_session: 3, // ceiling — `open`'s price; the schedule is per operation below
 
       // 4 credits
       summarize_content: 4,
@@ -635,6 +636,28 @@ class AuthManager {
       if (bookkeepingOps.has(params?.operation)) return 1;
     }
 
+    // browser_session bills per operation for the same reason: one session is
+    // many calls, and a flat price would charge the ceiling for every cheap
+    // one. `open` launches a browser and navigates, so it costs more than a
+    // `scrape` (2); `read` extracts content and is priced with `scrape`;
+    // `snapshot`, `act` and `screenshot` are one injected script or one action
+    // batch against a page that is already open. `close` and `list` are
+    // bookkeeping but still cost 1 — nothing here runs for free. An unknown or
+    // absent operation falls through to the flat 3: the published price is the
+    // ceiling, never the floor.
+    if (tool === 'browser_session') {
+      const operationCosts = new Map([
+        ['open', 3],
+        ['snapshot', 1],
+        ['act', 1],
+        ['read', 2],
+        ['screenshot', 1],
+        ['close', 1],
+        ['list', 1]
+      ]);
+      if (operationCosts.has(params?.operation)) return operationCosts.get(params.operation);
+    }
+
     // localize_search with a query runs a real web search through the same
     // adapter search_web uses, so it is priced as one rather than undercutting it.
     if (tool === 'localization' && params?.operation === 'localize_search' && params?.searchParams?.query) {
@@ -719,6 +742,9 @@ class AuthManager {
           ? 'Bookkeeping operation — launches no browser.'
           : 'Browser operation. configure/enable/disable/get_stats/cleanup cost 1 credit each.';
         break;
+      case 'browser_session':
+        note = `Priced per operation: open 3, read 2, snapshot/act/screenshot/close/list 1. This call bills ${projected}.`;
+        break;
       case 'serp_rank':
         note = projected === 0
           ? 'DataForSEO not configured — no-op, no credits charged. Set DATAFORSEO_LOGIN/PASSWORD to enable.'

package/src/core/ChangeTracker.js

@@ -720,14 +720,16 @@ export class ChangeTracker extends EventEmitter {
     significanceScore += Math.min(totalElements * 0.05, 1) * 
       (weights.additions + weights.removals + weights.modifications);
     
-    // Text changes impact
-    if (changeAnalysis.textChanges.length > 0) {
-      const textChangeRatio = changeAnalysis.textChanges.reduce(
-        (sum, change) => sum + (change.added?.length || 0) + (change.removed?.length || 0),
-        0
-      ) / 1000; // Normalize by character count
-      
-      significanceScore += Math.min(textChangeRatio, 1) * weights.textChanges;
+    // Text changes impact. textChanges holds diff GROUPS — {type:'word_diff',
+    // changes:[{added, removed, value}]} — not flat parts, so reading
+    // `change.added.length` off a group was always 0 and this term never
+    // fired: a feed that grew by a whole record scored on similarity alone.
+    // The USGS all-hour earthquake feed gained an event (622 words, 84%
+    // similar) and compare reported hasChanges:false, "No significant
+    // changes detected" (R21, 2026-09-09).
+    const changedChars = this.changedTextChars(changeAnalysis.textChanges);
+    if (changedChars > 0) {
+      significanceScore += Math.min(changedChars / 1000, 1) * weights.textChanges;
     }
     
     // Determine significance level
@@ -1317,6 +1319,21 @@ export class ChangeTracker extends EventEmitter {
     );
   }
 
+  /**
+   * Characters added or removed at the text level, for significance scoring.
+   * Same word-diff-then-line-diff choice as countTextChanges, and for the
+   * same reason: the two describe one edit.
+   */
+  changedTextChars(textChanges = []) {
+    const diff = textChanges.find(c => c.type === 'word_diff')
+      || textChanges.find(c => c.type === 'line_diff');
+    if (!diff) return 0;
+    return diff.changes.reduce(
+      (chars, part) => chars + ((part.added || part.removed) && typeof part.value === 'string' ? part.value.length : 0),
+      0
+    );
+  }
+
   generateChangeSummary(changeAnalysis, significance) {
     const { addedElements, removedElements, modifiedElements, similarity } = changeAnalysis;