crawlforge-mcp-server 6.14.0 → 6.17.1

package/src/server/inlineThreshold.js

@@ -20,9 +20,9 @@ export const MAX_INLINE_CHARS_PARAM = {
 /**
  * Per-tool rule. `textPaths` are dotted paths tried in order; the first one
  * holding a string becomes the text view the preview and read_result work
- * on. An empty list means the pretty-printed JSON is the view. `truncate:
- * false` keeps the whole result inline and only adds the handle
- * (extract_embedded_state's never-truncate rule). `when` gates on params.
+ * on. An empty list means the pretty-printed JSON is the view. `when` gates
+ * on params. extract_embedded_state has a shape of its own (shapeEmbeddedState
+ * below).
  */
 /** The browser_session operations that hand back content worth shaping. */
 const BROWSER_SESSION_CONTENT_OPERATIONS = new Set(['snapshot', 'act', 'read']);
@@ -52,7 +52,10 @@ export const INLINE_THRESHOLD_TOOLS = Object.freeze({
   },
   process_document: { textPaths: ['content.text'], truncate: true },
   deep_research: { textPaths: [], truncate: true },
-  extract_embedded_state: { textPaths: [], truncate: false }
+  // Truncated since plan Phase 3.1: returned whole, producthunt.com and
+  // zappos.com overflowed the client. Shaped by shapeEmbeddedState, which never
+  // cuts inside a JSON value.
+  extract_embedded_state: { textPaths: [], truncate: true }
 });
 
 /** param -> env (an int of at least 1,000) -> default. */
@@ -147,21 +150,8 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
   const viewDesc = view === 'text' ? `the ${view_path} text` : 'the pretty-printed JSON';
   const readWith = 'read it with read_result (1 credit) - operation "search" (query), "slice" (offset, length), "lines" or "json_path" (path) - and do not fetch the page again';
 
-  if (!config.truncate) {
-    const hint = `Result is ${json.length} chars as JSON, over the inline limit of ${maxInline}; it is returned whole (this tool never truncates) and is also kept for 1 hour under result_handle ${handle}: ${readWith}.`;
-    return {
-      result: {
-        ...resultObject,
-        result_handle: handle,
-        total_chars: text.length,
-        view,
-        view_path,
-        truncated: false,
-        expires_at,
-        warnings: [...warningsOf(resultObject), hint]
-      },
-      stored: true
-    };
+  if (toolName === 'extract_embedded_state') {
+    return { result: shapeEmbeddedState(resultObject, { json, text, maxInline, handle, expires_at, readWith }), stored: true };
   }
 
   const preview = text.slice(0, maxInline);
@@ -200,3 +190,75 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
   });
   return { result: shaped, stored: true };
 }
+
+/** Object.keys(data) for a plain object, "array(<n>)" for an array, null otherwise. */
+export function embeddedDataKeys(data) {
+  if (Array.isArray(data)) return `array(${data.length})`;
+  if (data !== null && typeof data === 'object') return Object.keys(data);
+  return null;
+}
+
+/**
+ * The longest run of whole lines from the start of `text` whose JSON-string
+ * form fits in `budget` characters. A line that does not fit ends the preview,
+ * so it never stops inside a value; with a budget too small for the first
+ * line it is empty.
+ */
+export function wholeLinePreview(text, budget) {
+  const room = budget - 2; // the string's own quotes
+  let used = 0;
+  let end = 0;
+  let start = 0;
+  while (start < text.length) {
+    const newline = text.indexOf('\n', start);
+    const stop = newline === -1 ? text.length : newline;
+    // A line's escaped length, plus the escaped newline that joins it to the last.
+    const cost = JSON.stringify(text.slice(start, stop)).length - 2 + (end > 0 ? 2 : 0);
+    if (used + cost > room) break;
+    used += cost;
+    end = stop;
+    start = stop + 1;
+  }
+  return text.slice(0, end);
+}
+
+/**
+ * extract_embedded_state over the limit (plan Phase 3.1). What stays inline is
+ * everything that says what the page carries — url, path, bytes, the whole
+ * `found` list, data_keys, the escalation report and window_state's list —
+ * then a preview of the pretty-printed JSON cut at a line boundary. The
+ * preview gets whatever max_inline_chars leaves after the rest, measured as
+ * compact JSON, so the whole inline result stays within the limit whenever
+ * the rest fits. The full result is under the handle for read_result
+ * json_path. The REST route shapes it the same way; keep the two in step.
+ */
+function shapeEmbeddedState(resultObject, { json, text, maxInline, handle, expires_at, readWith }) {
+  const dataKeys = embeddedDataKeys(resultObject.data);
+  const firstKey = Array.isArray(dataKeys) && dataKeys.length > 0 ? dataKeys[0] : null;
+  const example = firstKey && /^[A-Za-z_$][\w$]*$/.test(firstKey) ? ` Read one payload with operation "json_path", path "data.${firstKey}".` : '';
+  const hint = `Result is ${json.length} chars as JSON, over the inline limit of ${maxInline}; preview holds the pretty-printed JSON up to the last whole line that fits (${text.length} chars in total), data_keys lists the top-level keys of data, and the full result is kept for 1 hour under result_handle ${handle}: ${readWith}.${example} Or re-run with path, or keys_only:true, to ask for less.`;
+
+  const shaped = {};
+  for (const key of ['url', 'path', 'bytes', 'found']) {
+    if (key in resultObject) shaped[key] = resultObject[key];
+  }
+  shaped.data_keys = dataKeys;
+  if ('escalated' in resultObject) shaped.escalated = resultObject.escalated;
+  if (resultObject.stealth) shaped.stealth = resultObject.stealth;
+  if (resultObject.window_state && typeof resultObject.window_state === 'object') {
+    shaped.window_state = { note: resultObject.window_state.note, found: resultObject.window_state.found };
+  }
+  const tail = {
+    result_handle: handle,
+    total_chars: text.length,
+    view: 'json',
+    view_path: null,
+    truncated: true,
+    expires_at,
+    warnings: [...warningsOf(resultObject), hint]
+  };
+  // `"preview":"",` is what the key itself adds once the value is in.
+  const envelope = JSON.stringify({ ...shaped, preview: '', ...tail }).length;
+  const preview = wholeLinePreview(text, Math.max(0, maxInline - envelope + 2));
+  return { ...shaped, preview, ...tail };
+}

package/src/skills/agent-skills/crawlforge-getting-started/references/credits.md

@@ -22,7 +22,7 @@ metered; there is no free tier. Tools marked "scales" cost more as work grows.
 |------|-------|
 | `scrape` | Unified multi-format single fetch. A `highlights` or `question` format adds 1 once per call; `mode:"model"` on one of them adds 3 once. `escalate:true` adds 5 as the projected ceiling; the actual charge drops back to the base when the plain fetch succeeded and no escalation ran. |
 | `scrape_structured` | CSS-selector extraction. |
-| `extract_embedded_state` | Embedded JS state (`__NEXT_DATA__`, RSC, Nuxt, Apollo, Redux). |
+| `extract_embedded_state` | Embedded JS state (`__NEXT_DATA__`, RSC, Nuxt, Apollo, Redux). `escalate:true` adds 5 as the projected ceiling for the stealth re-read; the actual charge drops back to 2 when the plain fetch succeeded and no escalation ran. `keys_only` and `find` cost nothing extra. |
 | `extract_content` | Readability-cleaned article. |
 | `map_site` | URL discovery / sitemap. |
 | `process_document` | PDF / DOCX / TXT parsing. |

package/src/skills/agent-skills/crawlforge-structured-extraction/SKILL.md

@@ -36,15 +36,34 @@ already, exact and typed — that beats asking a model to read them off the rend
 { "tool": "extract_embedded_state", "params": { "url": "https://www.ticketmaster.com/discover/concerts", "path": "next_data.props.pageProps" } }
 ```
 
-Finds `__NEXT_DATA__`, `self.__next_f` (React Server Component payloads),
-`window.__NUXT__`, `__APOLLO_STATE__`, `__INITIAL_STATE__`, `__PRELOADED_STATE__`
-and `<script type="application/json">` blocks, keyed by source name. No LLM in
-the path, so values are exact rather than inferred.
+Finds `__NEXT_DATA__`, `self.__next_f` (React Server Component payloads, with
+`$<id>` references between rows resolved and a `data_rows` index of the rows
+carrying data), `window.__NUXT__` and Nuxt 3's `__NUXT_DATA__` (decoded as
+`nuxt_data`; `raw: true` also keeps the undecoded array), SvelteKit `kit.start`
+data, `__APOLLO_STATE__`, `__INITIAL_STATE__`, `__PRELOADED_STATE__`,
+`ytInitialData`, `ytInitialPlayerResponse`, Inertia `data-page`, Shopify meta and
+product JSON, JSON `data-*` attributes and `<script type="application/json">`
+blocks, keyed by source name. No LLM in the path, so values are exact rather
+than inferred.
 
 These payloads are routinely over a megabyte — pass `path` (dotted keys and
 array indexes, e.g. `next_data.props.pageProps` or `next_f[0].f`) to return one
-subtree. Without it, a large result comes back with a warning naming the biggest
-source and a ready-to-paste path.
+subtree, or `keys_only: true` first to see the first two levels of keys (types,
+no values) before choosing one. When you know a field's name but not where it
+is, `find: "<key>"` returns every path holding it (case-insensitive, up to 50,
+each with a 200-character preview) instead of the data; each path can be passed
+straight back as `path`. `find` and `keys_only` cannot be combined. A result
+over `max_inline_chars` comes back as a `preview` (whole lines of the JSON),
+`data_keys` and a `result_handle`: read the
+rest with `read_result` operation `json_path` (e.g. path `data.next_data.props`)
+instead of fetching again.
+
+On a site that blocks a plain fetch (403/429/challenge page), pass
+`escalate: true` (projected 7, charged 2 when the plain fetch works): the stealth
+browser re-reads the page, the same parser runs on it, and the framework globals
+are read off `window` after JavaScript ran — `ytInitialData`, `__NUXT__`,
+`__remixContext` and others, under `window_state` (path
+`window_state.ytInitialData.contents`). A 404 or 5xx never escalates.
 
 ## scrape_template — known sites, zero selectors (cost: 1)
 

package/src/tools/advanced/BrowserSessionTool.js

@@ -111,8 +111,10 @@ const BrowserSessionSchema = z.object({
   timeout: z.number().min(10000).max(120000).default(30000),
 
   // Applies to the call it is sent on: on `open` to the first load, on `act` to
-  // every navigate in that call. It is never remembered by the session, so an
-  // override has to be repeated as deliberately as it was made.
+  // every navigate in that call, and on any call to wherever the page has moved
+  // since it was last checked (a click, a form, a script). It is never
+  // remembered by the session, so an override has to be repeated as
+  // deliberately as it was made. A URL that passed is not asked about again.
   respect_robots: z.boolean().optional(),
 
   // snapshot
@@ -358,6 +360,8 @@ export class BrowserSessionTool {
 
     let session;
     try {
+      // The load was gated; a script may have moved the page on since.
+      await this.actionExecutor.assertPageAllowed(page, browserOptions);
       session = this.store.create({
         ownerId,
         page,
@@ -371,8 +375,9 @@ export class BrowserSessionTool {
         maxPerOwner: ownerId.startsWith(REST_OWNER_PREFIX) ? REST_MAX_SESSIONS_PER_OWNER : undefined
       });
     } catch (error) {
-      // A cap refusal arrives with a live page in hand. Give it back before
-      // rethrowing, or the refused call leaks the context it just pinned.
+      // A cap refusal, or a page the gate refused, arrives with a live page in
+      // hand. Give it back before rethrowing, or the refused call leaks the
+      // context it just pinned.
       await releasePage();
       throw error;
     }
@@ -438,13 +443,29 @@ export class BrowserSessionTool {
     }
   }
 
+  /**
+   * Read from a session's page once the URL it stands on has passed the gate.
+   *
+   * A session's page outlives the call that loaded it, and between calls it
+   * can go anywhere: the last `act` clicked a link, or a script redirected.
+   * ActionExecutor.readGated holds it to the SSRF guard and the blocklist/
+   * robots gate on both sides of the read, with this call's `respect_robots`.
+   */
+  readPage(session, params, read) {
+    return this.actionExecutor.readGated(
+      session.page,
+      { respectRobots: params.respect_robots, tool: 'browser_session' },
+      read
+    );
+  }
+
   async snapshotSession(params, ownerId) {
     const session = this.requireSession(params, ownerId);
 
-    const snapshot = await captureSnapshot(session.page, {
+    const snapshot = await this.readPage(session, params, () => captureSnapshot(session.page, {
       interactiveOnly: params.interactive_only,
       maxNodes: params.max_nodes
-    });
+    }));
 
     this.store.touch(session, session.page.url());
     return { success: true, operation: 'snapshot', ...sessionInfo(session), snapshot };
@@ -473,9 +494,11 @@ export class BrowserSessionTool {
     }
 
     // Every `navigate` in here re-runs the SSRF guard and the blocklist/robots
-    // gate inside executeNavigateAction — verified, and the reason the gate is
-    // not repeated here. A long-lived session is a repeatable navigation
-    // primitive, so that per-hop check is what stops it becoming an SSRF hop.
+    // gate inside executeNavigateAction, and every action is bracketed by the
+    // same check on wherever the page has moved (executeGatedAction) — the
+    // reason the gate is not repeated here. A long-lived session is a
+    // repeatable navigation primitive, so those checks are what stop it
+    // becoming an SSRF hop.
     const result = await this.actionExecutor.executeActionsOnPage(session.page, params.actions, {
       continueOnError: params.continue_on_error,
       timeout: params.timeout,
@@ -503,8 +526,10 @@ export class BrowserSessionTool {
 
   async readSession(params, ownerId) {
     const session = this.requireSession(params, ownerId);
-    const url = session.page.url();
-    const html = await session.page.content();
+    const { url, html } = await this.readPage(session, params, async () => ({
+      url: session.page.url(),
+      html: await session.page.content()
+    }));
 
     const options = {};
     if (params.formats.includes('markdown')) options.outputFormat = 'markdown';
@@ -564,12 +589,12 @@ export class BrowserSessionTool {
   async screenshotSession(params, ownerId) {
     const session = this.requireSession(params, ownerId);
 
-    const shot = await this.actionExecutor.captureScreenshot(session.page, {
+    const shot = await this.readPage(session, params, () => this.actionExecutor.captureScreenshot(session.page, {
       fullPage: params.full_page,
       format: params.format,
       quality: params.quality,
       selector: params.selector
-    });
+    }));
 
     this.store.touch(session, session.page.url());
     // The actionId is what lets the server publish the image as a

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -14,7 +14,7 @@ import { stealthDocumentVerdict } from '../../utils/stealthVerdict.js';
 import { elementText } from '../../utils/elementText.js';
 import { pageTitle } from '../../utils/pageTitle.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
-import { resolveStealthEngine } from '../../core/StealthBrowserManager.js';
+import { resolveStealthEngine, redactProxyEntry, assertProxyEngineAllowed } from '../../core/StealthBrowserManager.js';
 import { actionQueue as sharedActionQueue } from '../../core/browser/actionQueue.js';
 
 // Recording / replay helpers
@@ -27,6 +27,20 @@ import {
   recordedEntryToAction
 } from './scrapeWithActions/recorder.js';
 
+// metadata.browserOptions echoes the merged options, and a proxy entry carries
+// its password.
+function redactProxyCredentials(browserOptions) {
+  const proxies = browserOptions.proxyRotation?.proxies;
+  if (!proxies) return browserOptions;
+  return {
+    ...browserOptions,
+    proxyRotation: {
+      ...browserOptions.proxyRotation,
+      proxies: proxies.map(redactProxyEntry)
+    }
+  };
+}
+
 // Action schemas (re-using from ActionExecutor but with tool-specific additions)
 const BaseActionSchema = z.object({
   type: z.string(),
@@ -203,7 +217,15 @@ const ScrapeWithActionsSchema = z.object({
     // Answer a cookie/consent wall (autoconsent) after the initial load and
     // after each navigate action: "reject" opts out, "accept" opts in. Off by
     // default so a chain that clicks the banner itself keeps working.
-    consent: z.enum(['off', 'reject', 'accept']).default('off')
+    consent: z.enum(['off', 'reject', 'accept']).default('off'),
+    // The caller's own proxies for a stealth chain — the shape and parsing of
+    // stealth_mode's stealthConfig.proxyRotation. Refused without stealth:true
+    // (see executeSession); CrawlForge supplies none.
+    proxyRotation: z.object({
+      enabled: z.boolean().default(false),
+      proxies: z.array(z.string()).optional(),
+      rotationInterval: z.number().default(300000)
+    }).optional()
   }).optional(),
 
   // Content extraction options
@@ -425,6 +447,8 @@ export class ScrapeWithActionsTool extends EventEmitter {
     const warnings = [];
     if (browserOptions.stealth) {
       const resolved = await resolveStealthEngine(browserOptions.engine);
+      // Refused on the engine that will run, before the chain starts.
+      assertProxyEngineAllowed(resolved.engine, browserOptions);
       browserOptions.stealthMode = { enabled: true, engine: resolved.engine };
       if (resolved.fallbackWarning) warnings.push(resolved.fallbackWarning);
     } else if (browserOptions.engine === 'camoufox') {
@@ -439,6 +463,15 @@ export class ScrapeWithActionsTool extends EventEmitter {
         'to use the standard Chromium browser.'
       );
     }
+    if (browserOptions.proxyRotation?.enabled && !browserOptions.stealth) {
+      // Only the stealth browser takes a proxy; the standard pool would run
+      // the chain from this server's own address while the caller believes
+      // it is proxied.
+      throw new Error(
+        'browserOptions.proxyRotation requires browserOptions.stealth:true — only the stealth ' +
+        'browser routes through a proxy. Set stealth:true, or drop proxyRotation.'
+      );
+    }
 
     // The robots override rides with the browser options so ActionExecutor's
     // gate sees it on the initial load and on every `navigate` action.
@@ -534,7 +567,7 @@ export class ScrapeWithActionsTool extends EventEmitter {
       text: finalContent?.content?.text || '',
       html: chainResult?.finalHtml || '',
       status: chainResult?.navigationStatus ?? null
-    }, { allowEmpty: true });
+    }, { allowEmpty: true, fetcher: browserOptions.stealthMode?.enabled ? 'the stealth browser' : 'the browser' });
 
     return {
       success: chainResult.success && verdict.success,
@@ -553,6 +586,9 @@ export class ScrapeWithActionsTool extends EventEmitter {
       // The landing page's consent result; each navigate action's own is on
       // its result. Absent when consent handling is off.
       ...(chainResult.consent ? { consent: chainResult.consent } : {}),
+      // Every navigation of the reported attempt — [0] the initial load, then
+      // each navigate action — with its status and any wall it reached.
+      navigations: chainResult.navigations || [],
 
       actionResults,
       attempt: chainResult.attempt ?? attempts.length,
@@ -576,7 +612,7 @@ export class ScrapeWithActionsTool extends EventEmitter {
       replayedFrom: params.replayRecording || undefined,
 
       metadata: {
-        browserOptions,
+        browserOptions: redactProxyCredentials(browserOptions),
         formAutoFillApplied: !!params.formAutoFill,
         intermediateStatesCount: intermediateStates.length,
         screenshotsCount: sessionContext.screenshots.length,

package/src/tools/advanced/batchScrape/worker.js

@@ -10,6 +10,7 @@ import { config as appConfig } from '../../../constants/config.js';
 import { ssrfGuard, isSsrfError } from '../../../utils/ssrfGuard.js';
 import { noteRetryAfter } from '../../../utils/hostRateLimiter.js';
 import { preflightFetch } from '../../../utils/robotsGate.js';
+import { fetchResigned } from '../../../utils/resignedFetch.js';
 import { htmlToMarkdown } from '../../../utils/htmlToMarkdown.js';
 import { elementText } from '../../../utils/elementText.js';
 import { pageTitle } from '../../../utils/pageTitle.js';
@@ -31,9 +32,10 @@ export async function fetchUrl(url, options = {}) {
   const controller = new AbortController();
   const timeoutId = setTimeout(() => controller.abort(), timeout);
   try {
-    const response = await fetch(url, {
+    const response = await fetchResigned(url, {
       signal: controller.signal,
       headers: { ...gate.headers, ...headers },
+      onRedirect: gate.onRedirect,
       ...guard
     });
     if (response.status === 429 || response.status === 503) {

package/src/tools/basic/_fetch.js

@@ -8,6 +8,7 @@ import { config } from '../../constants/config.js';
 import { ssrfGuard, isSsrfError } from '../../utils/ssrfGuard.js';
 import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
+import { fetchResigned } from '../../utils/resignedFetch.js';
 
 /**
  * Fetch a URL with a configurable timeout and body-size cap.
@@ -51,12 +52,13 @@ export async function fetchWithTimeout(url, options = {}) {
   try {
     let response;
     try {
-      response = await fetch(url, {
+      response = await fetchResigned(url, {
         signal: controller.signal,
         headers: {
           ...gate.headers,
           ...headers
         },
+        onRedirect: gate.onRedirect,
         ...guard
       });
     } catch (error) {

package/src/tools/crawl/_sessionContext.js

@@ -223,7 +223,8 @@ export class SessionContext {
     const fetchOpts = {
       method,
       headers: requestHeaders,
-      redirect: 'follow'
+      redirect: 'follow',
+      onRedirect: gate.onRedirect
     };
 
     if (body) {

package/src/tools/crawl/mapSite.js

@@ -384,7 +384,8 @@ export class MapSiteTool {
     try {
       const response = await safeFetch(url, {
         signal: controller.signal,
-        headers: gate.headers
+        headers: gate.headers,
+        onRedirect: gate.onRedirect
       });
       clearTimeout(timeoutId);
       return response;

package/src/tools/extract/_fetchAndParse.js

@@ -147,6 +147,7 @@ export async function fetchAndParse(url, options = {}) {
       ...gate.headers,
       'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
     },
+    onRedirect: gate.onRedirect,
     signal: AbortSignal.timeout(timeoutMs)
   });
 

package/src/tools/extract/extractContent.js

@@ -176,7 +176,7 @@ export class ExtractContentTool {
             enableImages: options.preserveImageInfo,
             captureScreenshot: false
           }
-        });
+        }, gate);
 
         if (!browserResult.success) {
           throw new Error(`Browser processing failed: ${browserResult.error}`);
@@ -188,6 +188,7 @@ export class ExtractContentTool {
         // Simple HTTP fetch
         const response = await safeFetch(url, {
           headers: { ...gate.headers },
+          onRedirect: gate.onRedirect,
           signal: AbortSignal.timeout(15000)
         });