crawlforge-mcp-server 6.17.1 → 6.18.1

package/src/tools/extract/_fetchAndParse.js

@@ -11,7 +11,7 @@
  */
 
 import { load } from 'cheerio';
-import { readBody } from 'crawlforge-extractors';
+import { readBody, flattenText } from 'crawlforge-extractors';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 import { config } from '../../constants/config.js';
 import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
@@ -58,6 +58,13 @@ async function readTextWithSizeCap(response) {
   return readBody(response, { maxBytes: maxBodySize });
 }
 
+/** The refusal for a binary response; batch_scrape gives the same one (R24 1.5). */
+export function unsupportedContentTypeError(contentType) {
+  return new Error(
+    `Unsupported content type "${contentType}" — this looks like binary content, not HTML/text. Use process_document for PDFs/documents/binary files.`
+  );
+}
+
 /**
  * Classify a Content-Type header for the purposes of HTML parsing.
  * Missing header is treated as 'html' (permissive default — many servers,
@@ -65,7 +72,7 @@ async function readTextWithSizeCap(response) {
  * @param {string|null} contentType
  * @returns {'html'|'text'|'binary'}
  */
-function classifyContentType(contentType) {
+export function classifyContentType(contentType) {
   if (!contentType) return 'html';
   const type = contentType.split(';')[0].trim().toLowerCase();
   if (
@@ -83,35 +90,11 @@ function classifyContentType(contentType) {
 }
 
 /**
- * Flatten a parsed document's <body> to text while preserving line structure.
- *
- * Cheerio's .text() joins adjacent elements with no separator, which welds
- * table rows / list items together ("1.Story title329 points") and starves
- * downstream LLM extraction (AgentOrchestrator, extractStructured,
- * extractWithLlm) of any structure to parse. Works on a detached clone so the
- * caller's $ tree is untouched — unifiedScrape reuses $ for other formats.
- *
- * @param {import('cheerio').CheerioAPI} $
- * @returns {string}
+ * Flatten a parsed document's <body> to text, one line per block element.
+ * The flattener moved to crawlforge-extractors (`flattenText`) so the REST
+ * extract_text route shares it; this name stays for extractStructured.
  */
-export function flattenBodyText($) {
-  // Block boundaries are marked with a U+E000 private-use sentinel so the
-  // HTML source's own insignificant newlines can be collapsed to spaces
-  // first, and only the sentinels become line breaks. (NUL won't survive:
-  // .after()/.replaceWith() parse their argument as HTML and the parser
-  // strips NUL; U+E000 passes through and never occurs in real page text.)
-  const $body = $('body').clone();
-  $body.find('br').replaceWith('\uE000');
-  $body.find('td, th').after(' ');
-  $body
-    .find('p, div, li, tr, h1, h2, h3, h4, h5, h6, blockquote, pre, table, ul, ol, dl, section, article, header, footer')
-    .after('\uE000');
-  return $body
-    .text()
-    .replace(/\s+/g, ' ')
-    .replace(/ ?(?:\uE000 ?)+/g, '\n')
-    .trim();
-}
+export { flattenText as flattenBodyText };
 
 /**
  * Fetch a URL and return parsed HTML via Cheerio.
@@ -170,9 +153,7 @@ export async function fetchAndParse(url, options = {}) {
   }
 
   if (classification === 'binary') {
-    throw new Error(
-      `Unsupported content type "${contentType}" — this looks like binary content, not HTML/text. Use process_document for PDFs/documents/binary files.`
-    );
+    throw unsupportedContentTypeError(contentType);
   }
 
   const html = await readTextWithSizeCap(response);
@@ -190,7 +171,7 @@ export async function fetchAndParse(url, options = {}) {
     $(stripTags.join(', ')).remove();
   }
 
-  const textContent = flattenBodyText($);
+  const textContent = flattenText($);
 
   return { html, $, textContent, finalUrl: response.url, warnings, status: response.status };
 }

package/src/tools/scrape/unifiedScrape.js

@@ -699,16 +699,19 @@ export class UnifiedScrapeTool {
         case 'text':
           try {
             const { load } = await import('cheerio');
+            // cheerio parses a <noscript> body as raw text, so left in it
+            // reaches `text` as literal tags (lemonde.fr's challenge page,
+            // 2026-10-03); extract_text and the markdown format drop it too.
             if (onlyMainContent) {
               // Plain text from Readability main content via cheerio
               const $main = load(getMainHtml());
-              $main('script, style').remove();
+              $main('script, style, noscript').remove();
               content.text = extractBlockText($main);
             } else {
               // Strip script/style on a clone, not the shared $, so other
               // formats reading $ later aren't affected by format ordering.
               const $clone = load($.html());
-              $clone('script, style').remove();
+              $clone('script, style, noscript').remove();
               content.text = extractBlockText($clone);
             }
           } catch (err) {
@@ -790,6 +793,15 @@ export class UnifiedScrapeTool {
     if (queryScoped && !formats.includes('markdown')) {
       warnings.push('offsets index the "markdown" format of this call (same onlyMainContent); add "markdown" to formats to quote with a locator');
     }
+    // A success with nothing in it must say so: the caller otherwise cannot
+    // tell an empty page from a scrape that lost the content.
+    if (typeof content.markdown === 'string' && content.markdown.trim() === '' && !warnings.some((w) => w.startsWith('markdown:'))) {
+      warnings.push(
+        'markdown: the page produced no markdown' +
+        (onlyMainContent ? '; retry with onlyMainContent:false if main-content extraction dropped it, or' : ';') +
+        ' use escalate:true if the page is rendered by JavaScript'
+      );
+    }
     if (!queryScoped && typeof content.markdown === 'string' && content.markdown.length > OVERSIZED_MARKDOWN_CHARS) {
       warnings.push(`markdown: ${content.markdown.length} characters; ask for {type:"highlights", query} (1 extra credit, no model) to get only the matching sentences, table rows and code blocks`);
     }

package/src/tools/tracking/trackChanges/index.js

@@ -308,6 +308,15 @@ export class TrackChangesTool extends EventEmitter {
   async setupMonitoring(params) {
     const { url, monitoringOptions, trackingOptions, storageOptions, notificationOptions } = params;
 
+    // enabled:false turns the URL's polling monitor off and starts nothing.
+    if (monitoringOptions.enabled === false) {
+      return {
+        success: true, operation: 'monitor', url,
+        monitoring: { enabled: false, stopped: this.stopMonitoring(url) },
+        timestamp: Date.now()
+      };
+    }
+
     if (this.activeMonitors.has(url)) {
       clearInterval(this.activeMonitors.get(url).timer);
     }
@@ -498,6 +507,15 @@ export class TrackChangesTool extends EventEmitter {
   async stopScheduledMonitor(params) {
     const { url, scheduledMonitorOptions } = params;
     const monitorId = scheduledMonitorOptions?.monitorId;
+    // A polling monitor (operation:"monitor") is listed as poll:<url>.
+    if (monitorId?.startsWith('poll:')) {
+      const stopped = this.stopMonitoring(monitorId.slice(5));
+      return {
+        success: stopped, operation: 'stop_scheduled_monitor', monitorId, stopped,
+        ...(stopped ? {} : { error: `No polling monitor found with id ${monitorId}` }),
+        timestamp: Date.now()
+      };
+    }
     if (monitorId) {
       if (!this.monitorStore._loaded) await this.monitorStore.load();
       if (this.monitorStore.get(monitorId)) {
@@ -517,6 +535,7 @@ export class TrackChangesTool extends EventEmitter {
     }
     if (!url) throw new Error('stop_scheduled_monitor requires a url or scheduledMonitorOptions.monitorId');
     const result = await this.scheduler.stopByUrl(url);
+    const stoppedPolling = this.stopMonitoring(url) ? 1 : 0;
     // Only a hosted monitor whose every target is this URL — never a
     // multi-target monitor that merely includes it.
     let stoppedHosted = 0;
@@ -533,7 +552,7 @@ export class TrackChangesTool extends EventEmitter {
       hostedError = error.message;
     }
     return {
-      success: true, operation: 'stop_scheduled_monitor', url, stoppedMonitors: result.stopped, stoppedHosted,
+      success: true, operation: 'stop_scheduled_monitor', url, stoppedMonitors: result.stopped, stoppedPolling, stoppedHosted,
       ...(hostedError ? { hostedError } : {}),
       timestamp: Date.now()
     };
@@ -541,7 +560,10 @@ export class TrackChangesTool extends EventEmitter {
 
   async listScheduledMonitors() {
     if (!this.monitorStore._loaded) await this.monitorStore.load();
-    const local = this.scheduler.list().map((m) => ({ ...m, hosted: false }));
+    const local = [
+      ...this._pollingMonitors().map((m) => ({ ...m, hosted: false })),
+      ...this.scheduler.list().map((m) => ({ ...m, hosted: false }))
+    ];
     // The local list never fails because the website is unreachable.
     let hosted = [];
     let hostedError = null;
@@ -560,16 +582,21 @@ export class TrackChangesTool extends EventEmitter {
     };
   }
 
+  /** The in-process polling monitors operation:"monitor" started, as the dashboard and the list show them. */
+  _pollingMonitors() {
+    return Array.from(this.activeMonitors.entries()).map(([url, m]) => ({
+      id: `poll:${url}`, url, kind: 'polling', status: 'active',
+      interval: m.options?.interval ?? null, stats: m.stats
+    }));
+  }
+
   async getMonitoringDashboard(params) {
     // The schema defaults every flag to true, but only inside a dashboardOptions
     // object the caller supplied; an omitted object must mean the same thing.
     const dashboardOptions = { includeRecentAlerts: true, includeTrends: true, includeMonitorStatus: true, ...(params.dashboardOptions || {}) };
     const dashboard = this.changeTracker.getMonitoringDashboard();
     // The tracker only knows tracked pages; the monitors that poll them live here.
-    const polling = Array.from(this.activeMonitors.entries()).map(([url, m]) => ({
-      id: `poll:${url}`, url, kind: 'polling', status: 'active',
-      interval: m.options?.interval ?? null, stats: m.stats
-    }));
+    const polling = this._pollingMonitors();
     if (!this.monitorStore._loaded) await this.monitorStore.load();
     const scheduled = this.scheduler.list().map(m => ({
       id: m.id, url: m.url, kind: 'scheduled',

package/src/tools/tracking/trackChanges/schema.js

@@ -57,7 +57,7 @@ export const TRACK_CHANGES_INPUT_SHAPE = {
   }).optional().prefault({}).describe("Options for how changes are tracked and compared"),
 
   monitoringOptions: z.object({
-    enabled: z.boolean().default(false),
+    enabled: z.boolean().default(true).describe("operation:\"monitor\" starts polling the URL; false stops the URL's polling monitor and starts nothing"),
     interval: z.number().min(60000).max(24 * 60 * 60 * 1000).default(300000),
     maxRetries: z.number().min(0).max(5).default(3),
     retryDelay: z.number().min(1000).max(60000).default(5000),
@@ -113,7 +113,7 @@ export const TRACK_CHANGES_INPUT_SHAPE = {
     enabled: z.boolean().default(true),
     interval: z.number().min(60000).optional().describe("Polling interval in ms (default 1h)"),
     goal: z.string().optional().describe("Plain-English alert goal; an LLM judges whether a change matches (degrades to threshold if no LLM)"),
-    monitorId: z.string().optional().describe("Monitor id for stop_scheduled_monitor"),
+    monitorId: z.string().optional().describe("Monitor id for stop_scheduled_monitor, as list_scheduled_monitors shows it (poll:<url> for a polling monitor)"),
     notificationThreshold: z.enum(['minor', 'moderate', 'major', 'critical']).optional(),
     hosted: z.boolean().default(false).describe("Run the monitor on CrawlForge's servers: it fires from the hosted scheduler whether or not this process is alive and sends email and signed webhooks. Each check bills 3 credits per compared target from the account; blocked and errored targets are free. Default false = local, in-process."),
     name: z.string().min(1).max(80).optional().describe("Display name for a hosted monitor (default: the URL host)")

package/src/utils/fetchLadder.js

@@ -0,0 +1,243 @@
+/**
+ * fetchLadder — scrape's fetch ladder for extract_text and extract_links (fix
+ * plan Phase E2). Same rules and wording as the website's
+ * src/lib/tools/fetch-ladder.ts — keep the two in step.
+ *
+ *   1. A plain fetch (15 s), retried once when a 429/503 asks for at most
+ *      10 s with Retry-After.
+ *   2. A 2xx body that is not text (a PDF, an image, an archive) stops here as
+ *      UNSUPPORTED_CONTENT_TYPE; it never escalates.
+ *   3. Only with escalate:true, and only when a browser could change the
+ *      outcome (aBrowserMightPass, or a 444 refusal): the injected stealth
+ *      stage — server.js `stealthEscalation`, the one scrape and
+ *      extract_embedded_state run, behind the same compliance gate (impit
+ *      first under "auto", then the browser).
+ *   4. Otherwise E1's verdict stands unchanged (targetFailure).
+ *
+ * scrape and extract_embedded_state keep their own copies of steps 3–4; this
+ * reuses their exported pieces rather than moving them.
+ */
+
+import { load } from 'cheerio';
+import { z } from 'zod';
+import { fetchWithTimeout } from '../tools/basic/_fetch.js';
+import { plainVerdict, targetFailure } from '../tools/basic/_targetFailure.js';
+import { stealthDocumentVerdict } from './stealthVerdict.js';
+import { parseRetryAfter } from './hostRateLimiter.js';
+import { setActualCost } from '../server/requestContext.js';
+import { SCRAPE_ESCALATION_SHAPE, SCRAPE_ESCALATION_CREDITS, aBrowserMightPass } from '../tools/scrape/escalation.js';
+
+/** extract_text's and extract_links's table price; escalation adds SCRAPE_ESCALATION_CREDITS. */
+const EXTRACT_BASE_CREDITS = 1;
+
+export const LADDER_TIMEOUT_MS = 15000;
+
+/** A Retry-After longer than this is not waited out inside one call. */
+export const RETRY_AFTER_MAX_MS = 10000;
+
+export const CLIENT_RENDERED_WARNING = 'page renders client-side; use scrape';
+
+/** The two public fields both tools take, in tools/list order. */
+export const EXTRACT_ESCALATION_SHAPE = {
+  escalate: z.boolean().optional().default(false).describe('When the plain fetch comes back blocked (403/429/444/challenge page/empty shell), re-read the page once in the stealth browser and extract from the rendered document. Under escalate_engine "auto" a Chrome TLS handshake with the honest CrawlForge User-Agent (impit) is tried first, and the browser runs only when that does not get the page. A 404 or 5xx never escalates. Projected at 1+5; the actual charge stays at 1 when the plain fetch succeeded. Default: false'),
+  escalate_engine: SCRAPE_ESCALATION_SHAPE.escalate_engine
+};
+
+// Bodies these tools read. A missing Content-Type is read as HTML.
+const TEXTUAL_TYPE = /^(?:text\/[\w.+-]+|application\/(?:xhtml\+xml|xml|json)|application\/[\w.+-]+\+(?:xml|json))$/;
+const JSON_TYPE = /^application\/(?:[\w.+-]+\+)?json$/;
+
+/** The media type without parameters, lowercased; '' when absent. */
+function mediaType(response) {
+  return (response.headers?.get?.('content-type') ?? '').split(';')[0].trim().toLowerCase();
+}
+
+/** Whether a media type is JSON (application/json or a +json suffix). */
+export function isJsonType(type) {
+  return JSON_TYPE.test(type);
+}
+
+// A page that says it needs JavaScript (scrapingcourse.com's
+// /javascript-rendering serves "Enable JavaScript to see products" over empty
+// product cards, with no mount point to find).
+// No \b before the verbs: cheerio's text() joins <h1>Challenge</h1><p>Enable…
+// into one word.
+const JS_REQUIRED = /(?:enable|requires?|turn on)\s+javascript\b|\bjavascript\s+(?:is\s+)?(?:required|disabled)\b/i;
+
+/**
+ * A client-rendered shell: under 200 characters of visible text, and either an
+ * empty <div id="root">, "app" or "__next" for the framework to mount into or
+ * text saying the page needs JavaScript.
+ * @param {string} html
+ */
+export function isClientRenderedShell(html) {
+  const $ = load(html);
+  $('script, style, noscript').remove();
+  const text = $('body').text().replace(/\s+/g, ' ').trim();
+  if (text.length >= 200) return false;
+  if (JS_REQUIRED.test(text)) return true;
+  return $('div#root, div#app, div#__next').toArray().some((el) => $(el).text().trim() === '');
+}
+
+/**
+ * Retry-After in ms, or null when the header is absent or unreadable.
+ * @param {Response} response
+ */
+function retryAfterMs(response) {
+  const header = response.headers?.get?.('retry-after');
+  if (header == null) return null;
+  const raw = String(header).trim();
+  if (!/^\d+$/.test(raw) && Number.isNaN(Date.parse(raw))) return null;
+  return parseRetryAfter(raw);
+}
+
+/**
+ * fetchWithTimeout, with undici's bare "fetch failed" (DNS, refused, reset)
+ * given the URL and the reason it keeps in `cause`, as the REST route's
+ * FETCH_FAILED "Could not reach <url>" does.
+ */
+async function fetchOnce(url, options) {
+  try {
+    return await fetchWithTimeout(url, options);
+  } catch (error) {
+    if (!(error instanceof TypeError) || error.message !== 'fetch failed') throw error;
+    const reason = error.cause?.message || error.cause?.code;
+    throw new Error(`Could not reach ${url}${reason ? ` (${reason})` : ''}`, { cause: error });
+  }
+}
+
+/**
+ * The plain fetch, retried once on a 429/503 whose Retry-After is at most
+ * RETRY_AFTER_MAX_MS. fetchWithTimeout has already recorded the Retry-After
+ * (noteRetryAfter), and the second request's gate waits it out in the
+ * per-host throttle — so the wait happens once, there.
+ */
+async function plainFetch(url, options) {
+  const response = await fetchOnce(url, options);
+  if (response.status !== 429 && response.status !== 503) return response;
+  const waitMs = retryAfterMs(response);
+  if (waitMs === null || waitMs > RETRY_AFTER_MAX_MS) return response;
+  return fetchOnce(url, options);
+}
+
+/**
+ * @param {string} url
+ * @param {{ tool: 'extract_text'|'extract_links', userAgent?: string, respectRobots?: boolean,
+ *   escalate?: boolean, escalateEngine?: string,
+ *   escalateFetch?: (args: { url: string, engine: string, respectRobots?: boolean }) => Promise<object> }} options
+ * @returns {Promise<
+ *   { unsupported: string } |
+ *   { error: string, url: string, status?: number|null, blocked?: object, warnings: string[], fields: object } |
+ *   { html: string, url: string, type: string, warnings: string[], fields: object }>}
+ *   `fields` carries `escalated` (only when escalate was asked for), `stealth`
+ *   (when the stage ran) and `rendered: false` (a client-rendered shell)
+ */
+export async function fetchLadder(url, { tool, userAgent, respectRobots, escalate = false, escalateEngine = 'auto', escalateFetch } = {}) {
+  let escalationRan = false;
+  try {
+    const response = await plainFetch(url, { timeout: LADDER_TIMEOUT_MS, userAgent, respectRobots, tool });
+    const type = mediaType(response);
+    if (response.ok && type && !TEXTUAL_TYPE.test(type)) return { unsupported: type };
+
+    const warnings = [...(response._warnings ?? [])];
+    // Reported only to a caller who asked to escalate.
+    const fields = escalate ? { escalated: false } : {};
+    const verdict = plainVerdict(response, url);
+    const failed = verdict ? !verdict.success : !response.ok;
+
+    // A 404 or a 5xx never escalates (aBrowserMightPass); a 444 is a refusal
+    // of our network, which a browser on another identity can pass. A host
+    // whose Retry-After is still open (over 10 s, or repeated on the retry)
+    // asked us to back off, and a stealth request now would override it (G6).
+    const backingOff = (response.status === 429 || response.status === 503) && retryAfterMs(response) !== null;
+    if (escalate && failed && !backingOff && (aBrowserMightPass(verdict, response.status) || response.status === 444)) {
+      const vendorDetected = verdict?.blocked?.vendor ?? null;
+      if (!escalateFetch) {
+        warnings.push('escalate: no stealth stage is wired into this server build');
+      } else {
+        try {
+          const stealth = await escalateFetch({ url: response.url || url, engine: escalateEngine, respectRobots });
+          escalationRan = true;
+          const engine = stealth.engine || escalateEngine;
+          // The stage's gate repeats the plain fetch's robots notes; say each once.
+          for (const w of stealth.warnings ?? []) if (!warnings.includes(w)) warnings.push(w);
+          const html = stealth.html || '';
+          const finalUrl = stealth.url || response.url || url;
+          const status = stealth.status ?? null;
+          const rendered = stealthDocumentVerdict(
+            { url: finalUrl, title: stealth.title || '', text: stealth.text || '', html, status },
+            { waitedMs: stealth.gracedMs || 0, fetcher: 'the stealth browser', rendered: true, contentReturned: false }
+          );
+          warnings.push(
+            `escalate: the plain fetch ${vendorDetected ? `was blocked by ${vendorDetected}` : 'did not return the page'}; ` +
+            `${engine === 'impit' ? 'a Chrome TLS handshake (impit, honest User-Agent)' : `the ${engine} stealth browser`} ${rendered.success ? 'returned it' : 'did not get it either'}`
+          );
+          fields.escalated = true;
+          fields.stealth = { engine, vendor_detected: vendorDetected };
+          if (!rendered.success) {
+            return { error: rendered.error, url: finalUrl, status, ...(rendered.blocked ? { blocked: rendered.blocked } : {}), warnings, fields };
+          }
+          return { html, url: finalUrl, type: 'text/html', warnings, fields };
+        } catch (err) {
+          // A robots refusal inside the gate stamps markPreflightRefusal, and
+          // withAuth then bills the whole call zero: nothing was rendered.
+          warnings.push(`escalate: the stealth retry did not run — ${err.message}`);
+        }
+      }
+    }
+
+    const failure = targetFailure(response, url, verdict);
+    if (failure) {
+      return { error: failure, url: response.url || url, status: response.status, ...(verdict?.blocked ? { blocked: verdict.blocked } : {}), warnings, fields };
+    }
+
+    const html = response._body ?? '';
+    if (!isJsonType(type) && isClientRenderedShell(html)) {
+      warnings.push(CLIENT_RENDERED_WARNING);
+      fields.rendered = false;
+    }
+    return { html, url: response.url || url, type, warnings, fields };
+  } finally {
+    // The projection is 1+5 when escalating; the stage's 5 is owed only if it ran.
+    if (escalate) setActualCost(EXTRACT_BASE_CREDITS + (escalationRan ? SCRAPE_ESCALATION_CREDITS : 0));
+  }
+}
+
+/**
+ * The error result for a ladder that did not return a page. Unescalated, the
+ * E1 text error; with escalate asked for, a JSON body carrying `escalated`,
+ * so withAuth's hint does not send the caller back to the stealth browser.
+ * @param {string} tool
+ * @param {string} prefix e.g. 'Failed to extract text: '
+ * @param {object} ladder fetchLadder's result
+ */
+export function ladderErrorResult(tool, prefix, ladder) {
+  if (ladder.unsupported) {
+    return {
+      content: [{
+        type: 'text',
+        text: `${prefix}UNSUPPORTED_CONTENT_TYPE: Unsupported content type ${ladder.unsupported}: ${tool} reads ${tool === 'extract_links' ? 'HTML' : 'HTML and text'}\n` +
+          'Next step: process_document reads PDF and DOCX documents from a URL (2 credits); an image or an archive has no text to extract.'
+      }],
+      isError: true
+    };
+  }
+  if (!('escalated' in ladder.fields)) {
+    return { content: [{ type: 'text', text: `${prefix}${ladder.error}` }], isError: true };
+  }
+  return {
+    content: [{
+      type: 'text',
+      text: JSON.stringify({
+        success: false,
+        url: ladder.url,
+        ...(typeof ladder.status === 'number' ? { status: ladder.status } : {}),
+        error: ladder.error,
+        ...(ladder.blocked ? { blocked: ladder.blocked } : {}),
+        ...ladder.fields,
+        ...(ladder.warnings.length > 0 ? { warnings: ladder.warnings } : {})
+      }, null, 2)
+    }],
+    isError: true
+  };
+}