crawlforge-mcp-server 6.18.0 → 6.18.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.18.0",
+  "version": "6.18.1",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 31 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -121,7 +121,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.13.0",
+    "crawlforge-extractors": "^1.15.0",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",

package/server.js

@@ -113,7 +113,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.18.0",
+  version: "6.18.1",
   description: "Production-ready MCP server with 31 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, stateful browser sessions with element refs, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -1031,7 +1031,7 @@ registerToolIfEnabled("scrape_with_actions", {
       fields: z.array(z.object({
         selector: z.string(),
         value: z.string(),
-        type: z.enum(['text', 'select', 'checkbox', 'radio', 'file']).default('text'),
+        type: z.enum(['text', 'select', 'checkbox', 'radio', 'file']).default('text').describe("How the field is filled: text types the value, select picks the option by value or label, checkbox/radio check the input (a selector matching a group checks the member whose value attribute equals value; a checkbox with value \"false\" is unchecked). file is not supported and fails that action"),
         waitAfter: z.number().min(0).max(5000).default(100)
       })),
       submitSelector: z.string().optional(),

package/src/core/ActionExecutor.js

@@ -129,6 +129,17 @@ const SelectActionSchema = BaseActionSchema.extend({
   message: 'Select action requires value or values'
 });
 
+// Not on any tool's public action list: scrape_with_actions' formAutoFill emits
+// it for a field declared `type: "checkbox"` or `type: "radio"`. `fieldType`
+// is a plain string so that a field type nothing here can fill ("file")
+// arrives as an action that fails, instead of being typed into as text.
+const CheckActionSchema = BaseActionSchema.extend({
+  type: z.literal('check'),
+  selector: z.string(),
+  value: z.string(),
+  fieldType: z.string()
+});
+
 const HoverActionSchema = BaseActionSchema.extend({
   type: z.literal('hover'),
   selector: z.string(),
@@ -175,6 +186,7 @@ const ActionSchema = z.union([
   PressActionSchema,
   ScrollActionSchema,
   SelectActionSchema,
+  CheckActionSchema,
   HoverActionSchema,
   NavigateActionSchema,
   ScreenshotActionSchema,
@@ -1029,6 +1041,8 @@ export class ActionExecutor extends EventEmitter {
         return await this.executeScrollAction(page, action);
       case 'select':
         return await this.executeSelectAction(page, action);
+      case 'check':
+        return await this.executeCheckAction(page, action);
       case 'hover':
         return await this.executeHoverAction(page, action);
       case 'navigate':
@@ -1309,6 +1323,45 @@ export class ActionExecutor extends EventEmitter {
     };
   }
 
+  /**
+   * Execute check action — a formAutoFill checkbox or radio field.
+   *
+   * A selector that names a whole group (`input[name="size"]`) is narrowed to
+   * the member whose `value` attribute is the field's value, the way the form
+   * itself pairs name and value; `.first()` would check whichever came first.
+   * A selector that names one input checks that input, except a checkbox with
+   * value "false", which is unchecked.
+   * @param {Page} page - Playwright page
+   * @param {Object} action - Check action
+   * @returns {Promise<Object>} Check result
+   */
+  async executeCheckAction(page, action) {
+    if (action.fieldType !== 'checkbox' && action.fieldType !== 'radio') {
+      throw new Error(
+        `formAutoFill field type "${action.fieldType}" is not supported ` +
+        '(supported: text, select, checkbox, radio)'
+      );
+    }
+
+    const timeout = this.actionTimeout(action);
+    const matches = page.locator(this.resolveSelector(page, action.selector));
+    await matches.first().waitFor({ state: 'attached', timeout });
+
+    const isGroup = (await matches.count()) > 1;
+    const target = isGroup
+      ? matches.and(page.locator(`[value=${JSON.stringify(action.value)}]`)).first()
+      : matches.first();
+    const checked = isGroup || action.fieldType === 'radio' || action.value !== 'false';
+
+    await target.setChecked(checked, { timeout });
+
+    return {
+      selector: action.selector,
+      value: action.value,
+      checked
+    };
+  }
+
   /**
    * Execute hover action
    * @param {Page} page - Playwright page

package/src/core/AgentOrchestrator.js

@@ -172,6 +172,40 @@ export function isCurrentStateTask(prompt) {
   return CURRENT_STATE_RE.test(prompt || '');
 }
 
+/**
+ * The names a prompt uses: words carrying a capital letter that is not just
+ * the capital opening a sentence ("TypeScript", "Hacker", "News", "NASA"; not
+ * "What"). These are what the task is about, so a planned query that drops
+ * one has lost its subject.
+ */
+export function namedTerms(prompt) {
+  const words = (prompt || '').split(/\s+/);
+  const terms = [];
+  words.forEach((raw, i) => {
+    if (raw.includes('://')) return;
+    const word = raw.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, '').replace(/['’]s$/, '');
+    if (word.length < 2) return;
+    const opensSentence = i === 0 || /[.?!:]$/.test(words[i - 1]);
+    if (!/\p{Lu}/u.test(opensSentence ? word.slice(1) : word)) return;
+    if (!terms.some(t => t.toLowerCase() === word.toLowerCase())) terms.push(word);
+  });
+  return terms;
+}
+
+/**
+ * A task scoped to its seed URLs: the caller passed `urls` and the prompt
+ * points at them ("this page", "these URLs", "the given site"). The answer is
+ * then on those pages and nowhere else, so GATHER runs no web search — search
+ * results for the task's words are other people's pages. Asked how many books
+ * "this page" lists, the agent searched "count books on webpage" and merged
+ * Reddit threads about word counts into the evidence (R24, 2026-10-03).
+ */
+const SEED_SCOPE_RE = /\b(?:this|these|the (?:given|provided|above|following|seed))\s+(?:web\s?)?(?:pages?|urls?|sites?|websites?|links?|documents?|articles?)\b/i;
+
+export function isSeedScopedTask(prompt, seedUrls) {
+  return Array.isArray(seedUrls) && seedUrls.length > 0 && SEED_SCOPE_RE.test(prompt || '');
+}
+
 // ── Orchestrator ──────────────────────────────────────────────────────────────
 
 export class AgentOrchestrator {
@@ -358,6 +392,17 @@ export class AgentOrchestrator {
       );
     }
     // A current-state plan is told to make its first query the bare entity
+    // name, and for a prompt naming two things a small model can keep the
+    // wrong one: "latest stable version of the TypeScript npm package" was
+    // searched as the single word "npm" (R24, 2026-10-03). The first query
+    // carries every name the prompt uses; what the planner wrote stays as the
+    // qualifier ("TypeScript npm").
+    if (currentState) {
+      const first = searchQueries[0].toLowerCase();
+      const dropped = namedTerms(prompt).filter(t => !first.includes(t.toLowerCase()));
+      if (dropped.length > 0) searchQueries[0] = `${dropped.join(' ')} ${searchQueries[0]}`;
+    }
+    // A current-state plan is told to make its first query the bare entity
     // name, and a small model often stops there. The bare query surfaces the
     // live front page, which rarely states the fact asked for: "what does
     // Southwest charge for a first checked bag" fetched southwest.com's home,
@@ -397,7 +442,7 @@ export class AgentOrchestrator {
     /** url -> { title, text }: the engine's excerpt, evidence of last resort for a page that cannot be fetched. */
     const excerpts = new Map();
 
-    if (urlQueue.length < capUrls) {
+    if (urlQueue.length < capUrls && !isSeedScopedTask(prompt, seedUrls)) {
       try {
         const searchTool = await this._getSearchTool();
         for (const q of searchQueries) {
@@ -607,14 +652,24 @@ export class AgentOrchestrator {
           schema,
           provider: 'auto'
         });
+        // A required field left null is an incomplete answer, not a
+        // structured success (R24, 2026-10-03).
+        const missingRequired = result.success && Array.isArray(schema.required)
+          ? schema.required.filter(key => result.data?.[key] == null || result.data[key] === '')
+          : [];
+        const missingReason = missingRequired.length > 0
+          ? `Required ${missingRequired.length === 1 ? 'field' : 'fields'} ${missingRequired.map(k => `"${k}"`).join(', ')} ` +
+            `${missingRequired.length === 1 ? 'is' : 'are'} null: the fetched sources do not state ${missingRequired.length === 1 ? 'it' : 'them'}.`
+          : undefined;
+        if (missingReason) warnings.push(missingReason);
         return {
           success: result.success,
           answer: result.success ? result.data : null,
           structured: true,
           search_results: searchResults,
           evidence: evidence.map(e => (e.via ? { url: e.url, via: e.via } : { url: e.url })),
-          degraded: !result.success,
-          reason: result.success ? undefined : result.error,
+          degraded: !result.success || missingRequired.length > 0,
+          reason: result.success ? missingReason : result.error,
           steps: step,
           urls_fetched: urlsFetched,
           stealth_retries: usage.escalations,

package/src/core/crawlers/BFSCrawler.js

@@ -219,6 +219,12 @@ export class BFSCrawler {
       return;
     }
     this.visited.set(normalizedUrl, url);
+    // Budget spent: drop what is still waiting. Each of those tasks would only
+    // return at the cap check above, but the queue releases 10 a second, so a
+    // seed with 2,290 links held a max_pages:3 crawl open for 232 s (R24).
+    if (this.visited.size >= this.maxPages) {
+      this.queue.clear();
+    }
 
     try {
       // Check cache first
@@ -318,6 +324,7 @@ export class BFSCrawler {
         url,
         depth,
         error: error.message,
+        ...(error.code ? { code: error.code } : {}),
         timestamp: new Date().toISOString()
       });
     }

package/src/core/queue/QueueManager.js

@@ -77,6 +77,9 @@ export class QueueManager {
   getStats() {
     return {
       ...this.stats,
+      // Read live: the 'active' event fires when a task starts, never when the
+      // last one ends, so the stored value stayed at 8 on a finished crawl.
+      active: this.queue.pending,
       size: this.queue.size,
       pending: this.queue.pending,
       isPaused: this.queue.isPaused

package/src/schemas/toolOutputSchemas.js

@@ -181,6 +181,7 @@ const mapSiteShape = {
     url: z.string().optional(),
     score: z.number().optional()
   }).passthrough()).optional().describe('Present only when the `search` param was set'),
+  warnings: z.array(z.string()).optional().describe('Present when the map is partial, e.g. the start page could not be read and the URLs come from the sitemap only'),
   _cost: costShape
 };
 
@@ -378,16 +379,17 @@ const crawlDeepShape = {
   error: z.string().optional(),
   url: z.string().optional(),
   crawl_depth: z.number().optional(),
-  pages_crawled: z.number().optional(),
-  pages_found: z.number().optional(),
-  error_count: z.number().optional(),
+  pages_attempted: z.number().optional().describe('URLs the crawl tried: pages_crawled + error_count'),
+  pages_crawled: z.number().optional().describe('Pages fetched and returned in results'),
+  pages_found: z.number().optional().describe('Same as pages_crawled; kept for existing clients'),
+  error_count: z.number().optional().describe('URLs that failed; one entry each in errors[]'),
   duration_ms: z.number().optional(),
   pages_per_second: z.number().optional(),
   results: z.array(crawlDeepPageShape).optional(),
   errors: z.array(z.unknown()).optional(),
   stats: z.unknown().optional(),
   site_structure: z.object({
-    total_pages: z.number().optional(),
+    total_pages: z.number().optional().describe('Equals pages_crawled; failed URLs are not counted'),
     depth_distribution: z.record(z.number()).optional()
       .describe('Pages per crawl depth (links from the start URL)'),
     path_depth_distribution: z.record(z.number()).optional()

package/src/skills/agent-skills/crawlforge-batch-automation/references/actions.md

@@ -154,7 +154,7 @@ main frame only — elements inside iframes and shadow DOM get no refs.
 | `formats` | `["json"]` | `markdown`, `html`, `json`, `text`, `screenshots`. |
 | `captureIntermediateStates` | `false` | Snapshot after each action. |
 | `captureScreenshots` | `true` | Screenshot during execution. |
-| `formAutoFill` | — | Declarative form fill: `fields[]` + `submitSelector`. |
+| `formAutoFill` | — | Declarative form fill: `fields[]` + `submitSelector`. Each field's `type` picks how it is filled: `text` (default) types, `select` picks the option by value or label, `checkbox`/`radio` check the input (a group selector such as `input[name=size]` checks the member whose value attribute equals `value`). `file` is not supported and fails that action. |
 | `browserOptions` | — | `headless`, `userAgent`, `viewportWidth/Height`, `timeout`. |
 | `continueOnActionError` | `false` | Keep going if one action fails. |
 | `maxRetries` | `0` | 0–3 whole-chain retries on failure. A retry re-navigates to the starting URL and replays every action; each run is reported under `attempts[]`. |

package/src/tools/advanced/BrowserSessionTool.js

@@ -184,6 +184,18 @@ function withJsResult(result) {
   return { ...result, jsResult: result.result.result };
 }
 
+/**
+ * A screenshot action's image travels once, in `screenshots[]`, which the
+ * server publishes as crawlforge://screenshot/{actionId} — the action's id.
+ * Its copy in the action result is replaced by that URI, as scrape_with_actions
+ * does: left in, the same base64 came back inline beside the resource (R24).
+ */
+function withoutScreenshotData(result) {
+  if (result?.type !== 'screenshot' || typeof result.result?.data !== 'string') return result;
+  const { data, ...rest } = result.result;
+  return { ...result, result: { ...rest, resourceUri: `crawlforge://screenshot/${result.id}` } };
+}
+
 /**
  * The gate's warnings — a respect_robots override, a crawl-delay note — are
  * what the shared parameter description promises the caller gets back
@@ -517,7 +529,7 @@ export class BrowserSessionTool {
         : {}),
       ...gateWarningFields(session.page),
       error: result.error,
-      actionResults: result.results.map(withJsResult),
+      actionResults: result.results.map(withJsResult).map(withoutScreenshotData),
       screenshots: result.screenshots,
       ...(result.capturedStates.length > 0 ? { capturedStates: result.capturedStates } : {}),
       stats: result.stats

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -593,7 +593,10 @@ export class ScrapeWithActionsTool extends EventEmitter {
       actionResults,
       attempt: chainResult.attempt ?? attempts.length,
       attempts,
-      totalActions: params.actions.length,
+      // The chain that was run — the caller's actions plus the fills, waits
+      // and submit formAutoFill adds — so it is the denominator of the three
+      // counts below; counting only the caller's own read "1 of 11".
+      totalActions: actionChain.length,
       successfulActions: actionResults.filter(r => r.success).length,
       failedActions: actionResults.filter(r => !r.success).length,
       actionsExecuted: actionResults.length, // Total executed (for validation)
@@ -635,10 +638,18 @@ export class ScrapeWithActionsTool extends EventEmitter {
     if (Array.isArray(formAutoFill.fields)) {
       // Structured shape: { fields: [{selector, value, type, waitAfter}], submitSelector, waitAfterSubmit }
       for (const field of formAutoFill.fields) {
+        // Each field type is filled the way that control is: typing into a
+        // radio, a checkbox or a <select> "succeeds" and changes nothing, so
+        // the form was posted without them and nothing failed (R24). `check`
+        // also carries the types nothing can fill ("file"), which it refuses.
+        const fill = field.type === 'text'
+          ? { type: 'type', text: field.value }
+          : field.type === 'select'
+            ? { type: 'select', value: field.value }
+            : { type: 'check', value: field.value, fieldType: field.type };
         fillActions.push({
-          type: 'type',
+          ...fill,
           selector: field.selector,
-          text: field.value,
           description: `Auto-fill field: ${field.selector}`,
           continueOnError: true,
           retries: 1

package/src/tools/advanced/batchScrape/worker.js

@@ -14,6 +14,7 @@ import { fetchResigned } from '../../../utils/resignedFetch.js';
 import { htmlToMarkdown } from '../../../utils/htmlToMarkdown.js';
 import { elementText } from '../../../utils/elementText.js';
 import { pageTitle } from '../../../utils/pageTitle.js';
+import { classifyContentType, unsupportedContentTypeError } from '../../extract/_fetchAndParse.js';
 
 /**
  * Fetch a URL with AbortController timeout (SSRF-guarded + per-host throttled).
@@ -41,6 +42,13 @@ export async function fetchUrl(url, options = {}) {
     if (response.status === 429 || response.status === 503) {
       noteRetryAfter(url, response.headers?.get?.('retry-after'));
     }
+    // A PDF decoded as text came back success:true with "%PDF-1.4 …" as the
+    // page (R24 1.5). Refused before the body is read, with the wording
+    // `scrape` gives the same URL; an error status keeps its own message.
+    const contentType = response.headers?.get?.('content-type') || null;
+    if (response.ok && classifyContentType(contentType) === 'binary') {
+      throw unsupportedContentTypeError(contentType);
+    }
     // crawlforge-extractors' readBody: same size cap, but decoded with the
     // body's real charset — the local reader this replaces decoded everything
     // as UTF-8, which turned lua.org's ISO-8859-1 "português" into "portugu�s".

package/src/tools/advanced/scrapeWithActions/recorder.js

@@ -168,6 +168,8 @@ export function buildRecordedEntry(action, timestampMsSinceStart) {
   if (action.duration !== undefined) entry.duration = action.duration;
   if (action.url !== undefined) entry.url = action.url;
   if (action.value !== undefined) entry.value = action.value;
+  // check (a formAutoFill checkbox/radio field) does not replay without it
+  if (action.fieldType !== undefined) entry.fieldType = action.fieldType;
   if (action.direction !== undefined) entry.direction = action.direction;
   if (action.distance !== undefined) entry.distance = action.distance;
   // scroll: absolute scroll-to coordinates

package/src/tools/crawl/crawlDeep.js

@@ -261,19 +261,30 @@ export class CrawlDeepTool {
         });
         const duration = Date.now() - startTime;
 
+        // The counters, each defined once: a URL the crawl tried either
+        // produced a page (results) or an error (errors), so
+        // pages_attempted = pages_crawled + error_count. pages_crawled used to
+        // count attempts, which read "3 crawled, 1 found, 2 errors" (R24).
+        const succeeded = results.results.length;
+        const failed = results.errors.length;
+
         // Process and format results
         const response = {
           url: validated.url,
           crawl_depth: effectiveMaxDepth,
-          pages_crawled: results.urls.length,
-          pages_found: results.results.length,
-          error_count: results.errors.length,
+          pages_attempted: succeeded + failed,
+          pages_crawled: succeeded,
+          pages_found: succeeded,
+          error_count: failed,
           duration_ms: duration,
-          pages_per_second: results.urls.length / (duration / 1000),
+          // A seed refused before any I/O finishes in 0 ms; x / 0 is Infinity,
+          // which the output schema rejects as "expected number".
+          pages_per_second: succeeded / (Math.max(duration, 1) / 1000),
           results: this.formatResults(results.results, validated.extract_content, validated.content_max_length),
           errors: results.errors,
           stats: results.stats,
-          site_structure: this.analyzeSiteStructure(results.urls, results.results),
+          // Pages only: a URL that failed is in errors[], not in the site's structure.
+          site_structure: this.analyzeSiteStructure(results.results.map(page => page.url), results.results),
           domain_filter_config: domainFilter ? domainFilter.exportConfig() : null,
           link_analysis: results.linkAnalysis,
           session: sessionContext

package/src/tools/crawl/mapSite.js

@@ -48,6 +48,9 @@ export function searchScore(url, search) {
   return terms.filter((t) => haystack.includes(t)).length;
 }
 
+// A gate refusal is the answer to the request, not a page we failed to read.
+const REFUSAL_CODES = new Set(['ROBOTS_DISALLOWED', 'HOST_BLOCKED', 'USE_REDDIT_SEARCH', 'SSRF_BLOCKED']);
+
 // Lazy singleton — avoids creating a CacheManager timer per request
 let _ranker = null;
 function getRanker() {
@@ -159,13 +162,24 @@ export class MapSiteTool {
         sitemapUrls.forEach(url => urls.add(normalizeUrl(url)));
       }
 
-      // Fetch and parse the main page for additional URLs
-      const pageUrls = await this.fetchPageUrls(validated.url, domainFilter, identity);
-      pageUrls.forEach(url => {
-        if (urls.size < poolLimit) {
-          urls.add(normalizeUrl(url));
+      // Fetch and parse the main page for additional URLs. A start page that
+      // cannot be read is an error when the sitemap gave nothing either: an
+      // unresolvable host used to return a success with total_urls: 0 (R24).
+      try {
+        const pageUrls = await this.fetchPageUrls(validated.url, domainFilter, identity);
+        pageUrls.forEach(url => {
+          if (urls.size < poolLimit) {
+            urls.add(normalizeUrl(url));
+          }
+        });
+      } catch (error) {
+        if (REFUSAL_CODES.has(error.code) || error.message.startsWith('SSRF Protection:')) throw error;
+        const reason = error.cause?.message || error.message;
+        if (urls.size === 0) {
+          throw new Error(`${validated.url} could not be read (${reason}) and no sitemap was found`);
         }
-      });
+        warnings.push(`The start page could not be read (${reason}); the URLs come from the sitemap only.`);
+      }
 
       let pool = Array.from(urls);
       if (scopePath) {
@@ -304,46 +318,40 @@ export class MapSiteTool {
   }
 
   async fetchPageUrls(url, domainFilter = null, identity = {}) {
-    try {
-      const response = await this.fetchWithTimeout(url, identity);
-      if (!response.ok) {
-        return [];
-      }
+    // Throws when the page cannot be read — a gate or SSRF refusal, a network
+    // failure, a non-2xx status. execute() decides which of those end the call.
+    const response = await this.fetchWithTimeout(url, identity);
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}`);
+    }
 
-      const html = await response.text();
-      const $ = load(html);
-      const urls = new Set();
-      const baseUrl = getBaseUrl(url);
-
-      // Extract all links
-      $('a[href]').each((_, element) => {
-        const href = $(element).attr('href');
-        if (href && !href.startsWith('#') && !href.startsWith('javascript:')) {
-          try {
-            const absoluteUrl = new URL(href, url);
-            // Only include URLs from the same domain
-            if (absoluteUrl.origin === new URL(baseUrl).origin) {
-              const urlString = absoluteUrl.toString();
-              
-              // Apply domain filter if provided
-              if (!domainFilter || domainFilter.isAllowed(urlString).allowed) {
-                urls.add(urlString);
-              }
+    const html = await response.text();
+    const $ = load(html);
+    const urls = new Set();
+    const baseUrl = getBaseUrl(url);
+
+    // Extract all links
+    $('a[href]').each((_, element) => {
+      const href = $(element).attr('href');
+      if (href && !href.startsWith('#') && !href.startsWith('javascript:')) {
+        try {
+          const absoluteUrl = new URL(href, url);
+          // Only include URLs from the same domain
+          if (absoluteUrl.origin === new URL(baseUrl).origin) {
+            const urlString = absoluteUrl.toString();
+
+            // Apply domain filter if provided
+            if (!domainFilter || domainFilter.isAllowed(urlString).allowed) {
+              urls.add(urlString);
             }
-          } catch {
-            // Invalid URL, skip
           }
+        } catch {
+          // Invalid URL, skip
         }
-      });
+      }
+    });
 
-      return Array.from(urls);
-    } catch (error) {
-      // A gate refusal is the answer to the request, not a page we failed to
-      // read: surface it instead of returning an emptier map than the caller
-      // would notice.
-      if (error.code === 'ROBOTS_DISALLOWED' || error.code === 'HOST_BLOCKED' || error.code === 'USE_REDDIT_SEARCH') throw error;
-      return [];
-    }
+    return Array.from(urls);
   }
 
   async fetchMetadata(urls, metadataMap, identity = {}) {

package/src/tools/extract/_fetchAndParse.js

@@ -58,6 +58,13 @@ async function readTextWithSizeCap(response) {
   return readBody(response, { maxBytes: maxBodySize });
 }
 
+/** The refusal for a binary response; batch_scrape gives the same one (R24 1.5). */
+export function unsupportedContentTypeError(contentType) {
+  return new Error(
+    `Unsupported content type "${contentType}" — this looks like binary content, not HTML/text. Use process_document for PDFs/documents/binary files.`
+  );
+}
+
 /**
  * Classify a Content-Type header for the purposes of HTML parsing.
  * Missing header is treated as 'html' (permissive default — many servers,
@@ -65,7 +72,7 @@ async function readTextWithSizeCap(response) {
  * @param {string|null} contentType
  * @returns {'html'|'text'|'binary'}
  */
-function classifyContentType(contentType) {
+export function classifyContentType(contentType) {
   if (!contentType) return 'html';
   const type = contentType.split(';')[0].trim().toLowerCase();
   if (
@@ -146,9 +153,7 @@ export async function fetchAndParse(url, options = {}) {
   }
 
   if (classification === 'binary') {
-    throw new Error(
-      `Unsupported content type "${contentType}" — this looks like binary content, not HTML/text. Use process_document for PDFs/documents/binary files.`
-    );
+    throw unsupportedContentTypeError(contentType);
   }
 
   const html = await readTextWithSizeCap(response);

package/src/tools/scrape/unifiedScrape.js

@@ -699,16 +699,19 @@ export class UnifiedScrapeTool {
         case 'text':
           try {
             const { load } = await import('cheerio');
+            // cheerio parses a <noscript> body as raw text, so left in it
+            // reaches `text` as literal tags (lemonde.fr's challenge page,
+            // 2026-10-03); extract_text and the markdown format drop it too.
             if (onlyMainContent) {
               // Plain text from Readability main content via cheerio
               const $main = load(getMainHtml());
-              $main('script, style').remove();
+              $main('script, style, noscript').remove();
               content.text = extractBlockText($main);
             } else {
               // Strip script/style on a clone, not the shared $, so other
               // formats reading $ later aren't affected by format ordering.
               const $clone = load($.html());
-              $clone('script, style').remove();
+              $clone('script, style, noscript').remove();
               content.text = extractBlockText($clone);
             }
           } catch (err) {
@@ -790,6 +793,15 @@ export class UnifiedScrapeTool {
     if (queryScoped && !formats.includes('markdown')) {
       warnings.push('offsets index the "markdown" format of this call (same onlyMainContent); add "markdown" to formats to quote with a locator');
     }
+    // A success with nothing in it must say so: the caller otherwise cannot
+    // tell an empty page from a scrape that lost the content.
+    if (typeof content.markdown === 'string' && content.markdown.trim() === '' && !warnings.some((w) => w.startsWith('markdown:'))) {
+      warnings.push(
+        'markdown: the page produced no markdown' +
+        (onlyMainContent ? '; retry with onlyMainContent:false if main-content extraction dropped it, or' : ';') +
+        ' use escalate:true if the page is rendered by JavaScript'
+      );
+    }
     if (!queryScoped && typeof content.markdown === 'string' && content.markdown.length > OVERSIZED_MARKDOWN_CHARS) {
       warnings.push(`markdown: ${content.markdown.length} characters; ask for {type:"highlights", query} (1 extra credit, no model) to get only the matching sentences, table rows and code blocks`);
     }

package/src/tools/tracking/trackChanges/index.js

@@ -308,6 +308,15 @@ export class TrackChangesTool extends EventEmitter {
   async setupMonitoring(params) {
     const { url, monitoringOptions, trackingOptions, storageOptions, notificationOptions } = params;
 
+    // enabled:false turns the URL's polling monitor off and starts nothing.
+    if (monitoringOptions.enabled === false) {
+      return {
+        success: true, operation: 'monitor', url,
+        monitoring: { enabled: false, stopped: this.stopMonitoring(url) },
+        timestamp: Date.now()
+      };
+    }
+
     if (this.activeMonitors.has(url)) {
       clearInterval(this.activeMonitors.get(url).timer);
     }
@@ -498,6 +507,15 @@ export class TrackChangesTool extends EventEmitter {
   async stopScheduledMonitor(params) {
     const { url, scheduledMonitorOptions } = params;
     const monitorId = scheduledMonitorOptions?.monitorId;
+    // A polling monitor (operation:"monitor") is listed as poll:<url>.
+    if (monitorId?.startsWith('poll:')) {
+      const stopped = this.stopMonitoring(monitorId.slice(5));
+      return {
+        success: stopped, operation: 'stop_scheduled_monitor', monitorId, stopped,
+        ...(stopped ? {} : { error: `No polling monitor found with id ${monitorId}` }),
+        timestamp: Date.now()
+      };
+    }
     if (monitorId) {
       if (!this.monitorStore._loaded) await this.monitorStore.load();
       if (this.monitorStore.get(monitorId)) {
@@ -517,6 +535,7 @@ export class TrackChangesTool extends EventEmitter {
     }
     if (!url) throw new Error('stop_scheduled_monitor requires a url or scheduledMonitorOptions.monitorId');
     const result = await this.scheduler.stopByUrl(url);
+    const stoppedPolling = this.stopMonitoring(url) ? 1 : 0;
     // Only a hosted monitor whose every target is this URL — never a
     // multi-target monitor that merely includes it.
     let stoppedHosted = 0;
@@ -533,7 +552,7 @@ export class TrackChangesTool extends EventEmitter {
       hostedError = error.message;
     }
     return {
-      success: true, operation: 'stop_scheduled_monitor', url, stoppedMonitors: result.stopped, stoppedHosted,
+      success: true, operation: 'stop_scheduled_monitor', url, stoppedMonitors: result.stopped, stoppedPolling, stoppedHosted,
       ...(hostedError ? { hostedError } : {}),
       timestamp: Date.now()
     };
@@ -541,7 +560,10 @@ export class TrackChangesTool extends EventEmitter {
 
   async listScheduledMonitors() {
     if (!this.monitorStore._loaded) await this.monitorStore.load();
-    const local = this.scheduler.list().map((m) => ({ ...m, hosted: false }));
+    const local = [
+      ...this._pollingMonitors().map((m) => ({ ...m, hosted: false })),
+      ...this.scheduler.list().map((m) => ({ ...m, hosted: false }))
+    ];
     // The local list never fails because the website is unreachable.
     let hosted = [];
     let hostedError = null;
@@ -560,16 +582,21 @@ export class TrackChangesTool extends EventEmitter {
     };
   }
 
+  /** The in-process polling monitors operation:"monitor" started, as the dashboard and the list show them. */
+  _pollingMonitors() {
+    return Array.from(this.activeMonitors.entries()).map(([url, m]) => ({
+      id: `poll:${url}`, url, kind: 'polling', status: 'active',
+      interval: m.options?.interval ?? null, stats: m.stats
+    }));
+  }
+
   async getMonitoringDashboard(params) {
     // The schema defaults every flag to true, but only inside a dashboardOptions
     // object the caller supplied; an omitted object must mean the same thing.
     const dashboardOptions = { includeRecentAlerts: true, includeTrends: true, includeMonitorStatus: true, ...(params.dashboardOptions || {}) };
     const dashboard = this.changeTracker.getMonitoringDashboard();
     // The tracker only knows tracked pages; the monitors that poll them live here.
-    const polling = Array.from(this.activeMonitors.entries()).map(([url, m]) => ({
-      id: `poll:${url}`, url, kind: 'polling', status: 'active',
-      interval: m.options?.interval ?? null, stats: m.stats
-    }));
+    const polling = this._pollingMonitors();
     if (!this.monitorStore._loaded) await this.monitorStore.load();
     const scheduled = this.scheduler.list().map(m => ({
       id: m.id, url: m.url, kind: 'scheduled',

package/src/tools/tracking/trackChanges/schema.js

@@ -57,7 +57,7 @@ export const TRACK_CHANGES_INPUT_SHAPE = {
   }).optional().prefault({}).describe("Options for how changes are tracked and compared"),
 
   monitoringOptions: z.object({
-    enabled: z.boolean().default(false),
+    enabled: z.boolean().default(true).describe("operation:\"monitor\" starts polling the URL; false stops the URL's polling monitor and starts nothing"),
     interval: z.number().min(60000).max(24 * 60 * 60 * 1000).default(300000),
     maxRetries: z.number().min(0).max(5).default(3),
     retryDelay: z.number().min(1000).max(60000).default(5000),
@@ -113,7 +113,7 @@ export const TRACK_CHANGES_INPUT_SHAPE = {
     enabled: z.boolean().default(true),
     interval: z.number().min(60000).optional().describe("Polling interval in ms (default 1h)"),
     goal: z.string().optional().describe("Plain-English alert goal; an LLM judges whether a change matches (degrades to threshold if no LLM)"),
-    monitorId: z.string().optional().describe("Monitor id for stop_scheduled_monitor"),
+    monitorId: z.string().optional().describe("Monitor id for stop_scheduled_monitor, as list_scheduled_monitors shows it (poll:<url> for a polling monitor)"),
     notificationThreshold: z.enum(['minor', 'moderate', 'major', 'critical']).optional(),
     hosted: z.boolean().default(false).describe("Run the monitor on CrawlForge's servers: it fires from the hosted scheduler whether or not this process is alive and sends email and signed webhooks. Each check bills 3 credits per compared target from the account; blocked and errored targets are free. Default false = local, in-process."),
     name: z.string().min(1).max(80).optional().describe("Display name for a hosted monitor (default: the URL host)")

package/src/utils/fetchLadder.js

@@ -91,6 +91,21 @@ function retryAfterMs(response) {
   return parseRetryAfter(raw);
 }
 
+/**
+ * fetchWithTimeout, with undici's bare "fetch failed" (DNS, refused, reset)
+ * given the URL and the reason it keeps in `cause`, as the REST route's
+ * FETCH_FAILED "Could not reach <url>" does.
+ */
+async function fetchOnce(url, options) {
+  try {
+    return await fetchWithTimeout(url, options);
+  } catch (error) {
+    if (!(error instanceof TypeError) || error.message !== 'fetch failed') throw error;
+    const reason = error.cause?.message || error.cause?.code;
+    throw new Error(`Could not reach ${url}${reason ? ` (${reason})` : ''}`, { cause: error });
+  }
+}
+
 /**
  * The plain fetch, retried once on a 429/503 whose Retry-After is at most
  * RETRY_AFTER_MAX_MS. fetchWithTimeout has already recorded the Retry-After
@@ -98,11 +113,11 @@ function retryAfterMs(response) {
  * per-host throttle — so the wait happens once, there.
  */
 async function plainFetch(url, options) {
-  const response = await fetchWithTimeout(url, options);
+  const response = await fetchOnce(url, options);
   if (response.status !== 429 && response.status !== 503) return response;
   const waitMs = retryAfterMs(response);
   if (waitMs === null || waitMs > RETRY_AFTER_MAX_MS) return response;
-  return fetchWithTimeout(url, options);
+  return fetchOnce(url, options);
 }
 
 /**