crawlforge-mcp-server 5.2.8 → 5.3.0

package/src/core/research/claimFilters.js

@@ -0,0 +1,235 @@
+/**
+ * Claim admission filters for deep_research synthesis.
+ *
+ * Claims are extractive sentences pulled out of source pages by the summarize
+ * tool, so whatever the page starts with is a candidate claim. On a live run
+ * (2026-08-28, "What anti-bot systems do major websites use in 2026") the top
+ * finding was an arXiv front-matter block beginning "DOI: XXXXXXX.XXXXXXX" —
+ * document furniture, not a research claim. These predicates reject that class
+ * of text before it can become a key finding.
+ */
+
+// Document furniture: DOI stubs, retrieval notes, ACM/arXiv front matter,
+// copyright rows, contact emails. Matched anywhere in the candidate claim.
+const FRONT_MATTER_PATTERNS = [
+  /\bdoi\s*:/i,
+  /\bdoi\.org\//i,
+  /\barxiv\s*:\s*\d/i,
+  /\bccs concepts?\b/i,
+  /\bacm reference format\b/i,
+  /\bretrieved (?:from|on)\b/i,
+  /\bissn\b|\bisbn\b/i,
+  /\ball rights reserved\b/i,
+  /\bcopyright\s*(?:©|\(c\)|\d{4})/i,
+  /[\w.+-]+@[\w-]+\.[a-z]{2,}/i,
+  // "Permission to make digital or hard copies…" — the ACM licence block.
+  /\bpermission to make digital\b/i,
+  // Bot-challenge interstitials. A research run on anti-bot systems scores
+  // these as highly relevant — they are literally about bot detection — so the
+  // relevance gate cannot catch them (live 2026-08-28: "Checking your browser.
+  // This only takes a moment." was the top finding). Phrases are specific to
+  // the challenge chrome; a claim that merely discusses CAPTCHAs is untouched.
+  /\bchecking your browser\b/i,
+  /\bjust a moment\b/i,
+  /\bi am not a robot\b/i,
+  /\bthis check is for\b/i,
+  /\bverify (?:you are|that you are) (?:a )?human\b/i,
+  /\bplease enable (?:javascript|cookies)\b/i,
+  /\bray id\b/i,
+  /\bddos protection by\b/i,
+  /\bperformance (?:&|and) security by\b/i
+];
+
+// Tokens that mark an affiliation line rather than a sentence about the topic.
+const AFFILIATION_TOKENS = new Set([
+  'university', 'universities', 'institute', 'institut', 'department', 'dept',
+  'faculty', 'college', 'school', 'laboratory', 'laboratoire', 'academy',
+  'inc', 'llc', 'ltd', 'gmbh', 'corp', 'corporation', 'foundation'
+]);
+
+// Closed-class finite verbs. A declarative claim contains one of these or an
+// inflected lexical verb (see hasFiniteVerb).
+const AUXILIARY_AND_MODAL_VERBS = new Set([
+  'is', 'are', 'was', 'were', 'am', 'be', 'been', 'being',
+  'has', 'have', 'had', 'do', 'does', 'did',
+  'can', 'could', 'will', 'would', 'shall', 'should', 'may', 'might', 'must'
+]);
+
+// Frequent base-form verbs. A plural subject takes an uninflected verb
+// ("Akamai and Imperva rely on device fingerprinting"), which the -s/-ed test
+// below cannot see.
+const COMMON_BASE_VERBS = new Set([
+  'use', 'rely', 'block', 'detect', 'deploy', 'prevent', 'provide', 'offer',
+  'include', 'require', 'allow', 'make', 'run', 'work', 'help', 'need', 'show',
+  'report', 'find', 'remain', 'become', 'appear', 'support', 'handle',
+  'protect', 'track', 'monitor', 'serve', 'apply', 'target', 'return', 'know',
+  'take', 'give', 'keep', 'add', 'build', 'create', 'enable', 'identify',
+  'reduce', 'increase', 'mean', 'occur', 'exist', 'vary', 'differ', 'depend',
+  'contain', 'combine', 'measure', 'generate'
+]);
+
+// Lowercase words that end in -s/-es/-ed but are not verbs; without these the
+// inflection test in hasFiniteVerb reads plural nouns as verbs.
+const NOT_VERBS = new Set([
+  'as', 'its', 'this', 'thus', 'less', 'gas', 'plus', 'versus', 'yes',
+  'always', 'perhaps', 'various', 'previous', 'obvious', 'serious', 'numerous',
+  'analysis', 'access', 'process', 'business', 'address', 'success', 'progress'
+]);
+
+// Marketing register. Present in a vendor's own copy, rare in a factual claim.
+const PROMOTIONAL_CUES = [
+  /\bbest\b/i, /\bleading\b/i, /\bindustry[- ]leading\b/i, /\bworld[- ]class\b/i,
+  /\bfastest\b/i, /\bmost (?:accurate|reliable|powerful|advanced)\b/i,
+  /\btrusted by\b/i, /\breliable\b/i, /\beffortless/i, /\bseamless/i,
+  /\bpowerful\b/i, /\bsolution\b/i, /\bunlimited\b/i, /\bguarantee/i,
+  /\bget started\b/i, /\bsign up\b/i, /\bfree trial\b/i, /\bno credit card\b/i,
+  /\bstart (?:scraping|crawling|building)\b/i, /\bcontact sales\b/i,
+  /\bpricing\b/i, /\bplans start\b/i, /\b99\.9\d*%\b/i,
+  /\bmost widely used\b/i, /\bmost popular\b/i
+];
+
+// A named commercial offering: a capitalized name bound to a product noun
+// ("Zyte API", "Scrapy Cloud", "Managed Data service"), or a third-person
+// possessive standing in for one ("their API"). This is the SUBJECT half of a
+// recommendation. Nothing here names a company — the test is grammatical.
+const NAMED_OFFERING_PATTERNS = [
+  /\b[A-Z][A-Za-z0-9.\-]+\s+(?:APIs?|SDKs?|[Pp]latform|[Pp]roduct|[Ss]ervice|[Ss]uite|[Tt]oolkit|[Cc]loud)\b/,
+  /\b(?:their|its|our)\s+(?:api|sdk|platform|product|service|suite|toolkit|cloud|tool)\b/i,
+  /\bflagship product\b/i
+];
+
+// The PREDICATE half: an offering credited with doing the work for you, or
+// ranked above its peers. Deliberately excludes plain capability verbs
+// ("detects", "blocks", "scores", "protects"), which is how factual sentences
+// about anti-bot systems read — those must survive to answer the question.
+const PITCH_PREDICATE_PATTERNS = [
+  /\bdesigned to\b/i, /\bbuilt to\b/i, /\bpurpose[- ]built\b/i,
+  /\boffers?\b/i, /\bprovides?\b/i, /\bdelivers?\b/i,
+  /\b(?:lets|helps|allows) you\b/i, /\bso you can\b/i,
+  /\bautomates?\b/i, /\bunblocks?\b/i, /\bhandles? (?:everything|it all|the (?:complexity|hard part))/i,
+  /\branked (?:#\s*1|number one|first)\b/i, /\ball[- ]in[- ]one\b/i,
+  /\bout of the box\b/i, /\bno code (?:required|needed)\b/i,
+  /\bstarts? at \$/i, /\bfree tier\b/i
+];
+
+/** Text that is document furniture rather than a claim about the topic. */
+export function isFrontMatter(text) {
+  return FRONT_MATTER_PATTERNS.some(pattern => pattern.test(text));
+}
+
+/**
+ * Does the text contain a finite verb?
+ *
+ * Deliberately permissive: a closed list of auxiliaries, modals and frequent
+ * base-form verbs, plus any lowercase token inflected as -s/-es/-ed. It
+ * rejects verbless fragments
+ * (author rows, keyword lists, DOI stubs), not bad grammar. Capitalized tokens
+ * are never counted, so "Jane Doe, University of Somewhere" has no verb while
+ * "Cloudflare uses TLS fingerprinting" does.
+ */
+export function hasFiniteVerb(text) {
+  const tokens = text.split(/[^A-Za-z'-]+/).filter(Boolean);
+
+  return tokens.some(token => {
+    const lower = token.toLowerCase();
+    if (AUXILIARY_AND_MODAL_VERBS.has(lower)) return true;
+    // Lexical verbs only in their lowercase form — a capitalized token here is
+    // a proper noun ("Systems Inc"), not a verb.
+    if (token !== lower || NOT_VERBS.has(lower)) return false;
+    return COMMON_BASE_VERBS.has(lower) || /^[a-z]{3,}(?:s|es|ed)$/.test(lower);
+  });
+}
+
+/**
+ * Share of tokens that look like proper nouns or affiliation words, ignoring
+ * the first token (a normal sentence starts capitalized).
+ */
+export function properNounDensity(text) {
+  const tokens = text.split(/[^A-Za-z'-]+/).filter(Boolean).slice(1);
+  if (tokens.length === 0) return 0;
+
+  const marked = tokens.filter(
+    token => /^[A-Z]/.test(token) || AFFILIATION_TOKENS.has(token.toLowerCase())
+  ).length;
+
+  return marked / tokens.length;
+}
+
+// Above this share of proper-noun/affiliation tokens the text reads as an
+// author or affiliation block rather than a sentence about the topic.
+const PROPER_NOUN_DENSITY_MAX = 0.5;
+
+/** A candidate claim is admissible when it is a verb-bearing, non-front-matter sentence. */
+export function isAdmissibleClaim(text) {
+  if (typeof text !== 'string' || text.trim().length === 0) return false;
+  if (isFrontMatter(text)) return false;
+  if (!hasFiniteVerb(text)) return false;
+  return properNounDensity(text) <= PROPER_NOUN_DENSITY_MAX;
+}
+
+/** Brand token of a URL's host: "https://scrape.do/blog" -> "scrape". */
+export function brandFromUrl(url) {
+  try {
+    const host = new URL(url).hostname.toLowerCase().replace(/^www\./, '');
+    const labels = host.split('.');
+    // Drop the public suffix; for "scrape.do" that leaves "scrape", for
+    // "api.datadome.co.uk" it leaves "datadome".
+    const brand = labels.length > 2 && labels[labels.length - 2].length <= 3
+      ? labels[labels.length - 3]
+      : labels[labels.length - 2];
+    return brand || null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Is this claim a vendor describing its own product in marketing register?
+ *
+ * Signal: the text refers to the site it came from (its brand token, or a
+ * first-person "we/our") AND uses at least one marketing cue. That is enough
+ * to stop "Scrape.do offers reliable scraping with effective anti-bot bypass"
+ * being laundered into a research conclusion.
+ *
+ * On its own it does not catch one vendor pitching another vendor's product —
+ * see looksLikeVendorPage + isProductPitch, which cover that case.
+ */
+export function isVendorSelfPromotion(text, sourceUrl) {
+  if (typeof text !== 'string' || text.length === 0) return false;
+
+  const brand = brandFromUrl(sourceUrl);
+  const selfReference =
+    (brand && brand.length >= 3 && new RegExp(`\\b${brand}\\b`, 'i').test(text)) ||
+    /\b(?:we|our|us)\b/i.test(text);
+
+  return Boolean(selfReference) && PROMOTIONAL_CUES.some(cue => cue.test(text));
+}
+
+/**
+ * Is this claim a recommendation rather than evidence?
+ *
+ * A recommendation has a shape: its subject is a named commercial offering and
+ * its predicate credits that offering with doing the work or ranking above its
+ * peers. "Their flagship product, Zyte API, is a web scraping API designed to
+ * unblock, render, and extract data from any website" has both halves; "CF-RAY
+ * in the response header means Cloudflare" and "Modern anti-bot systems
+ * fingerprint your TLS handshake" have neither, and both must survive — they
+ * are the answer to an anti-bot question.
+ *
+ * The test is grammatical and domain-independent: no company or product name
+ * appears anywhere in this module, and a page describing its own product and a
+ * page describing a peer's are treated identically.
+ *
+ * It does NOT catch a recommendation written without a product noun ("Scrapy is
+ * what most teams reach for"), and it will flag a factual sentence that
+ * describes a named product as doing work for you — including an anti-bot
+ * vendor's own product page. Both halves must match, which is what keeps
+ * ordinary claims about detection systems out of it.
+ */
+export function isProductRecommendation(text) {
+  if (typeof text !== 'string' || text.length === 0) return false;
+
+  return NAMED_OFFERING_PATTERNS.some(pattern => pattern.test(text)) &&
+    PITCH_PREDICATE_PATTERNS.some(pattern => pattern.test(text));
+}
+

package/src/schemas/toolOutputSchemas.js

@@ -246,6 +246,7 @@ const searchWebShape = {
 // ── extract_structured ───────────────────────────────────────────────────────
 
 const extractStructuredShape = {
+  success: z.boolean().optional().describe('False when the extraction errored or a required field came back missing or empty'),
   url: z.string().optional(),
   data: z.record(z.unknown()).optional().describe('Extracted fields matching the requested schema'),
   extraction_method: z.string().optional().describe('"llm" | "css_fallback" | "keyword_fallback" | "none"'),
@@ -290,7 +291,10 @@ const crawlDeepShape = {
   stats: z.unknown().optional(),
   site_structure: z.object({
     total_pages: z.number().optional(),
-    depth_distribution: z.record(z.number()).optional(),
+    depth_distribution: z.record(z.number()).optional()
+      .describe('Pages per crawl depth (links from the start URL)'),
+    path_depth_distribution: z.record(z.number()).optional()
+      .describe('Pages per URL path-segment depth'),
     path_patterns: z.record(z.number()).optional(),
     file_types: z.record(z.number()).optional(),
     subdomains: z.array(z.string()).optional()

package/src/security/wave3-security.js

@@ -9,6 +9,7 @@ import { createHash, randomBytes, timingSafeEqual } from 'crypto';
 import path from 'path';
 import { URL } from 'url';
 import DOMPurify from 'isomorphic-dompurify';
+import { identityHeaders } from '../utils/fetchIdentity.js';
 
 // Security configuration
 const SECURITY_CONFIG = {
@@ -125,7 +126,7 @@ export class SSRFProtection {
     const fetchOptions = {
       timeout: options.timeout || 30000,
       headers: {
-        'User-Agent': 'CrawlForge/3.0 (Security-Hardened)',
+        ...identityHeaders({ role: 'health-check' }),
         ...options.headers
       },
       ...options

package/src/server/requestContext.js

@@ -24,3 +24,26 @@ export const requestContext = new AsyncLocalStorage();
 export function isInternalRequest() {
   return requestContext.getStore()?.internal === true;
 }
+
+/**
+ * Record that the compliance gate refused this invocation before anything was
+ * fetched — robots.txt disallowed the path, or the host is on the permanent
+ * blocklist.
+ *
+ * withAuth reads this when it decides the charge. The flag rather than the
+ * error's `code` is deliberate: tool handlers catch their own errors and
+ * return `{ isError: true }` with only a message, so the typed error never
+ * reaches withAuth. It also survives both routes a refusal can take — thrown,
+ * or swallowed into an isError result.
+ *
+ * @param {string} code 'ROBOTS_DISALLOWED' | 'HOST_BLOCKED'
+ */
+export function markPreflightRefusal(code) {
+  const store = requestContext.getStore();
+  if (store) store.preflightRefusal = code;
+}
+
+/** The refusal code recorded for this invocation, or null. */
+export function preflightRefusal() {
+  return requestContext.getStore()?.preflightRefusal ?? null;
+}

package/src/server/withAuth.js

@@ -15,7 +15,7 @@
 
 import { createHash } from 'node:crypto';
 import { recordToolInvocation } from '../observability/tracing.js';
-import { isInternalRequest } from './requestContext.js';
+import { isInternalRequest, preflightRefusal, requestContext } from './requestContext.js';
 
 export function hashParams(params) {
   try {
@@ -33,7 +33,7 @@ export function hashParams(params) {
  */
 export function makeWithAuth({ authManager, logger, metrics = null }) {
   return function withAuth(toolName, handler) {
-    return async (params) => {
+    const invoke = async (params) => {
       const startTime = Date.now();
       const paramHash = hashParams(params);
       const creatorMode = authManager.isCreatorMode();
@@ -82,7 +82,14 @@ export function makeWithAuth({ authManager, logger, metrics = null }) {
         // it as an error, honoring CLAUDE.md's "half credits on error" contract.
         const isErrorResult = result?.isError === true;
         outcome = isErrorResult ? 'error' : 'success';
-        const charge = creditCost === 0
+        // A pre-flight refusal (robots.txt disallowed the path, or the host is
+        // blocklisted) means we fetched nothing at all, so it costs nothing —
+        // not even the half-credit error rate, which exists for work that ran
+        // and then failed. Only when the refusal actually sank the call: a
+        // multi-URL tool that skipped one disallowed URL and still returned a
+        // result did real work and bills for it.
+        const refused = isErrorResult && preflightRefusal() !== null;
+        const charge = creditCost === 0 || refused
           ? 0
           : (isErrorResult ? Math.max(1, Math.floor(creditCost * 0.5)) : creditCost);
 
@@ -121,7 +128,7 @@ export function makeWithAuth({ authManager, logger, metrics = null }) {
         // DataForSEO is unconfigured — a no-op). Emit NO usage event at all so
         // the backend has nothing to (re-)price; reporting 0 would still create
         // a serp_rank record the backend could recompute to full cost.
-        if (!creatorMode && creditCost > 0) {
+        if (!creatorMode && creditCost > 0 && !refused) {
           await authManager.reportUsage(toolName, charge, params, isErrorResult ? 500 : 200, Date.now() - startTime);
         }
 
@@ -132,7 +139,7 @@ export function makeWithAuth({ authManager, logger, metrics = null }) {
         // Half-charge on error — but never charge a free (0-cost) call, never
         // let Math.max(1, …) floor a 0 up to 1 credit, and never bill at all
         // if the handler never ran (e.g. the credit check itself threw).
-        if (!creatorMode && creditCost > 0 && handlerStarted) {
+        if (!creatorMode && creditCost > 0 && handlerStarted && preflightRefusal() === null) {
           await authManager.reportUsage(
             toolName,
             Math.max(1, Math.floor(creditCost * 0.5)),
@@ -183,5 +190,14 @@ export function makeWithAuth({ authManager, logger, metrics = null }) {
         }, thrown);
       }
     };
+
+    // Every invocation runs in its own context so the compliance gate can stamp
+    // a refusal where the billing decision can see it. Any outer store (the
+    // HTTP transport's `internal` flag) is spread in, not replaced — and stdio
+    // callers, who have no transport-provided store, get one here.
+    return async (params) => requestContext.run(
+      { ...(requestContext.getStore() ?? {}), preflightRefusal: null },
+      () => invoke(params)
+    );
   };
 }

package/src/skills/agent-skills/crawlforge-deep-research/SKILL.md

@@ -30,7 +30,7 @@ skill (`scrape` / `extract_content`) instead.
 | Need | Tool | Cost |
 |------|------|------|
 | A ranked list of result URLs + snippets | `search_web` | 5 |
-| Reddit posts, comments, or a full thread | `reddit_search` | 2 |
+| Reddit posts, comments, or a full thread | `reddit_search` | 5 |
 | A direct answer, agent decides what to read | `agent` | 8 (scales) |
 | A synthesized, multi-source, cited report | `deep_research` | 10+ (scales) |
 

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -83,6 +83,33 @@ const ScrollActionSchema = BaseActionSchema.extend({
   y: z.number().min(0).optional()
 });
 
+const SelectActionSchema = BaseActionSchema.extend({
+  type: z.literal('select'),
+  selector: z.string(),
+  // A plain string matches an <option> by value OR label — see ActionExecutor's
+  // SelectActionSchema.
+  value: z.string().optional(),
+  values: z.array(z.string()).optional()
+}).refine(data => data.value !== undefined || (data.values && data.values.length > 0), {
+  message: 'Select action requires value or values'
+});
+
+const HoverActionSchema = BaseActionSchema.extend({
+  type: z.literal('hover'),
+  selector: z.string(),
+  force: z.boolean().default(false),
+  position: z.object({
+    x: z.number(),
+    y: z.number()
+  }).optional()
+});
+
+const NavigateActionSchema = BaseActionSchema.extend({
+  type: z.literal('navigate'),
+  url: z.string().url(),
+  waitUntil: z.enum(['load', 'domcontentloaded', 'networkidle', 'commit']).optional()
+});
+
 const ScreenshotActionSchema = BaseActionSchema.extend({
   type: z.literal('screenshot'),
   selector: z.string().optional(),
@@ -104,6 +131,9 @@ const ActionSchema = z.union([
   TypeActionSchema,
   PressActionSchema,
   ScrollActionSchema,
+  SelectActionSchema,
+  HoverActionSchema,
+  NavigateActionSchema,
   ScreenshotActionSchema,
   ExecuteJavaScriptActionSchema
 ]);
@@ -145,7 +175,11 @@ const ScrapeWithActionsSchema = z.object({
     userAgent: z.string().optional(),
     viewportWidth: z.number().min(800).max(1920).default(1280),
     viewportHeight: z.number().min(600).max(1080).default(720),
-    timeout: z.number().min(10000).max(120000).default(30000)
+    timeout: z.number().min(10000).max(120000).default(30000),
+    // Run the chain in the stealth browser (StealthBrowserManager) instead of
+    // the standard pool. executeSession turns this into the stealthMode object
+    // ActionExecutor/BrowserProcessor read.
+    stealth: z.boolean().default(false)
   }).optional(),
 
   // Content extraction options
@@ -156,6 +190,10 @@ const ScrapeWithActionsSchema = z.object({
     includeImages: z.boolean().default(true)
   }).optional(),
 
+  // Per-request robots.txt override (G5). Undefined leaves the configured
+  // default in force; false is honoured, warned about and audited by the gate.
+  respect_robots: z.boolean().optional(),
+
   // Error handling
   continueOnActionError: z.boolean().default(false),
   maxRetries: z.number().min(0).max(3).default(1),
@@ -341,6 +379,16 @@ export class ScrapeWithActionsTool extends EventEmitter {
       ...params.browserOptions
     };
 
+    // `stealth: true` is the caller-facing switch; everything downstream reads
+    // browserOptions.stealthMode.
+    if (browserOptions.stealth) {
+      browserOptions.stealthMode = { enabled: true };
+    }
+
+    // The robots override rides with the browser options so ActionExecutor's
+    // gate sees it on the initial load and on every `navigate` action.
+    browserOptions.respectRobots = params.respect_robots;
+
     // Build action chain with form auto-fill if provided
     let actionChain = [...params.actions];
 

package/src/tools/advanced/batchScrape/schema.js

@@ -23,6 +23,10 @@ export const BatchScrapeSchema = z.object({
     signingSecret: z.string().optional()
   }).optional(),
   extractionSchema: z.record(z.string()).optional(),
+  // Compliance overrides, per request: identify as yourself for a target you
+  // have your own agreement with, and take responsibility for ignoring robots.
+  user_agent: z.string().optional(),
+  respect_robots: z.boolean().optional(),
   maxConcurrency: z.number().min(1).max(20).default(10),
   delayBetweenRequests: z.number().min(0).max(10000).default(100),
   includeMetadata: z.boolean().default(true),

package/src/tools/advanced/batchScrape/worker.js

@@ -7,33 +7,37 @@
 import { load } from 'cheerio';
 import { config as appConfig } from '../../../constants/config.js';
 import { ssrfGuard, isSsrfError } from '../../../utils/ssrfGuard.js';
-import { throttleHost } from '../../../utils/hostRateLimiter.js';
+import { noteRetryAfter } from '../../../utils/hostRateLimiter.js';
+import { preflightFetch } from '../../../utils/robotsGate.js';
 import { htmlToMarkdown } from '../../../utils/htmlToMarkdown.js';
 
-const USER_AGENT = 'MCP-WebScraper-BatchTool/1.0.0';
-
 /**
  * Fetch a URL with AbortController timeout (SSRF-guarded + per-host throttled).
  * The timeout stays live through the body read (not just until headers
  * arrive), and the body is size-capped, so a server that sends headers then
  * drips the body can't hold a semaphore slot indefinitely or exhaust memory.
- * Returns { response, html } rather than the raw Response.
+ * Returns { response, html, warnings } rather than the raw Response.
  */
 export async function fetchUrl(url, options = {}) {
-  const { timeout = 15000, headers = {} } = options;
+  const { timeout = 15000, headers = {}, userAgent, respectRobots, apiKey } = options;
   const maxBodySize = appConfig.fetch.maxBodySize;
   const guard = ssrfGuard(url); // SSRF pre-flight (throws before connecting)
-  await throttleHost(url);
+  // robots.txt / blocklist gate + per-host throttle. Throws if it refuses,
+  // which scrapeUrl turns into a failure for this URL alone.
+  const gate = await preflightFetch(url, { userAgent, respectRobots, apiKey, tool: 'batch_scrape' });
   const controller = new AbortController();
   const timeoutId = setTimeout(() => controller.abort(), timeout);
   try {
     const response = await fetch(url, {
       signal: controller.signal,
-      headers: { 'User-Agent': USER_AGENT, ...headers },
+      headers: { ...gate.headers, ...headers },
       ...guard
     });
+    if (response.status === 429 || response.status === 503) {
+      noteRetryAfter(url, response.headers?.get?.('retry-after'));
+    }
     const html = await readBodyCapped(response, maxBodySize);
-    return { response, html };
+    return { response, html, warnings: gate.warnings };
   } catch (error) {
     if (isSsrfError(error)) throw new Error(error.cause?.message || error.message);
     if (error.name === 'AbortError') throw new Error(`Request timeout after ${timeout}ms`);
@@ -93,9 +97,12 @@ async function readBodyCapped(response, maxBodySize) {
 export async function scrapeUrl(config, options, defaultTimeout) {
   const startTime = Date.now();
   try {
-    const { response, html } = await fetchUrl(config.url, {
+    const { response, html, warnings } = await fetchUrl(config.url, {
       headers: config.headers,
-      timeout: config.timeout || defaultTimeout
+      timeout: config.timeout || defaultTimeout,
+      userAgent: options.user_agent,
+      respectRobots: options.respect_robots,
+      apiKey: options.apiKey
     });
 
     if (!response.ok) throw new Error(`HTTP ${response.status}: ${response.statusText}`);
@@ -115,6 +122,8 @@ export async function scrapeUrl(config, options, defaultTimeout) {
       }
     };
 
+    if (warnings.length > 0) result.warnings = warnings;
+
     if (options.extractionSchema || config.selectors) {
       result.extracted = extractStructuredData($, { ...config.selectors, ...options.extractionSchema });
     }

package/src/tools/basic/_fetch.js

@@ -1,18 +1,13 @@
 /**
  * Shared HTTP fetch helper for basic tools.
- * Applies an AbortController timeout and a default User-Agent.
+ * Applies an AbortController timeout and the shared pre-fetch gate.
  */
 
 import { readBody } from 'crawlforge-extractors';
 import { config } from '../../constants/config.js';
-import { createRequire } from 'module';
 import { ssrfGuard, isSsrfError } from '../../utils/ssrfGuard.js';
-import { throttleHost } from '../../utils/hostRateLimiter.js';
-
-// Derive User-Agent from package version so it reflects the actual release.
-const _require = createRequire(import.meta.url);
-const _pkg = _require('../../../package.json');
-const CRAWLFORGE_UA = `CrawlForge/${_pkg.version} (+https://crawlforge.dev)`;
+import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
+import { preflightFetch } from '../../utils/robotsGate.js';
 
 /**
  * Fetch a URL with a configurable timeout and body-size cap.
@@ -23,19 +18,21 @@ const CRAWLFORGE_UA = `CrawlForge/${_pkg.version} (+https://crawlforge.dev)`;
  * default 25 MB).
  *
  * @param {string} url
- * @param {{ timeout?: number, headers?: Record<string,string> }} [options]
- * @returns {Promise<Response & { _body: string }>}
+ * @param {{ timeout?: number, headers?: Record<string,string>, userAgent?: string,
+ *   respectRobots?: boolean, tool?: string, apiKey?: string }} [options]
+ * @returns {Promise<Response & { _body: string, _warnings: string[] }>}
  */
 export async function fetchWithTimeout(url, options = {}) {
-  const { timeout = 10000, headers = {} } = options;
+  const { timeout = 10000, headers = {}, userAgent, respectRobots, tool, apiKey } = options;
   const maxBodySize = config.fetch.maxBodySize;
 
   // SSRF pre-flight (protocol / metadata host). Throws a clear error before any
   // connection is attempted; `guard.dispatcher` enforces IP rules at connect time.
   const guard = ssrfGuard(url);
 
-  // Per-host politeness throttle (before the timeout window starts).
-  await throttleHost(url);
+  // robots.txt / blocklist gate + per-host politeness throttle, before the
+  // timeout window starts. Throws if the gate refuses the URL.
+  const gate = await preflightFetch(url, { userAgent, respectRobots, tool, apiKey });
 
   const controller = new AbortController();
   const timeoutId = setTimeout(() => controller.abort(), timeout);
@@ -57,7 +54,7 @@ export async function fetchWithTimeout(url, options = {}) {
       response = await fetch(url, {
         signal: controller.signal,
         headers: {
-          'User-Agent': CRAWLFORGE_UA,
+          ...gate.headers,
           ...headers
         },
         ...guard
@@ -72,6 +69,12 @@ export async function fetchWithTimeout(url, options = {}) {
       throw error;
     }
 
+    // A host asking us to back off is honoured on the *next* request to it,
+    // rather than retrying straight into the wall.
+    if (response.status === 429 || response.status === 503) {
+      noteRetryAfter(url, response.headers?.get?.('retry-after'));
+    }
+
     // Reading is delegated to crawlforge-extractors so the REST API applies
     // the same cap and the same charset handling to the same page.
     let bodyText;
@@ -89,7 +92,8 @@ export async function fetchWithTimeout(url, options = {}) {
       text: () => Promise.resolve(bodyText),
       json: () => Promise.resolve(JSON.parse(bodyText)),
       _body: bodyText,
-      _responseTime: Date.now() - startedAt
+      _responseTime: Date.now() - startedAt,
+      _warnings: gate.warnings
     });
   } finally {
     clearTimeout(timeoutId);

package/src/tools/basic/extractLinks.js

@@ -7,11 +7,16 @@ import { load } from 'cheerio';
 import { fetchWithTimeout } from './_fetch.js';
 
 /**
- * @param {{ url: string, filter_external?: boolean, base_url?: string }} params
+ * @param {{ url: string, filter_external?: boolean, base_url?: string,
+ *   user_agent?: string, respect_robots?: boolean }} params
  */
-export async function extractLinksHandler({ url, filter_external, base_url }) {
+export async function extractLinksHandler({ url, filter_external, base_url, user_agent, respect_robots }) {
   try {
-    const response = await fetchWithTimeout(url);
+    const response = await fetchWithTimeout(url, {
+      userAgent: user_agent,
+      respectRobots: respect_robots,
+      tool: 'extract_links'
+    });
     if (!response.ok) {
       throw new Error(`HTTP ${response.status}: ${response.statusText}`);
     }

package/src/tools/basic/extractMetadata.js

@@ -60,11 +60,15 @@ function parseMicrodata($) {
 }
 
 /**
- * @param {{ url: string }} params
+ * @param {{ url: string, user_agent?: string, respect_robots?: boolean }} params
  */
-export async function extractMetadataHandler({ url }) {
+export async function extractMetadataHandler({ url, user_agent, respect_robots }) {
   try {
-    const response = await fetchWithTimeout(url);
+    const response = await fetchWithTimeout(url, {
+      userAgent: user_agent,
+      respectRobots: respect_robots,
+      tool: 'extract_metadata'
+    });
     if (!response.ok) {
       throw new Error(`HTTP ${response.status}: ${response.statusText}`);
     }