crawlforge-mcp-server 5.2.9 → 5.3.1

package/src/core/llm/OllamaProvider.js

@@ -14,28 +14,37 @@ export class OllamaProvider extends LLMProvider {
     super(options);
 
     // Resolved lazily: choosing the best installed model needs an HTTP call.
+    // An explicit model applies to every role; otherwise each role resolves
+    // (and caches) its own choice.
     this.model = options.model || null;
+    this.modelByRole = new Map();
     this.embeddingModel = options.embeddingModel || process.env.OLLAMA_EMBEDDING_MODEL || null;
     this.timeout = options.timeout || 120000;
   }
 
-  /** The model to use, selecting the best installed one on first use. */
-  async resolveModel() {
-    if (!this.model) this.model = await selectOllamaModel();
-    return this.model;
+  /**
+   * The model to use, selecting the best installed one for the role on first
+   * use. Extraction and judgement have different winners — see JUDGEMENT_MODELS.
+   * @param {'default'|'judgement'} [role]
+   */
+  async resolveModel(role = 'default') {
+    if (this.model) return this.model;
+    if (!this.modelByRole.has(role)) this.modelByRole.set(role, await selectOllamaModel(role));
+    return this.modelByRole.get(role);
   }
 
   async generateCompletion(prompt, options = {}) {
-    const model = await this.resolveModel();
     const {
       maxTokens = 1000,
       temperature = 0.7,
       systemPrompt = null,
+      role = 'default',
       // 'json' constrains the model to emit a parseable object, or pass a JSON
       // Schema to constrain the shape as well. Small local models otherwise
       // wrap JSON in prose and the caller's JSON.parse fails.
       format = null
     } = options;
+    const model = await this.resolveModel(role);
 
     const messages = [];
     if (systemPrompt) {

package/src/core/processing/BrowserProcessor.js

@@ -385,6 +385,33 @@ export class BrowserProcessor {
     return page;
   }
 
+  /**
+   * Release a stealth page and hand its context slot back to the pool.
+   *
+   * createStealthPage() registers every context in activeContexts and in
+   * StealthBrowserManager's capped pool (MAX_BROWSER_CONTEXTS, default 10).
+   * Nothing reuses a context after the call that created it, so a caller that
+   * only closes the page keeps the slot: after 10 stealth runs the next
+   * createStealthContext() waits for a free slot and then throws. Going
+   * through the manager's closeContext() returns the slot and the renderer.
+   * @param {Page} page - Stealth page returned by initializePage()
+   * @returns {Promise<void>}
+   */
+  async releaseStealthPage(page) {
+    try { await page.close(); } catch (_) { /* ignore close errors */ }
+
+    for (const [contextId, contextData] of this.activeContexts.entries()) {
+      if (contextData.page !== page) continue;
+      this.activeContexts.delete(contextId);
+      try {
+        await this.stealthManager?.closeContext(contextId);
+      } catch (error) {
+        console.warn(`Failed to close stealth context ${contextId}:`, error.message);
+      }
+      return;
+    }
+  }
+
   /**
    * Apply additional stealth middleware to page
    * @param {Page} page - Playwright page

package/src/core/processing/ContentProcessor.js

@@ -7,6 +7,7 @@ import { Readability } from '@mozilla/readability';
 import { JSDOM } from 'jsdom';
 import * as cheerio from 'cheerio';
 import { z } from 'zod';
+import { ContentQualityAssessor } from '../../utils/contentUtils.js';
 
 const ContentProcessorSchema = z.object({
   html: z.string(),
@@ -381,7 +382,11 @@ export class ContentProcessor {
   }
 
   /**
-   * Calculate readability score using simple metrics
+   * Calculate readability metrics. The Flesch score, level and syllable
+   * counting come from ContentQualityAssessor.calculateSimpleReadability —
+   * the single Flesch implementation — so this never disagrees with a
+   * qualityAssessment computed over the same text. Score is unclamped; see
+   * that method for why.
    * @param {string} text - Text content
    * @returns {Object} - Readability metrics
    */
@@ -399,55 +404,22 @@ export class ContentProcessor {
       return null;
     }
 
-    const avgWordsPerSentence = words.length / sentences.length;
+    const readability = ContentQualityAssessor.calculateSimpleReadability(text);
     const avgCharsPerWord = charactersNoSpaces / words.length;
-    const avgSyllablesPerWord = words.reduce((sum, w) => sum + this._countSyllables(w), 0) / words.length;
-
-    // Flesch Reading-Ease: higher score = easier to read
-    const readabilityScore = 206.835 - (1.015 * avgWordsPerSentence) - (84.6 * avgSyllablesPerWord);
 
     return {
       sentences: sentences.length,
       words: words.length,
       characters,
       charactersNoSpaces,
-      avgWordsPerSentence: Math.round(avgWordsPerSentence * 100) / 100,
+      avgWordsPerSentence: readability.avgWordsPerSentence,
       avgCharsPerWord: Math.round(avgCharsPerWord * 100) / 100,
-      avgSyllablesPerWord: Math.round(avgSyllablesPerWord * 100) / 100,
-      readabilityScore: Math.round(readabilityScore * 100) / 100,
-      readabilityLevel: this.getReadabilityLevel(readabilityScore)
+      avgSyllablesPerWord: readability.avgSyllablesPerWord,
+      readabilityScore: readability.score,
+      readabilityLevel: readability.level
     };
   }
 
-  /**
-   * Get readability level based on score
-   * @param {number} score - Readability score
-   * @returns {string} - Readability level
-   */
-  getReadabilityLevel(score) {
-    if (score >= 90) return 'Very Easy';
-    if (score >= 80) return 'Easy';
-    if (score >= 70) return 'Fairly Easy';
-    if (score >= 60) return 'Standard';
-    if (score >= 50) return 'Fairly Difficult';
-    if (score >= 30) return 'Difficult';
-    return 'Very Difficult';
-  }
-
-  /**
-   * Count syllables in a word (heuristic)
-   * @param {string} word
-   * @returns {number}
-   */
-  _countSyllables(word) {
-    const w = word.toLowerCase().replace(/[^a-z]/g, '');
-    if (w.length <= 3) return 1;
-    // Remove trailing silent e
-    const stripped = w.replace(/e$/, '');
-    const matches = stripped.match(/[aeiouy]+/g);
-    return Math.max(1, matches ? matches.length : 1);
-  }
-
   /**
    * Extract fallback content when Readability fails
    * @param {string} html - HTML content

package/src/core/processing/PDFProcessor.js

@@ -9,6 +9,7 @@ import fs from 'fs/promises';
 import path from 'path';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 import { config } from '../../constants/config.js';
+import { identityHeaders } from '../../utils/fetchIdentity.js';
 
 const PDFProcessorSchema = z.object({
   source: z.string().min(1),
@@ -237,9 +238,7 @@ export class PDFProcessor {
       // ignores unknown properties, so only `signal` actually enforces a
       // deadline here.
       const response = await safeFetch(url, {
-        headers: {
-          'User-Agent': 'Mozilla/5.0 (compatible; MCP-WebScraper/2.0; PDF-Processor)'
-        },
+        headers: identityHeaders(),
         signal: AbortSignal.timeout(30000)
       });
 

package/src/core/research/claimFilters.js

@@ -0,0 +1,235 @@
+/**
+ * Claim admission filters for deep_research synthesis.
+ *
+ * Claims are extractive sentences pulled out of source pages by the summarize
+ * tool, so whatever the page starts with is a candidate claim. On a live run
+ * (2026-08-28, "What anti-bot systems do major websites use in 2026") the top
+ * finding was an arXiv front-matter block beginning "DOI: XXXXXXX.XXXXXXX" —
+ * document furniture, not a research claim. These predicates reject that class
+ * of text before it can become a key finding.
+ */
+
+// Document furniture: DOI stubs, retrieval notes, ACM/arXiv front matter,
+// copyright rows, contact emails. Matched anywhere in the candidate claim.
+const FRONT_MATTER_PATTERNS = [
+  /\bdoi\s*:/i,
+  /\bdoi\.org\//i,
+  /\barxiv\s*:\s*\d/i,
+  /\bccs concepts?\b/i,
+  /\bacm reference format\b/i,
+  /\bretrieved (?:from|on)\b/i,
+  /\bissn\b|\bisbn\b/i,
+  /\ball rights reserved\b/i,
+  /\bcopyright\s*(?:©|\(c\)|\d{4})/i,
+  /[\w.+-]+@[\w-]+\.[a-z]{2,}/i,
+  // "Permission to make digital or hard copies…" — the ACM licence block.
+  /\bpermission to make digital\b/i,
+  // Bot-challenge interstitials. A research run on anti-bot systems scores
+  // these as highly relevant — they are literally about bot detection — so the
+  // relevance gate cannot catch them (live 2026-08-28: "Checking your browser.
+  // This only takes a moment." was the top finding). Phrases are specific to
+  // the challenge chrome; a claim that merely discusses CAPTCHAs is untouched.
+  /\bchecking your browser\b/i,
+  /\bjust a moment\b/i,
+  /\bi am not a robot\b/i,
+  /\bthis check is for\b/i,
+  /\bverify (?:you are|that you are) (?:a )?human\b/i,
+  /\bplease enable (?:javascript|cookies)\b/i,
+  /\bray id\b/i,
+  /\bddos protection by\b/i,
+  /\bperformance (?:&|and) security by\b/i
+];
+
+// Tokens that mark an affiliation line rather than a sentence about the topic.
+const AFFILIATION_TOKENS = new Set([
+  'university', 'universities', 'institute', 'institut', 'department', 'dept',
+  'faculty', 'college', 'school', 'laboratory', 'laboratoire', 'academy',
+  'inc', 'llc', 'ltd', 'gmbh', 'corp', 'corporation', 'foundation'
+]);
+
+// Closed-class finite verbs. A declarative claim contains one of these or an
+// inflected lexical verb (see hasFiniteVerb).
+const AUXILIARY_AND_MODAL_VERBS = new Set([
+  'is', 'are', 'was', 'were', 'am', 'be', 'been', 'being',
+  'has', 'have', 'had', 'do', 'does', 'did',
+  'can', 'could', 'will', 'would', 'shall', 'should', 'may', 'might', 'must'
+]);
+
+// Frequent base-form verbs. A plural subject takes an uninflected verb
+// ("Akamai and Imperva rely on device fingerprinting"), which the -s/-ed test
+// below cannot see.
+const COMMON_BASE_VERBS = new Set([
+  'use', 'rely', 'block', 'detect', 'deploy', 'prevent', 'provide', 'offer',
+  'include', 'require', 'allow', 'make', 'run', 'work', 'help', 'need', 'show',
+  'report', 'find', 'remain', 'become', 'appear', 'support', 'handle',
+  'protect', 'track', 'monitor', 'serve', 'apply', 'target', 'return', 'know',
+  'take', 'give', 'keep', 'add', 'build', 'create', 'enable', 'identify',
+  'reduce', 'increase', 'mean', 'occur', 'exist', 'vary', 'differ', 'depend',
+  'contain', 'combine', 'measure', 'generate'
+]);
+
+// Lowercase words that end in -s/-es/-ed but are not verbs; without these the
+// inflection test in hasFiniteVerb reads plural nouns as verbs.
+const NOT_VERBS = new Set([
+  'as', 'its', 'this', 'thus', 'less', 'gas', 'plus', 'versus', 'yes',
+  'always', 'perhaps', 'various', 'previous', 'obvious', 'serious', 'numerous',
+  'analysis', 'access', 'process', 'business', 'address', 'success', 'progress'
+]);
+
+// Marketing register. Present in a vendor's own copy, rare in a factual claim.
+const PROMOTIONAL_CUES = [
+  /\bbest\b/i, /\bleading\b/i, /\bindustry[- ]leading\b/i, /\bworld[- ]class\b/i,
+  /\bfastest\b/i, /\bmost (?:accurate|reliable|powerful|advanced)\b/i,
+  /\btrusted by\b/i, /\breliable\b/i, /\beffortless/i, /\bseamless/i,
+  /\bpowerful\b/i, /\bsolution\b/i, /\bunlimited\b/i, /\bguarantee/i,
+  /\bget started\b/i, /\bsign up\b/i, /\bfree trial\b/i, /\bno credit card\b/i,
+  /\bstart (?:scraping|crawling|building)\b/i, /\bcontact sales\b/i,
+  /\bpricing\b/i, /\bplans start\b/i, /\b99\.9\d*%\b/i,
+  /\bmost widely used\b/i, /\bmost popular\b/i
+];
+
+// A named commercial offering: a capitalized name bound to a product noun
+// ("Zyte API", "Scrapy Cloud", "Managed Data service"), or a third-person
+// possessive standing in for one ("their API"). This is the SUBJECT half of a
+// recommendation. Nothing here names a company — the test is grammatical.
+const NAMED_OFFERING_PATTERNS = [
+  /\b[A-Z][A-Za-z0-9.\-]+\s+(?:APIs?|SDKs?|[Pp]latform|[Pp]roduct|[Ss]ervice|[Ss]uite|[Tt]oolkit|[Cc]loud)\b/,
+  /\b(?:their|its|our)\s+(?:api|sdk|platform|product|service|suite|toolkit|cloud|tool)\b/i,
+  /\bflagship product\b/i
+];
+
+// The PREDICATE half: an offering credited with doing the work for you, or
+// ranked above its peers. Deliberately excludes plain capability verbs
+// ("detects", "blocks", "scores", "protects"), which is how factual sentences
+// about anti-bot systems read — those must survive to answer the question.
+const PITCH_PREDICATE_PATTERNS = [
+  /\bdesigned to\b/i, /\bbuilt to\b/i, /\bpurpose[- ]built\b/i,
+  /\boffers?\b/i, /\bprovides?\b/i, /\bdelivers?\b/i,
+  /\b(?:lets|helps|allows) you\b/i, /\bso you can\b/i,
+  /\bautomates?\b/i, /\bunblocks?\b/i, /\bhandles? (?:everything|it all|the (?:complexity|hard part))/i,
+  /\branked (?:#\s*1|number one|first)\b/i, /\ball[- ]in[- ]one\b/i,
+  /\bout of the box\b/i, /\bno code (?:required|needed)\b/i,
+  /\bstarts? at \$/i, /\bfree tier\b/i
+];
+
+/** Text that is document furniture rather than a claim about the topic. */
+export function isFrontMatter(text) {
+  return FRONT_MATTER_PATTERNS.some(pattern => pattern.test(text));
+}
+
+/**
+ * Does the text contain a finite verb?
+ *
+ * Deliberately permissive: a closed list of auxiliaries, modals and frequent
+ * base-form verbs, plus any lowercase token inflected as -s/-es/-ed. It
+ * rejects verbless fragments
+ * (author rows, keyword lists, DOI stubs), not bad grammar. Capitalized tokens
+ * are never counted, so "Jane Doe, University of Somewhere" has no verb while
+ * "Cloudflare uses TLS fingerprinting" does.
+ */
+export function hasFiniteVerb(text) {
+  const tokens = text.split(/[^A-Za-z'-]+/).filter(Boolean);
+
+  return tokens.some(token => {
+    const lower = token.toLowerCase();
+    if (AUXILIARY_AND_MODAL_VERBS.has(lower)) return true;
+    // Lexical verbs only in their lowercase form — a capitalized token here is
+    // a proper noun ("Systems Inc"), not a verb.
+    if (token !== lower || NOT_VERBS.has(lower)) return false;
+    return COMMON_BASE_VERBS.has(lower) || /^[a-z]{3,}(?:s|es|ed)$/.test(lower);
+  });
+}
+
+/**
+ * Share of tokens that look like proper nouns or affiliation words, ignoring
+ * the first token (a normal sentence starts capitalized).
+ */
+export function properNounDensity(text) {
+  const tokens = text.split(/[^A-Za-z'-]+/).filter(Boolean).slice(1);
+  if (tokens.length === 0) return 0;
+
+  const marked = tokens.filter(
+    token => /^[A-Z]/.test(token) || AFFILIATION_TOKENS.has(token.toLowerCase())
+  ).length;
+
+  return marked / tokens.length;
+}
+
+// Above this share of proper-noun/affiliation tokens the text reads as an
+// author or affiliation block rather than a sentence about the topic.
+const PROPER_NOUN_DENSITY_MAX = 0.5;
+
+/** A candidate claim is admissible when it is a verb-bearing, non-front-matter sentence. */
+export function isAdmissibleClaim(text) {
+  if (typeof text !== 'string' || text.trim().length === 0) return false;
+  if (isFrontMatter(text)) return false;
+  if (!hasFiniteVerb(text)) return false;
+  return properNounDensity(text) <= PROPER_NOUN_DENSITY_MAX;
+}
+
+/** Brand token of a URL's host: "https://scrape.do/blog" -> "scrape". */
+export function brandFromUrl(url) {
+  try {
+    const host = new URL(url).hostname.toLowerCase().replace(/^www\./, '');
+    const labels = host.split('.');
+    // Drop the public suffix; for "scrape.do" that leaves "scrape", for
+    // "api.datadome.co.uk" it leaves "datadome".
+    const brand = labels.length > 2 && labels[labels.length - 2].length <= 3
+      ? labels[labels.length - 3]
+      : labels[labels.length - 2];
+    return brand || null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Is this claim a vendor describing its own product in marketing register?
+ *
+ * Signal: the text refers to the site it came from (its brand token, or a
+ * first-person "we/our") AND uses at least one marketing cue. That is enough
+ * to stop "Scrape.do offers reliable scraping with effective anti-bot bypass"
+ * being laundered into a research conclusion.
+ *
+ * On its own it does not catch one vendor pitching another vendor's product —
+ * see looksLikeVendorPage + isProductPitch, which cover that case.
+ */
+export function isVendorSelfPromotion(text, sourceUrl) {
+  if (typeof text !== 'string' || text.length === 0) return false;
+
+  const brand = brandFromUrl(sourceUrl);
+  const selfReference =
+    (brand && brand.length >= 3 && new RegExp(`\\b${brand}\\b`, 'i').test(text)) ||
+    /\b(?:we|our|us)\b/i.test(text);
+
+  return Boolean(selfReference) && PROMOTIONAL_CUES.some(cue => cue.test(text));
+}
+
+/**
+ * Is this claim a recommendation rather than evidence?
+ *
+ * A recommendation has a shape: its subject is a named commercial offering and
+ * its predicate credits that offering with doing the work or ranking above its
+ * peers. "Their flagship product, Zyte API, is a web scraping API designed to
+ * unblock, render, and extract data from any website" has both halves; "CF-RAY
+ * in the response header means Cloudflare" and "Modern anti-bot systems
+ * fingerprint your TLS handshake" have neither, and both must survive — they
+ * are the answer to an anti-bot question.
+ *
+ * The test is grammatical and domain-independent: no company or product name
+ * appears anywhere in this module, and a page describing its own product and a
+ * page describing a peer's are treated identically.
+ *
+ * It does NOT catch a recommendation written without a product noun ("Scrapy is
+ * what most teams reach for"), and it will flag a factual sentence that
+ * describes a named product as doing work for you — including an anti-bot
+ * vendor's own product page. Both halves must match, which is what keeps
+ * ordinary claims about detection systems out of it.
+ */
+export function isProductRecommendation(text) {
+  if (typeof text !== 'string' || text.length === 0) return false;
+
+  return NAMED_OFFERING_PATTERNS.some(pattern => pattern.test(text)) &&
+    PITCH_PREDICATE_PATTERNS.some(pattern => pattern.test(text));
+}
+

package/src/schemas/toolOutputSchemas.js

@@ -246,6 +246,7 @@ const searchWebShape = {
 // ── extract_structured ───────────────────────────────────────────────────────
 
 const extractStructuredShape = {
+  success: z.boolean().optional().describe('False when the extraction errored or a required field came back missing or empty'),
   url: z.string().optional(),
   data: z.record(z.unknown()).optional().describe('Extracted fields matching the requested schema'),
   extraction_method: z.string().optional().describe('"llm" | "css_fallback" | "keyword_fallback" | "none"'),
@@ -290,7 +291,10 @@ const crawlDeepShape = {
   stats: z.unknown().optional(),
   site_structure: z.object({
     total_pages: z.number().optional(),
-    depth_distribution: z.record(z.number()).optional(),
+    depth_distribution: z.record(z.number()).optional()
+      .describe('Pages per crawl depth (links from the start URL)'),
+    path_depth_distribution: z.record(z.number()).optional()
+      .describe('Pages per URL path-segment depth'),
     path_patterns: z.record(z.number()).optional(),
     file_types: z.record(z.number()).optional(),
     subdomains: z.array(z.string()).optional()

package/src/security/wave3-security.js

@@ -9,6 +9,7 @@ import { createHash, randomBytes, timingSafeEqual } from 'crypto';
 import path from 'path';
 import { URL } from 'url';
 import DOMPurify from 'isomorphic-dompurify';
+import { identityHeaders } from '../utils/fetchIdentity.js';
 
 // Security configuration
 const SECURITY_CONFIG = {
@@ -125,7 +126,7 @@ export class SSRFProtection {
     const fetchOptions = {
       timeout: options.timeout || 30000,
       headers: {
-        'User-Agent': 'CrawlForge/3.0 (Security-Hardened)',
+        ...identityHeaders({ role: 'health-check' }),
         ...options.headers
       },
       ...options

package/src/server/requestContext.js

@@ -24,3 +24,26 @@ export const requestContext = new AsyncLocalStorage();
 export function isInternalRequest() {
   return requestContext.getStore()?.internal === true;
 }
+
+/**
+ * Record that the compliance gate refused this invocation before anything was
+ * fetched — robots.txt disallowed the path, or the host is on the permanent
+ * blocklist.
+ *
+ * withAuth reads this when it decides the charge. The flag rather than the
+ * error's `code` is deliberate: tool handlers catch their own errors and
+ * return `{ isError: true }` with only a message, so the typed error never
+ * reaches withAuth. It also survives both routes a refusal can take — thrown,
+ * or swallowed into an isError result.
+ *
+ * @param {string} code 'ROBOTS_DISALLOWED' | 'HOST_BLOCKED'
+ */
+export function markPreflightRefusal(code) {
+  const store = requestContext.getStore();
+  if (store) store.preflightRefusal = code;
+}
+
+/** The refusal code recorded for this invocation, or null. */
+export function preflightRefusal() {
+  return requestContext.getStore()?.preflightRefusal ?? null;
+}

package/src/server/withAuth.js

@@ -15,7 +15,7 @@
 
 import { createHash } from 'node:crypto';
 import { recordToolInvocation } from '../observability/tracing.js';
-import { isInternalRequest } from './requestContext.js';
+import { isInternalRequest, preflightRefusal, requestContext } from './requestContext.js';
 
 export function hashParams(params) {
   try {
@@ -33,7 +33,7 @@ export function hashParams(params) {
  */
 export function makeWithAuth({ authManager, logger, metrics = null }) {
   return function withAuth(toolName, handler) {
-    return async (params) => {
+    const invoke = async (params) => {
       const startTime = Date.now();
       const paramHash = hashParams(params);
       const creatorMode = authManager.isCreatorMode();
@@ -82,7 +82,14 @@ export function makeWithAuth({ authManager, logger, metrics = null }) {
         // it as an error, honoring CLAUDE.md's "half credits on error" contract.
         const isErrorResult = result?.isError === true;
         outcome = isErrorResult ? 'error' : 'success';
-        const charge = creditCost === 0
+        // A pre-flight refusal (robots.txt disallowed the path, or the host is
+        // blocklisted) means we fetched nothing at all, so it costs nothing —
+        // not even the half-credit error rate, which exists for work that ran
+        // and then failed. Only when the refusal actually sank the call: a
+        // multi-URL tool that skipped one disallowed URL and still returned a
+        // result did real work and bills for it.
+        const refused = isErrorResult && preflightRefusal() !== null;
+        const charge = creditCost === 0 || refused
           ? 0
           : (isErrorResult ? Math.max(1, Math.floor(creditCost * 0.5)) : creditCost);
 
@@ -121,7 +128,7 @@ export function makeWithAuth({ authManager, logger, metrics = null }) {
         // DataForSEO is unconfigured — a no-op). Emit NO usage event at all so
         // the backend has nothing to (re-)price; reporting 0 would still create
         // a serp_rank record the backend could recompute to full cost.
-        if (!creatorMode && creditCost > 0) {
+        if (!creatorMode && creditCost > 0 && !refused) {
           await authManager.reportUsage(toolName, charge, params, isErrorResult ? 500 : 200, Date.now() - startTime);
         }
 
@@ -132,7 +139,7 @@ export function makeWithAuth({ authManager, logger, metrics = null }) {
         // Half-charge on error — but never charge a free (0-cost) call, never
         // let Math.max(1, …) floor a 0 up to 1 credit, and never bill at all
         // if the handler never ran (e.g. the credit check itself threw).
-        if (!creatorMode && creditCost > 0 && handlerStarted) {
+        if (!creatorMode && creditCost > 0 && handlerStarted && preflightRefusal() === null) {
           await authManager.reportUsage(
             toolName,
             Math.max(1, Math.floor(creditCost * 0.5)),
@@ -183,5 +190,14 @@ export function makeWithAuth({ authManager, logger, metrics = null }) {
         }, thrown);
       }
     };
+
+    // Every invocation runs in its own context so the compliance gate can stamp
+    // a refusal where the billing decision can see it. Any outer store (the
+    // HTTP transport's `internal` flag) is spread in, not replaced — and stdio
+    // callers, who have no transport-provided store, get one here.
+    return async (params) => requestContext.run(
+      { ...(requestContext.getStore() ?? {}), preflightRefusal: null },
+      () => invoke(params)
+    );
   };
 }

package/src/skills/agent-skills/crawlforge-deep-research/SKILL.md

@@ -30,7 +30,7 @@ skill (`scrape` / `extract_content`) instead.
 | Need | Tool | Cost |
 |------|------|------|
 | A ranked list of result URLs + snippets | `search_web` | 5 |
-| Reddit posts, comments, or a full thread | `reddit_search` | 2 |
+| Reddit posts, comments, or a full thread | `reddit_search` | 5 |
 | A direct answer, agent decides what to read | `agent` | 8 (scales) |
 | A synthesized, multi-source, cited report | `deep_research` | 10+ (scales) |
 
@@ -127,5 +127,10 @@ With no LLM configured, `deep_research` returns structured **raw evidence** for
 the calling assistant (e.g. Claude Code) to synthesize — this is expected, do
 not suggest adding API keys.
 
+On local Ollama, claim judgement (relevance, grouping, contradiction) uses
+`gemma3:12b` when it is installed and the extraction model otherwise; conflicts
+are reported only with that model or a cloud provider, so `conflictsFound: 0`
+on a machine without it is expected, not a failure.
+
 See [research workflows](references/workflows.md) for pipelines, depth tiers,
 and parameter detail.

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -83,6 +83,33 @@ const ScrollActionSchema = BaseActionSchema.extend({
   y: z.number().min(0).optional()
 });
 
+const SelectActionSchema = BaseActionSchema.extend({
+  type: z.literal('select'),
+  selector: z.string(),
+  // A plain string matches an <option> by value OR label — see ActionExecutor's
+  // SelectActionSchema.
+  value: z.string().optional(),
+  values: z.array(z.string()).optional()
+}).refine(data => data.value !== undefined || (data.values && data.values.length > 0), {
+  message: 'Select action requires value or values'
+});
+
+const HoverActionSchema = BaseActionSchema.extend({
+  type: z.literal('hover'),
+  selector: z.string(),
+  force: z.boolean().default(false),
+  position: z.object({
+    x: z.number(),
+    y: z.number()
+  }).optional()
+});
+
+const NavigateActionSchema = BaseActionSchema.extend({
+  type: z.literal('navigate'),
+  url: z.string().url(),
+  waitUntil: z.enum(['load', 'domcontentloaded', 'networkidle', 'commit']).optional()
+});
+
 const ScreenshotActionSchema = BaseActionSchema.extend({
   type: z.literal('screenshot'),
   selector: z.string().optional(),
@@ -104,6 +131,9 @@ const ActionSchema = z.union([
   TypeActionSchema,
   PressActionSchema,
   ScrollActionSchema,
+  SelectActionSchema,
+  HoverActionSchema,
+  NavigateActionSchema,
   ScreenshotActionSchema,
   ExecuteJavaScriptActionSchema
 ]);
@@ -145,7 +175,11 @@ const ScrapeWithActionsSchema = z.object({
     userAgent: z.string().optional(),
     viewportWidth: z.number().min(800).max(1920).default(1280),
     viewportHeight: z.number().min(600).max(1080).default(720),
-    timeout: z.number().min(10000).max(120000).default(30000)
+    timeout: z.number().min(10000).max(120000).default(30000),
+    // Run the chain in the stealth browser (StealthBrowserManager) instead of
+    // the standard pool. executeSession turns this into the stealthMode object
+    // ActionExecutor/BrowserProcessor read.
+    stealth: z.boolean().default(false)
   }).optional(),
 
   // Content extraction options
@@ -156,6 +190,10 @@ const ScrapeWithActionsSchema = z.object({
     includeImages: z.boolean().default(true)
   }).optional(),
 
+  // Per-request robots.txt override (G5). Undefined leaves the configured
+  // default in force; false is honoured, warned about and audited by the gate.
+  respect_robots: z.boolean().optional(),
+
   // Error handling
   continueOnActionError: z.boolean().default(false),
   maxRetries: z.number().min(0).max(3).default(1),
@@ -341,6 +379,16 @@ export class ScrapeWithActionsTool extends EventEmitter {
       ...params.browserOptions
     };
 
+    // `stealth: true` is the caller-facing switch; everything downstream reads
+    // browserOptions.stealthMode.
+    if (browserOptions.stealth) {
+      browserOptions.stealthMode = { enabled: true };
+    }
+
+    // The robots override rides with the browser options so ActionExecutor's
+    // gate sees it on the initial load and on every `navigate` action.
+    browserOptions.respectRobots = params.respect_robots;
+
     // Build action chain with form auto-fill if provided
     let actionChain = [...params.actions];
 

package/src/tools/advanced/batchScrape/schema.js

@@ -23,6 +23,10 @@ export const BatchScrapeSchema = z.object({
     signingSecret: z.string().optional()
   }).optional(),
   extractionSchema: z.record(z.string()).optional(),
+  // Compliance overrides, per request: identify as yourself for a target you
+  // have your own agreement with, and take responsibility for ignoring robots.
+  user_agent: z.string().optional(),
+  respect_robots: z.boolean().optional(),
   maxConcurrency: z.number().min(1).max(20).default(10),
   delayBetweenRequests: z.number().min(0).max(10000).default(100),
   includeMetadata: z.boolean().default(true),