crawlforge-mcp-server 5.6.10 → 5.8.0

package/src/tools/scrape/unifiedScrape.js

@@ -5,57 +5,76 @@
  * Mirrors the output shape of ScrapeWithActionsTool.generateFormats():
  *   content.html, content.rawHtml, content.text, content.markdown,
  *   content.links, content.metadata, content.screenshots, content.json
+ * plus the query-scoped formats (Phase 1): content.highlights for
+ * {type:"highlights"} and content.answer for {type:"question"}, both built
+ * from the markdown this same call produces, verbatim with offsets into it.
  *
  * onlyMainContent maps to Readability boilerplate removal (same as extractContent).
  * Partial success: per-format warnings[] never fail the whole call.
  */
 
 import { z } from 'zod';
+import { load } from 'cheerio';
+import { documentVerdict, segmentUnits, rankUnits } from 'crawlforge-extractors';
+import {
+  SCRAPE_STRING_FORMATS, JsonFormatSchema, HighlightsFormatSchema, QuestionFormatSchema, FormatSchema,
+  scrapeFormatSurcharge
+} from './formats.js';
+import { toPublicUnit, parseChosenIndexes, groundingCheck } from './_highlights.js';
+import { setActualCost } from '../../server/requestContext.js';
+import { fenceUntrusted } from '../../utils/untrustedContent.js';
 import { fetchAndParse } from '../extract/_fetchAndParse.js';
 import { extractMainContent, isThinMainContent } from './_mainContent.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
 import { stripHiddenFromDom } from '../../utils/hiddenContent.js';
 import { extractBlockText } from '../basic/extractText.js';
 import { pageTitle } from '../../utils/pageTitle.js';
+import { noteHostBlocked, clearHostBlocked } from '../../utils/hostRateLimiter.js';
 
 // ── Schema ────────────────────────────────────────────────────────────────────
 
-const JsonFormatSchema = z.object({
-  type: z.literal('json'),
-  schema: z.record(z.any()).optional(),
-  prompt: z.string().optional()
-});
+export { SCRAPE_STRING_FORMATS, JsonFormatSchema, HighlightsFormatSchema, QuestionFormatSchema, FormatSchema };
+
+// Above this, a markdown result is the whole page in the context window;
+// the warning names the format that returns only the matching units (1.4).
+const OVERSIZED_MARKDOWN_CHARS = 40000;
+// How many extractive units a question's answer rests on.
+const QUESTION_EVIDENCE_UNITS = 5;
+
+const isQueryFormat = (fmt) => Boolean(fmt) && typeof fmt === 'object' && (fmt.type === 'highlights' || fmt.type === 'question');
 
-const FormatSchema = z.union([
-  z.enum(['markdown', 'html', 'rawHtml', 'text', 'links', 'metadata', 'screenshot', 'branding']),
-  JsonFormatSchema
-]);
+// The six public fields in tools/list order, each carrying the description
+// the client sees. server.js spreads this with COMPLIANCE_PARAMS instead of
+// keeping its own copy (Phase 0, 0.3).
+export const SCRAPE_INPUT_SHAPE = {
+  url: z.string().url().describe('The URL to scrape'),
+  formats: z.array(FormatSchema).min(1).optional().default(['markdown']).describe('Formats to return (default: ["markdown"])'),
+  onlyMainContent: z.boolean().optional().default(true).describe('Strip boilerplate via Readability (default: true)'),
+  // Pass-through to fetchAndParse
+  timeoutMs: z.number().min(1000).max(60000).optional().default(15000).describe('Fetch timeout in ms'),
+  // Optional, additive: only consulted when 'branding' / 'screenshot' is requested.
+  brandingOptions: z.object({
+    fetchLinkedCss: z.boolean().optional().default(true).describe('Fetch linked stylesheets for richer color/font extraction'),
+    maxStylesheets: z.number().min(0).max(20).optional().default(10).describe('Max linked stylesheets to fetch')
+  }).optional().describe('Options for the "branding" format'),
+  screenshotOptions: z.object({
+    fullPage: z.boolean().optional().default(false).describe('Capture the full scrollable page'),
+    format: z.enum(['png', 'jpeg']).optional().default('png'),
+    quality: z.number().min(0).max(100).optional().describe('JPEG quality (jpeg only)')
+  }).optional().describe('Options for the "screenshot" format')
+};
 
 export const UnifiedScrapeSchema = z.object({
-  url: z.string().url(),
-  formats: z.array(FormatSchema).min(1).default(['markdown']),
-  onlyMainContent: z.boolean().optional().default(true),
+  ...SCRAPE_INPUT_SHAPE,
   // Remove content a browser would not paint (screen-reader-only labels,
   // state-gated theme badges) before deriving any format. "linked" also fetches
   // the page's stylesheets, which is what resolves class-driven display:none;
   // "inline" uses only the document's own <style> blocks and costs no requests.
   resolveHiddenContent: z.enum(['linked', 'inline', 'off']).optional().default('linked'),
-  // Pass-through to fetchAndParse
-  timeoutMs: z.number().min(1000).max(60000).optional().default(15000),
   // Compliance overrides, per request: identify as yourself for a target you
   // have your own agreement with, and take responsibility for ignoring robots.
   user_agent: z.string().optional(),
-  respect_robots: z.boolean().optional(),
-  // Optional, additive: only consulted when 'branding' / 'screenshot' is requested.
-  brandingOptions: z.object({
-    fetchLinkedCss: z.boolean().optional().default(true),
-    maxStylesheets: z.number().min(0).max(20).optional().default(10)
-  }).optional(),
-  screenshotOptions: z.object({
-    fullPage: z.boolean().optional().default(false),
-    format: z.enum(['png', 'jpeg']).optional().default('png'),
-    quality: z.number().min(0).max(100).optional()
-  }).optional()
+  respect_robots: z.boolean().optional()
 });
 
 // ── Helpers ───────────────────────────────────────────────────────────────────
@@ -175,6 +194,61 @@ export class UnifiedScrapeTool {
     // Optional shared ActionExecutor (injected from server.js so we reuse the
     // existing browser pool rather than spinning up a second one).
     this._actionExecutor = options.actionExecutor || null;
+    this._mcpServer = null;
+  }
+
+  /** Wire the MCP server so the mode:"model" step can fall back to client sampling. */
+  setMcpServer(mcpServer) {
+    this._mcpServer = mcpServer;
+  }
+
+  /**
+   * One completion through the SamplingClient chain (Ollama → server keys →
+   * MCP sampling); throws when no route exists. Page text must already be
+   * fenced by the caller.
+   */
+  async _complete(prompt, options) {
+    const { SamplingClient } = await import('../../core/SamplingClient.js');
+    return new SamplingClient({ mcpServer: this._mcpServer }).complete(prompt, options);
+  }
+
+  /**
+   * mode:"model" for highlights: the model picks which of the extractive
+   * candidates to keep; it never writes text. Returns the chosen units in
+   * extractive order, or the extractive top `limit` when the reply does not
+   * name any candidate.
+   */
+  async _chooseHighlights(candidates, query, limit, warnings) {
+    const listing = candidates.map((unit, index) => `[${index}] ${unit.text}`).join('\n');
+    const { text } = await this._complete(
+      `${fenceUntrusted(listing, 'numbered page excerpts')}\nQuery: ${query}\n` +
+      `Reply with the numbers of the ${limit} excerpts most relevant to the query.`,
+      {
+        maxTokens: Math.max(64, limit * 4),
+        systemPrompt: 'You choose which numbered excerpts from a web page answer a query. ' +
+          'Reply with the numbers only, comma-separated, most relevant first. Do not write anything else.'
+      }
+    );
+    const chosen = parseChosenIndexes(text, candidates.length, limit);
+    if (chosen.length === 0) {
+      warnings.push('highlights: the model named no candidate; the extractive order is returned');
+      return candidates.slice(0, limit);
+    }
+    return chosen.sort((a, b) => a - b).map((index) => candidates[index]);
+  }
+
+  /** mode:"model" for question: an answer written from the fenced evidence alone. */
+  async _answerQuestion(evidence, question) {
+    const { text } = await this._complete(
+      `${fenceUntrusted(evidence.map((unit) => unit.text).join('\n'), 'page evidence')}\nQuestion: ${question}\n` +
+      'Answer from the evidence only; if the evidence does not say, say so. Reply with the answer only.',
+      {
+        maxTokens: 256,
+        systemPrompt: 'You answer a question from evidence quoted from a web page. Use only the evidence. ' +
+          'If the evidence does not contain the answer, say that it does not. Reply with the answer only, no preamble.'
+      }
+    );
+    return text.trim();
   }
 
   /** Lazy-load ExtractWithLlm to avoid pulling in heavy deps unless needed. */
@@ -205,19 +279,61 @@ export class UnifiedScrapeTool {
     const { url, formats, onlyMainContent, timeoutMs, brandingOptions, screenshotOptions, resolveHiddenContent } = validated;
 
     // Single fetch
-    let html, $, finalUrl, fetchWarnings;
+    let html, $, finalUrl, fetchWarnings, status;
     try {
-      ({ html, $, finalUrl, warnings: fetchWarnings } = await fetchAndParse(url, {
+      ({ html, $, finalUrl, warnings: fetchWarnings, status } = await fetchAndParse(url, {
         timeoutMs,
         userAgent: validated.user_agent,
         respectRobots: validated.respect_robots,
         tool: 'scrape',
-        stripTags: [] // we handle boilerplate ourselves
+        stripTags: [], // we handle boilerplate ourselves
+        // A real wall reaches a plain fetch as a 403 with the challenge in
+        // the body; the verdict below needs that body and the status.
+        errorDocuments: true
       }));
     } catch (err) {
       throw new Error(`scrape: fetch failed for ${url}: ${err.message}`);
     }
 
+    // A bot wall, an HTTP error page, an empty shell or an error placeholder
+    // arrives with a title and prose of its own; reported as a successful
+    // scrape it hid the block for three rounds (producthunt.com, R10 Q1 →
+    // R15). The stealth path has named these since 5.6.2; this path never
+    // looked, and threw away every non-2xx body before it could (Phase 0,
+    // 0.1). Scripts stay in $ for the rawHtml and metadata formats,
+    // so the text is measured on a copy without them — the vendor tables
+    // count the characters a reader would see, as the browser path does.
+    // Nothing below runs on a failed verdict: a screenshot would launch a
+    // browser on the wall and branding would fetch its stylesheets.
+    const title = pageTitle($);
+    const $visible = load(html);
+    $visible('script, style, noscript, template').remove();
+    const verdict = documentVerdict(
+      { url: finalUrl, title, text: $visible('body').text(), html, status },
+      { fetcher: 'a plain fetch', rendered: false, contentReturned: false }
+    );
+    // Host memory (0.6): remember the vendor that walled this host, keyed by
+    // the URL the caller passed as well as the final one when a redirect
+    // moved hosts, since a later call looks it up by what it was given.
+    const hosts = [finalUrl, url];
+    if (verdict.blocked) {
+      for (const h of hosts) noteHostBlocked(h, verdict.blocked.vendor);
+    } else if (verdict.success) {
+      for (const h of hosts) clearHostBlocked(h);
+    }
+    if (!verdict.success) {
+      return {
+        success: false,
+        url: finalUrl,
+        status,
+        title,
+        error: verdict.error,
+        ...(verdict.blocked ? { blocked: verdict.blocked } : {}),
+        content: {},
+        warnings: fetchWarnings.length > 0 ? fetchWarnings : undefined
+      };
+    }
+
     // Resolve <base href> once per document (if present) so link resolution
     // matches how a browser would navigate, instead of always using finalUrl.
     let docBaseUrl = finalUrl;
@@ -249,6 +365,25 @@ export class UnifiedScrapeTool {
       return mainHtml;
     }
 
+    // The markdown format and the query-scoped formats read the same string,
+    // produced once, so a highlight's offset always indexes what `markdown`
+    // returned at this call's onlyMainContent.
+    let markdown = null;
+    function getMarkdown() {
+      if (markdown === null) {
+        markdown = onlyMainContent ? htmlToMarkdown(getMainHtml()) : htmlToMarkdown($.html('body') || html);
+      }
+      return markdown;
+    }
+    let units = null;
+    function getUnits() {
+      if (units === null) units = segmentUnits(getMarkdown());
+      return units;
+    }
+    // Whether a model completed for a mode:"model" format; the model
+    // surcharge is charged only then.
+    let modelUsed = false;
+
     const content = {};
     // The gate's warnings (e.g. a respect_robots override) travel with the
     // per-format ones, so the caller sees the decision in the response.
@@ -339,13 +474,66 @@ export class UnifiedScrapeTool {
         continue;
       }
 
+      // Query-scoped formats (Phase 1): verbatim units of this call's
+      // markdown, ranked against the query. mode:"model" lets a model choose
+      // among (highlights) or answer from (question) those units; when no
+      // LLM route exists the extractive result stands and the model
+      // surcharge is dropped below. An empty match is a warning, not an error.
+      if (isQueryFormat(fmt)) {
+        const modelUnavailable = (label) =>
+          warnings.push(`${label}: mode "model" was unavailable (no Ollama, API key or client sampling); the extractive result is returned at the extractive price`);
+        try {
+          if (fmt.type === 'highlights') {
+            let chosen = rankUnits(getUnits(), fmt.query, { maxUnits: fmt.max_highlights });
+            if (fmt.mode === 'model' && chosen.length > 0) {
+              const candidates = rankUnits(getUnits(), fmt.query, { maxUnits: fmt.max_highlights * 3 });
+              if (candidates.length > fmt.max_highlights) {
+                try {
+                  chosen = await this._chooseHighlights(candidates, fmt.query, fmt.max_highlights, warnings);
+                  modelUsed = true;
+                } catch {
+                  modelUnavailable('highlights');
+                }
+              } else {
+                warnings.push('highlights: no more candidates than max_highlights, so the model had nothing to choose; charged at the extractive price');
+              }
+            }
+            content.highlights = chosen.map(toPublicUnit);
+            if (chosen.length === 0) warnings.push(`highlights: no sentence, table row or code block matched "${fmt.query}"`);
+          } else {
+            const evidence = rankUnits(getUnits(), fmt.question, { maxUnits: QUESTION_EVIDENCE_UNITS });
+            const evidenceText = evidence.map((unit) => unit.text).join('\n');
+            let text = evidenceText;
+            let grounded = true;
+            if (fmt.mode === 'model' && evidence.length > 0) {
+              try {
+                text = await this._answerQuestion(evidence, fmt.question);
+                modelUsed = true;
+                const check = groundingCheck(text, evidenceText, fmt.question);
+                grounded = check.grounded;
+                if (!grounded) {
+                  warnings.push(`question: the model's answer has ${check.unbacked.length} token(s) not found in the evidence or the question (${check.unbacked.join(', ')}); grounded: false`);
+                }
+              } catch {
+                modelUnavailable('question');
+              }
+            }
+            content.answer = { text, grounded, evidence: evidence.map(toPublicUnit) };
+            if (evidence.length === 0) warnings.push(`question: no sentence, table row or code block matched "${fmt.question}"`);
+          }
+        } catch (err) {
+          if (fmt.type === 'highlights') content.highlights = [];
+          else content.answer = { text: '', grounded: true, evidence: [] };
+          warnings.push(`${fmt.type}: ${err.message}`);
+        }
+        continue;
+      }
+
       // String formats
       switch (fmt) {
         case 'markdown':
           try {
-            content.markdown = onlyMainContent
-              ? htmlToMarkdown(getMainHtml())
-              : htmlToMarkdown($.html('body') || html);
+            content.markdown = getMarkdown();
           } catch (err) {
             content.markdown = '';
             warnings.push(`markdown: ${err.message}`);
@@ -457,6 +645,19 @@ export class UnifiedScrapeTool {
       }
     }
 
+    const queryScoped = formats.some(isQueryFormat);
+    if (queryScoped && !formats.includes('markdown')) {
+      warnings.push('offsets index the "markdown" format of this call (same onlyMainContent); add "markdown" to formats to quote with a locator');
+    }
+    if (!queryScoped && typeof content.markdown === 'string' && content.markdown.length > OVERSIZED_MARKDOWN_CHARS) {
+      warnings.push(`markdown: ${content.markdown.length} characters; ask for {type:"highlights", query} (1 extra credit, no model) to get only the matching sentences, table rows and code blocks`);
+    }
+    // The projection charged the model step; nothing ran, so the charge
+    // drops to the extractive price (2 is scrape's base in AuthManager;
+    // withAuth clamps to the projection either way).
+    const surcharge = scrapeFormatSurcharge(formats);
+    if (surcharge.model > 0 && !modelUsed) setActualCost(2 + surcharge.query);
+
     return {
       success: true,
       url: finalUrl,

package/src/utils/challengeDetection.js

@@ -1,79 +1,10 @@
 /**
- * challengeDetection.js — recognise a bot-wall interstitial served as a page.
+ * challengeDetection.js — re-export of the challenge-vendor tables.
  *
- * Cloudflare, Amazon, DataDome, PerimeterX and Akamai all answer a blocked
- * request with HTTP 200 and a page of their own: a title, some prose and a
- * challenge script. Reported as a successful scrape, that page hides the
- * block — producthunt.com came back "success:true, title: Just a moment..."
- * for three regression rounds (R10 Q1 → R15, 2026-09-04). A title match is
- * definitive; a script or form marker is definitive only on a short page,
- * because a real page can legitimately embed a Turnstile widget.
+ * The tables moved to crawlforge-extractors 1.7.0 so the REST `scrape` route
+ * reaches the same verdict as the MCP server (Phase 0, 0.1). Callers keep
+ * importing from here.
  */
 
-const SHORT_PAGE_CHARS = 4000;
-
-const CHALLENGES = [
-  {
-    vendor: 'cloudflare',
-    title: /^just a moment/i,
-    markers: /challenges\.cloudflare\.com|cf-chl-|_cf_chl_opt|cf_chl_rc_|window\._cf_chl/i,
-    evidence: 'a Cloudflare challenge script'
-  },
-  {
-    vendor: 'amazon',
-    // The robot check is the only Amazon page whose form posts to validateCaptcha.
-    definitive: /action="[^"]*validateCaptcha/i,
-    evidence: 'the validateCaptcha form'
-  },
-  {
-    vendor: 'datadome',
-    markers: /captcha-delivery\.com\/captcha|geo\.captcha-delivery\.com|dd\.captcha/i,
-    evidence: 'a DataDome captcha frame'
-  },
-  {
-    vendor: 'perimeterx',
-    markers: /px-captcha|_pxCaptcha|human-challenge/i,
-    evidence: 'a PerimeterX / HUMAN challenge element'
-  },
-  {
-    vendor: 'akamai',
-    title: /^access denied/i,
-    markers: /errors\.edgesuite\.net/i,
-    evidence: 'an Akamai access-denied page'
-  },
-  {
-    // Vercel's Attack Challenge Mode answers with HTTP 429, an
-    // x-vercel-mitigated: challenge header and a JavaScript interstitial
-    // titled "Vercel Security Checkpoint" (lesswrong.com, hashicorp.com,
-    // bombas.com, R17 2026-09-04). Chromium solves it and reloads; camoufox
-    // was left on the interstitial, which then read as a successful scrape.
-    vendor: 'vercel',
-    title: /^vercel security checkpoint/i,
-    markers: /vercel\.link\/security-checkpoint|_vercel\/challenge|x-vercel-challenge-token/i,
-    evidence: 'a Vercel Security Checkpoint page'
-  }
-];
-
-/**
- * @param {{ title?: string, html?: string, text?: string }} page
- * @returns {{ vendor: string, evidence: string } | null}
- */
-export function detectChallengePage({ title = '', html = '', text = '' } = {}) {
-  const cleanTitle = (title || '').trim();
-  const visible = (text || '').replace(/\s+/g, ' ').trim();
-  const shortPage = visible.length < SHORT_PAGE_CHARS;
-  for (const challenge of CHALLENGES) {
-    if (challenge.title && challenge.title.test(cleanTitle)) {
-      return { vendor: challenge.vendor, evidence: `title "${cleanTitle}"` };
-    }
-    if (challenge.definitive && challenge.definitive.test(html)) {
-      return { vendor: challenge.vendor, evidence: challenge.evidence };
-    }
-    if (shortPage && challenge.markers && challenge.markers.test(html)) {
-      return { vendor: challenge.vendor, evidence: `${challenge.evidence} on a ${visible.length}-character page` };
-    }
-  }
-  return null;
-}
-
-export default detectChallengePage;
+export { detectChallengePage } from 'crawlforge-extractors';
+export { detectChallengePage as default } from 'crawlforge-extractors';

package/src/utils/hostRateLimiter.js

@@ -18,6 +18,11 @@
  *     request to that host waits instead of retrying straight into the wall.
  * Both are host-scoped and survive RATE_LIMIT_PER_DOMAIN=false: an operator
  * turning off our own throttle is not a licence to ignore the site's.
+ *
+ * The same per-host state remembers the bot-defence vendor that last walled
+ * the host (Phase 0, 0.6). The scrape verdict writes it; Phase 3 reads it
+ * before fetching. In-memory only, bounded to HOST_STATE_MAX hosts with the
+ * least recently touched dropped first.
  */
 import { RateLimiter } from './rateLimiter.js';
 import { config } from '../constants/config.js';
@@ -34,7 +39,10 @@ function limiter() {
   return _limiter;
 }
 
-/** host → { lastRequestAt, notBefore } */
+/** Hosts remembered at once; past this the least recently touched is dropped. */
+export const HOST_STATE_MAX = 1000;
+
+/** host → { lastRequestAt, notBefore, blockedUntil, vendor }, least recently touched first */
 const hostState = new Map();
 
 function hostOf(url) {
@@ -47,10 +55,14 @@ function hostOf(url) {
 
 function stateFor(host) {
   let state = hostState.get(host);
-  if (!state) {
-    state = { lastRequestAt: 0, notBefore: 0 };
-    hostState.set(host, state);
+  if (state) {
+    // Re-insert so Map order stays least recently touched first.
+    hostState.delete(host);
+  } else {
+    state = { lastRequestAt: 0, notBefore: 0, blockedUntil: 0, vendor: null };
+    if (hostState.size >= HOST_STATE_MAX) hostState.delete(hostState.keys().next().value);
   }
+  hostState.set(host, state);
   return state;
 }
 
@@ -105,6 +117,44 @@ export function getHostBackoffMs(url) {
   return state ? Math.max(0, state.notBefore - Date.now()) : 0;
 }
 
+/**
+ * Remember that this URL's host answered with a bot wall. Nothing reads it
+ * yet — Phase 3 will, before fetching.
+ * @param {string} url
+ * @param {string} vendor
+ * @param {{ ttlMs?: number, now?: number }} [options] `now` for deterministic tests
+ */
+export function noteHostBlocked(url, vendor, { ttlMs = 24 * 60 * 60 * 1000, now = Date.now() } = {}) {
+  const host = hostOf(url);
+  if (!host) return;
+  const state = stateFor(host);
+  state.blockedUntil = now + ttlMs;
+  state.vendor = vendor;
+}
+
+/** Forget a block after a clean fetch of the same host. */
+export function clearHostBlocked(url) {
+  const host = hostOf(url);
+  const state = host ? hostState.get(host) : null;
+  if (!state) return;
+  state.blockedUntil = 0;
+  state.vendor = null;
+}
+
+/**
+ * The remembered block for this URL's host, or null when there is none or
+ * it has expired.
+ * @param {string} url
+ * @param {number} [now] epoch ms, for deterministic tests
+ * @returns {{ vendor: string, blockedUntil: number } | null}
+ */
+export function getHostBlock(url, now = Date.now()) {
+  const host = hostOf(url);
+  const state = host ? hostState.get(host) : null;
+  if (!state || state.blockedUntil <= now) return null;
+  return { vendor: state.vendor, blockedUntil: state.blockedUntil };
+}
+
 /**
  * Wait (if necessary) until another request to this URL's host is allowed.
  * Never throws — a limiter failure must not block a legitimate fetch.

package/src/utils/secretMask.js

@@ -1,16 +1,23 @@
 /**
  * secretMask -- redact sensitive values from objects/strings before they reach logs.
  *
+ * Masks by key name (SECRET_KEYS_RE) and, structurally, the text a
+ * scrape_with_actions type/fill/press step sends to the page.
+ *
  * Usage:
  *   import { maskSecrets, maskString } from './secretMask.js';
  *   logger.error('fetch failed', maskSecrets({ apiKey, url, error }));
  */
 
-const SECRET_KEYS_RE = /api[_-]?key|apikey|x-api-key|password|passwd|secret|token|authorization|auth|credential|private[_-]?key|access[_-]?key|proxy_url|proxyurl|cookie/i;
+const SECRET_KEYS_RE = /api[_-]?key|apikey|x-api-key|password|passwd|secret|token|authorization|auth|credential|login|formautofill|private[_-]?key|access[_-]?key|proxy_url|proxyurl|cookie/i;
 
 const MASK = '[REDACTED]';
 const PARTIAL_MASK_LEN = 4; // show last N chars of long secrets
 
+// scrape_with_actions: these action types send the text/value they carry to the page.
+const TYPED_ACTION_TYPES = new Set(['type', 'fill', 'press']);
+const TYPED_ACTION_FIELDS = ['text', 'value'];
+
 /**
  * Mask a single string value.
  * Shows last 4 chars if string is long enough to give context, else full mask.
@@ -23,6 +30,23 @@ export function maskString(value) {
   return `${MASK}...${value.slice(-PARTIAL_MASK_LEN)}`;
 }
 
+/**
+ * Typed text hides under neutral keys (actions[].text), so key matching misses it.
+ * Fully masked: the last-4 tail that tells API keys apart would only leak a password.
+ * Selectors and key names stay. Returns a copy; never mutates.
+ * @param {*} action
+ * @returns {*}
+ */
+function maskTypedAction(action) {
+  if (action === null || typeof action !== 'object' || Array.isArray(action)) return action;
+  if (typeof action.type !== 'string' || !TYPED_ACTION_TYPES.has(action.type.toLowerCase())) return action;
+  const result = { ...action };
+  for (const field of TYPED_ACTION_FIELDS) {
+    if (field in result) result[field] = MASK;
+  }
+  return result;
+}
+
 /**
  * Deep-clone obj and redact any key whose name matches SECRET_KEYS_RE.
  * Handles plain objects, arrays, and primitive values.
@@ -43,6 +67,8 @@ export function maskSecrets(obj, depth = 0) {
     for (const [key, value] of Object.entries(obj)) {
       if (SECRET_KEYS_RE.test(key)) {
         result[key] = typeof value === 'string' ? maskString(value) : MASK;
+      } else if (key === 'actions' && Array.isArray(value)) {
+        result[key] = value.map(item => maskSecrets(maskTypedAction(item), depth + 1));
       } else {
         result[key] = maskSecrets(value, depth + 1);
       }

package/src/utils/stealthVerdict.js

@@ -1,82 +1,10 @@
 /**
- * stealthVerdict.js — decide what a stealth-rendered document is.
+ * stealthVerdict.js — re-export of the document verdict.
  *
- * A browser scrape that reached a document is not a scrape that succeeded.
- * Round 18 (2026-09-04, pricing sweep) returned success:true for Edmunds'
- * "403 - Access Denied" page, Lufthansa's "Page not found", Home Depot's
- * "Error Page" and Hilton's "Something went wrong" — each an HTTP error or a
- * soft block served with a title and a few lines of prose, so the R15
- * challenge-vendor check and the R16 empty-document check both passed it.
- * The HTTP status of the navigation and a short error-titled document are
- * now failures too; the content is still returned so the caller can read
- * what the site served.
+ * `stealthDocumentVerdict` moved to crawlforge-extractors 1.7.0 as
+ * `documentVerdict` — the same behaviour and messages for the stealth callers,
+ * plus a `fetcher` option so the plain `scrape` path on both surfaces reaches
+ * the same verdict (Phase 0, 0.1). Callers keep importing from here.
  */
 
-import { detectChallengePage } from './challengeDetection.js';
-
-// A document this short with one of these titles is an error placeholder,
-// not a page. Real pages with these words in a longer title (a news story
-// about an outage) carry far more text than the cap.
-const ERROR_TITLE = /^(?:(?:\d{3}\s*[-–—|:]\s*)?(?:error(?: page)?|access denied|forbidden|(?:page )?not found|service unavailable|internal server error|bad gateway|something went wrong|oops!?[^\n]{0,60}))$/i;
-export const SOFT_ERROR_MAX_CHARS = 1500;
-
-/**
- * @param {{ url?: string, title?: string, text?: string, html?: string, status?: number|null }} scraped
- * @param {{ waitedMs?: number, allowEmpty?: boolean }} [options]
- * @returns {{ success: boolean, status: number|null, error?: string, blocked?: { vendor: string, evidence: string } }}
- */
-export function stealthDocumentVerdict(scraped, { waitedMs = 0, allowEmpty = false } = {}) {
-  const status = Number.isInteger(scraped?.status) ? scraped.status : null;
-  const url = scraped?.url || '';
-  const title = String(scraped?.title || '').trim();
-  const text = String(scraped?.text || '').trim();
-
-  const challenge = detectChallengePage(scraped || {});
-  if (challenge) {
-    return {
-      success: false,
-      status,
-      blocked: challenge,
-      error: `${challenge.vendor} served a challenge page instead of the content (${challenge.evidence}); the stealth browser did not pass it.`
-    };
-  }
-
-  if (status !== null && status >= 400) {
-    const why = status === 403
-      ? 'A 403 with no challenge vendor on the page is an IP-reputation or WAF block; the site will not serve this network.'
-      : status === 404
-        ? 'The site says the URL does not exist — check the path.'
-        : status === 429
-          ? 'The site is rate-limiting this network; wait before retrying.'
-          : 'Retry later; the server, not the page, failed.';
-    return {
-      success: false,
-      status,
-      error: `HTTP ${status}: ${url} answered with an error page${title ? ` titled "${title}"` : ''}, not the resource; the content returned is that page. ${why}`
-    };
-  }
-
-  if (!title && !text) {
-    if (allowEmpty) return { success: true, status };
-    return {
-      success: false,
-      status,
-      error:
-        `The stealth browser reached ${url} but the document rendered no title and no text` +
-        ` after ${waitedMs}ms of extra wait (${(scraped?.html || '').length} bytes of HTML).` +
-        ' A JavaScript-rendered page needs a longer wait_for; an empty document means the server sent nothing to render.'
-    };
-  }
-
-  if (text.length < SOFT_ERROR_MAX_CHARS && ERROR_TITLE.test(title)) {
-    return {
-      success: false,
-      status,
-      error:
-        `${url} rendered an error page titled "${title}" (${text.length} characters of text) instead of the resource` +
-        ' — a soft block or an application error. Retry later, or with a longer wait_for if the site paints content after a placeholder.'
-    };
-  }
-
-  return { success: true, status };
-}
+export { documentVerdict as stealthDocumentVerdict, SOFT_ERROR_MAX_CHARS } from 'crawlforge-extractors';