crawlforge-mcp-server 5.6.6 → 5.6.8

package/src/server/fallbackHints.js

@@ -0,0 +1,66 @@
+/**
+ * fallbackHints — one sentence appended to every error result naming the
+ * tool (or parameter change) to try next.
+ *
+ * Why: invocation logs (2026-09) show that after a failure the model's most
+ * common move is the same tool with the same params. The hint gives it a
+ * better move than a retry, and the server `instructions` tell it to follow
+ * the hint. Text errors get a trailing "Next step:" line; JSON errors get a
+ * `next_step` field. Applied by withAuth, so no tool handler needs to know.
+ */
+
+export const FALLBACK_HINTS = Object.freeze({
+  fetch_url: 'For a JS-rendered page or an empty shell use scrape; after a 403/429/CAPTCHA/challenge page use stealth_mode operation:"scrape". Do not repeat the same fetch_url call.',
+  extract_text: 'Use scrape formats:["markdown"] (renders more pages); after a 403/429/challenge page use stealth_mode operation:"scrape".',
+  extract_links: 'Use scrape formats:["links"]; for a whole site use map_site.',
+  extract_metadata: 'Use scrape formats:["metadata"]; after a 403/429/challenge page use stealth_mode operation:"scrape".',
+  extract_content: 'Use scrape formats:["markdown"] (same clean output, renders more pages); after a 403/429/challenge page use stealth_mode operation:"scrape".',
+  extract_embedded_state: 'Call again without `path` to see the top-level keys, or use scrape formats:["markdown"] if the page has no framework payload.',
+  scrape_structured: 'Check the selectors against scrape formats:["html"] output, or use extract_structured when the markup varies.',
+  search_web: 'Shorten the query or drop the filters; for Reddit use reddit_search. Fall back to the client\'s built-in search only if CrawlForge is out of credits.',
+  serp_rank: 'configured:false means DataForSEO is not set up - do not retry; approximate visibility with search_web and a site: filter.',
+  reddit_search: 'Add subreddit or author to query the archive directly, or set source:"web_discovery" for a Reddit-wide keyword search; a thread needs mode:"thread" with link_id.',
+  crawl_deep: 'Use map_site for the URL list alone, or batch_scrape when you already have the URLs.',
+  map_site: 'Use crawl_deep with extract_content:false to discover URLs by following links when there is no sitemap.',
+  batch_scrape: 'For an async job poll get_batch_results with the batchId; scrape one of the URLs alone to diagnose a per-URL failure.',
+  get_batch_results: 'An unknown or expired batchId cannot be recovered - run batch_scrape again only if the results are still needed.',
+  process_document: 'For an HTML page use scrape formats:["markdown"]; sourceType:"pdf_url" needs a URL that serves a PDF.',
+  summarize_content: 'Pass the text itself (e.g. markdown from a scrape result), not a URL.',
+  analyze_content: 'Pass the text itself (e.g. markdown from a scrape result), not a URL.',
+  extract_structured: 'Use scrape_structured with known selectors, or extract_with_llm with provider:"openai"/"anthropic" if no local model is available.',
+  extract_with_llm: 'If Ollama is unreachable pass provider:"openai" or "anthropic" with a key, or use extract_structured (CSS fallback needs no LLM).',
+  list_ollama_models: 'Ollama is not reachable - use extract_with_llm with provider:"openai"/"anthropic", or extract_structured.',
+  scrape_with_actions: 'Check the selector against scrape formats:["html"] output; for a one-shot render of a blocked page use stealth_mode operation:"scrape".',
+  deep_research: 'Use agent for a shorter answer, or search_web followed by scrape on the sources that matter.',
+  scrape: 'After a 403/429/CAPTCHA/challenge page or an empty shell use stealth_mode operation:"scrape"; if the content needs a click or login use scrape_with_actions.',
+  agent: 'Use deep_research for exhaustive sourcing, or search_web followed by scrape on the sources that matter.',
+  track_changes: 'compare needs an existing baseline - run operation:"create_baseline" for this URL first; for a one-off read use scrape.',
+  generate_llms_txt: 'Run map_site first to confirm the site is crawlable.',
+  stealth_mode: 'Use scrape_with_actions with browserOptions.stealth:true for a click/scroll/wait chain. Do not retry the same URL with fetch_url or scrape - they are weaker.',
+  localization: 'Use stealth_mode operation:"scrape" after configure_country, or set an Accept-Language header via fetch_url headers.',
+  scrape_template: 'Use template:"list" to see valid template ids, template:"auto" to pick from the URL, or scrape for a site without a template.'
+});
+
+/**
+ * Append the hint for `toolName` to an error result in place. No-op for
+ * success results, unknown tools, non-text content, or a hint already present.
+ */
+export function appendFallbackHint(toolName, result) {
+  const hint = FALLBACK_HINTS[toolName];
+  if (!hint || result?.isError !== true || !Array.isArray(result.content)) return result;
+  const first = result.content[0];
+  if (first?.type !== 'text' || typeof first.text !== 'string') return result;
+  if (first.text.includes(hint)) return result;
+  try {
+    const parsed = JSON.parse(first.text);
+    if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
+      parsed.next_step = hint;
+      first.text = JSON.stringify(parsed, null, 2);
+      return result;
+    }
+  } catch {
+    // plain text
+  }
+  first.text = `${first.text}\nNext step: ${hint}`;
+  return result;
+}

package/src/server/toolFilter.js

@@ -1,7 +1,7 @@
 /**
  * toolFilter — client-side tool selection (Phase 6).
  *
- * Lets an MCP client load a subset of the 27 registered tools via env vars,
+ * Lets an MCP client load a subset of the 29 registered tools via env vars,
  * cutting context bloat (mirrors Bright Data / Exa's TOOLS / GROUPS pattern).
  *
  * Pure module: no I/O, no logging; process.env is only read via

package/src/server/withAuth.js

@@ -9,6 +9,8 @@
  *   - try/finally guarantees a single `tool invocation` log line per call
  *   - log payload: { toolName, paramHash, durationMs, outcome, creditCost, creatorMode }
  *   - outcome ∈ { 'success' | 'error' | 'insufficient_credits' }
+ *   - error results get a "Next step:" hint naming the tool to try next
+ *     (src/server/fallbackHints.js) so a failure is not followed by a blind retry
  *   - emits an OTel span via src/observability/tracing.js (no-op if disabled)
  *   - increments Prometheus counters via src/observability/metrics.js (if registry passed)
  */
@@ -16,6 +18,7 @@
 import { createHash } from 'node:crypto';
 import { recordToolInvocation } from '../observability/tracing.js';
 import { isInternalRequest, preflightRefusal, requestContext } from './requestContext.js';
+import { appendFallbackHint } from './fallbackHints.js';
 
 export function hashParams(params) {
   try {
@@ -124,6 +127,11 @@ export function makeWithAuth({ authManager, logger, metrics = null }) {
           // Cost injection must never break the request path
         }
 
+        // Selection hint: tell the model what to try instead of the same call.
+        if (isErrorResult) {
+          try { appendFallbackHint(toolName, result); } catch { /* never break the request path */ }
+        }
+
         // creditCost === 0 means a genuinely free call (e.g. serp_rank when
         // DataForSEO is unconfigured — a no-op). Emit NO usage event at all so
         // the backend has nothing to (re-)price; reporting 0 would still create

package/src/skills/agent-skills/crawlforge-batch-automation/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-batch-automation
 description: "Automates large scraping jobs and browser interactions with CrawlForge's batch_scrape, get_batch_results, scrape_with_actions, and generate_llms_txt tools. Use when the user wants to scrape many URLs at once, batch-scrape a list of pages, collect dozens of product, news, or competitor pages, run browser actions (click, type, scroll, wait) before scraping, log in or fill a form before extracting, or generate an llms.txt for a site. Use sync mode for up to about 25 URLs and async with a webhook for large batches; retrieve paginated output with get_batch_results."
 metadata:
-  version: 4.8.0
+  version: 5.6.6
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-change-tracking/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-change-tracking
 description: "Monitors web pages for changes over time with CrawlForge's track_changes tool. Use when the user wants to track changes to a page, watch a URL, monitor competitor pricing, detect when content updates, get notified of regulation or product-availability changes, or diff a page against a saved baseline. Workflow: create a baseline with operation create_baseline, then periodically compare with operation compare to get a change percentage and a diff; supports CSS-selector scoping, webhooks, and scheduled monitoring."
 metadata:
-  version: 4.8.0
+  version: 5.6.6
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-deep-research/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-deep-research
 description: "Runs multi-source web research and autonomous question-answering with CrawlForge's deep_research, agent, search_web, and reddit_search tools. Use when the user wants to research a topic, do a deep dive, compare competitors, gather facts with citations, answer a question from the web, search the web, or get a synthesized report from many sources, or search Reddit posts, comments, and threads. deep_research expands queries, fetches and dedupes sources, then synthesizes; agent autonomously plans searches and navigation from a plain-English prompt with no URLs needed; search_web returns ranked results; reddit_search searches Reddit via community archives (reddit.com blocks scrapers). Caps costs via max_urls and confirms before expensive runs."
 metadata:
-  version: 4.8.0
+  version: 5.6.6
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-getting-started/SKILL.md

@@ -1,8 +1,8 @@
 ---
 name: crawlforge-getting-started
-description: "Orientation and tool-selection guide for the CrawlForge MCP server's 28 web tools. Use when the user is getting started with CrawlForge, asks which CrawlForge tool to use, how to set up the API key, how skills or the CLI work, what a tool costs in credits, or when one tool fails and a fallback is needed. Routes requests to the right specialized skill (web scraping, deep research, stealth, structured extraction, change tracking, batch automation), and explains MCP-tools-vs-CLI, the Ollama-first LLM fallback chain, and per-tool credit costs."
+description: "Orientation and tool-selection guide for the CrawlForge MCP server's 29 web tools. Use when the user is getting started with CrawlForge, asks which CrawlForge tool to use, how to set up the API key, how skills or the CLI work, what a tool costs in credits, or when one tool fails and a fallback is needed. Routes requests to the right specialized skill (web scraping, deep research, stealth, structured extraction, change tracking, batch automation), and explains MCP-tools-vs-CLI, the Ollama-first LLM fallback chain, and per-tool credit costs."
 metadata:
-  version: 4.8.0
+  version: 5.6.6
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-stealth-browsing/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-stealth-browsing
 description: "Bypasses bot detection and geo-restrictions with CrawlForge's stealth_mode and localization tools. Use when a site returns 403 or 429, CAPTCHAs, 'please enable JavaScript', or empty content, or is protected by Cloudflare, DataDome, or PerimeterX, or when the user needs region-specific pricing, geo-blocked content, or a specific locale, timezone, or currency. stealth_mode runs a stealth browser (playwright by default, camoufox for advanced fingerprinting) and can screenshot; localization emulates a country and language. Explains when to escalate from a normal scrape to stealth."
 metadata:
-  version: 4.8.0
+  version: 5.6.6
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-structured-extraction/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-structured-extraction
 description: "Extracts structured JSON and analyzes content with CrawlForge's extract_structured, extract_with_llm, extract_embedded_state, scrape_structured, scrape_template, process_document, analyze_content, summarize_content, and list_ollama_models tools. Use when the user wants to extract specific fields, pull data into a JSON schema, extract by natural-language prompt, scrape with CSS selectors, read a page's embedded JavaScript state (__NEXT_DATA__, React Server Components, Nuxt, Apollo, Redux), get product, profile, or repo data from known sites (Amazon, LinkedIn, GitHub, YouTube, Reddit, and more), parse a PDF or DOCX, summarize a page, or analyze sentiment, entities, or keywords. Defaults to local Ollama for LLM extraction; OpenAI and Anthropic optional."
 metadata:
-  version: 4.8.0
+  version: 5.6.6
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-web-scraping/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-web-scraping
 description: "Scrapes web pages and returns clean Markdown, HTML, plain text, links, or metadata using CrawlForge's scrape, fetch_url, extract_content, extract_text, extract_links, extract_metadata, map_site, and crawl_deep tools. Use when the user wants to scrape a URL, fetch a page, get the markdown or text of a website, extract links or metadata, read an article, map a site's URLs, generate a sitemap, or crawl a whole website or documentation site. Prefer the unified scrape tool to get multiple formats in one call; use map_site to discover URLs and crawl_deep to walk an entire site."
 metadata:
-  version: 4.8.0
+  version: 5.6.6
   source: crawlforge-mcp-server
 ---
 
@@ -133,9 +133,11 @@ prompt (elicitation). CLI: `crawlforge crawl https://docs.example.com --depth 3
 
 ## Cost note
 
-Cheapest first: `fetch_url`, `extract_text`, `extract_links`,
-`extract_metadata` = 1 credit · `scrape`, `extract_content`, `map_site` =
-2 · `crawl_deep` = 4 (grows with `max_pages`). Prefer one `scrape` call over
-several single-format calls when you need multiple formats.
+`scrape` (2 credits) is the default page read: one fetch returns every format
+you ask for. `fetch_url`, `extract_text`, `extract_links`, `extract_metadata`
+(1 each) are cheaper only when one of them is the whole job; `fetch_url` followed
+by an `extract_*` call on the same URL is a double fetch that costs more than one
+`scrape`. `extract_content`, `map_site` = 2 · `crawl_deep` = 4 (grows with
+`max_pages`). Never re-fetch a page whose content is already in the conversation.
 
 See [tool reference](references/tool-reference.md) for the full parameter list.

package/src/skills/installer.js

@@ -37,6 +37,10 @@ const __dirname = dirname(__filename);
 // New source of truth: one folder per skill, each containing a SKILL.md.
 const AGENT_SKILLS_DIR = join(__dirname, 'agent-skills');
 
+// The version an installed skill reports. Read from package.json rather than
+// the SKILL.md files, whose metadata.version sat at 4.8.0 through 5.6.x.
+const PKG_VERSION = JSON.parse(readFileSync(join(__dirname, '../../package.json'), 'utf8')).version;
+
 // Pre-4.8.0 bare files that may linger in ~/.claude/skills (migration cleanup).
 const LEGACY_SKILL_FILES = [
   'crawlforge-mcp.md',
@@ -89,8 +93,23 @@ export function concatenateSkills() {
     .join('\n\n---\n\n');
 }
 
+/**
+ * Rewrite metadata.version in a SKILL.md's frontmatter to the package version,
+ * so an installed copy always says which release wrote it.
+ * @param {string} skillMdPath
+ * @param {string} [version]
+ */
+export function stampSkillVersion(skillMdPath, version = PKG_VERSION) {
+  const raw = readFileSync(skillMdPath, 'utf8');
+  const m = raw.match(/^---\n[\s\S]*?\n---/);
+  if (!m) return;
+  const fm = m[0].replace(/(\n\s*version:\s*)[^\n]+/, `$1${version}`);
+  if (fm !== m[0]) writeFileSync(skillMdPath, fm + raw.slice(m[0].length), 'utf8');
+}
+
 function copySkillFolder(srcDir, destDir) {
   cpSync(srcDir, destDir, { recursive: true, force: true });
+  stampSkillVersion(join(destDir, 'SKILL.md'));
 }
 
 /**

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -11,6 +11,7 @@ import { load } from 'cheerio';
 // Import existing tool for content extraction
 import ExtractContentTool from '../extract/extractContent.js';
 import { elementText } from '../../utils/elementText.js';
+import { pageTitle } from '../../utils/pageTitle.js';
 
 // Recording / replay helpers
 import {
@@ -625,7 +626,7 @@ export class ScrapeWithActionsTool extends EventEmitter {
         const state = {
           capturePoint: stateData.afterActionIndex + 1,
           url: stateData.url,
-          title: $('title').text().trim(),
+          title: pageTitle($),
           timestamp: stateData.timestamp,
           content: {}
         };

package/src/tools/advanced/batchScrape/worker.js

@@ -12,6 +12,7 @@ import { noteRetryAfter } from '../../../utils/hostRateLimiter.js';
 import { preflightFetch } from '../../../utils/robotsGate.js';
 import { htmlToMarkdown } from '../../../utils/htmlToMarkdown.js';
 import { elementText } from '../../../utils/elementText.js';
+import { pageTitle } from '../../../utils/pageTitle.js';
 
 /**
  * Fetch a URL with AbortController timeout (SSRF-guarded + per-host throttled).
@@ -133,7 +134,7 @@ export function generateFormats($, html, formats) {
   if (formats.includes('markdown')) content.markdown = buildMarkdown($);
   if (formats.includes('json')) {
     content.json = {
-      title: $('title').text().trim(),
+      title: pageTitle($),
       headings: extractHeadings($),
       links: extractLinks($),
       images: extractImages($),
@@ -144,7 +145,7 @@ export function generateFormats($, html, formats) {
 }
 
 function buildMarkdown($) {
-  const title = $('title').text().trim();
+  const title = pageTitle($);
 
   const selectors = ['article', 'main', '.content', '#content', '.post-content', '.entry-content'];
   let $body = null;
@@ -203,7 +204,7 @@ function extractImages($) {
 
 function extractMetadata($) {
   const m = {};
-  m.title = $('title').text().trim();
+  m.title = pageTitle($);
   m.description = $('meta[name="description"]').attr('content') || '';
   m.og = {};
   $('meta[property^="og:"]').each((_, el) => {

package/src/tools/crawl/mapSite.js

@@ -8,6 +8,7 @@ import { ResultRanker } from '../search/ranking/ResultRanker.js';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 import { CRAWLFORGE_USER_AGENT } from '../../utils/fetchIdentity.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
+import { pageTitle } from '../../utils/pageTitle.js';
 
 // Lazy singleton — avoids creating a CacheManager timer per request
 let _ranker = null;
@@ -280,7 +281,7 @@ export class MapSiteTool {
           const $ = load(html);
           
           metadataMap.set(url, {
-            title: $('title').text().trim(),
+            title: pageTitle($),
             description: $('meta[name="description"]').attr('content') || '',
             keywords: $('meta[name="keywords"]').attr('content') || '',
             h1: $('h1').first().text().trim(),

package/src/tools/extract/_fetchAndParse.js

@@ -11,6 +11,7 @@
  */
 
 import { load } from 'cheerio';
+import { readBody } from 'crawlforge-extractors';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 import { config } from '../../constants/config.js';
 import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
@@ -23,8 +24,15 @@ const DEFAULT_TIMEOUT_MS = 15000;
  * the same cap _fetch.js applies to basic tools, so a large/hostile response
  * (multi-hundred-MB file, endpoint streaming zeros) can't be buffered whole
  * into a JS string and handed to cheerio/JSDOM/Turndown downstream.
- * Content-Length is checked up front; actual bytes read are counted as a
- * backstop for servers that omit or lie about it.
+ *
+ * Decoding goes through crawlforge-extractors' readBody, which honours the
+ * Content-Type charset and the <meta charset> sniff. The local reader it
+ * replaces decoded every body as UTF-8, so scrape, extract_structured,
+ * extract_embedded_state, extract_with_llm and agent all returned mojibake
+ * for Shift_JIS pages (kakaku.com, vector.co.jp, 2ch.sc — R17, 2026-09-04)
+ * while batch_scrape and extract_metadata, already on readBody, read them
+ * correctly. Content-Length is checked up front; actual bytes read are
+ * counted as a backstop for servers that omit or lie about it.
  * @param {Response} response
  * @returns {Promise<string>}
  */
@@ -47,30 +55,7 @@ async function readTextWithSizeCap(response) {
     return response.text();
   }
 
-  const reader = response.body.getReader();
-  const chunks = [];
-  let totalBytes = 0;
-
-  while (true) {
-    const { done, value } = await reader.read();
-    if (done) break;
-    totalBytes += value.byteLength;
-    if (totalBytes > maxBodySize) {
-      reader.cancel();
-      throw new Error(
-        `Response body too large: exceeded limit of ${maxBodySize} bytes`
-      );
-    }
-    chunks.push(value);
-  }
-
-  const mergedBytes = new Uint8Array(totalBytes);
-  let offset = 0;
-  for (const chunk of chunks) {
-    mergedBytes.set(chunk, offset);
-    offset += chunk.byteLength;
-  }
-  return new TextDecoder().decode(mergedBytes);
+  return readBody(response, { maxBytes: maxBodySize });
 }
 
 /**

package/src/tools/extract/extractContent.js

@@ -12,6 +12,8 @@ import { isThinMainContent } from '../scrape/_mainContent.js';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
 import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
+import { readBody } from 'crawlforge-extractors';
+import { config as appConfig } from '../../constants/config.js';
 
 const ExtractContentSchema = z.object({
   url: z.string().url(),
@@ -196,7 +198,9 @@ export class ExtractContentTool {
           throw new Error(`HTTP ${response.status}: ${response.statusText}`);
         }
 
-        html = await response.text();
+        // readBody decodes with the body's real charset; response.text() is
+        // UTF-8-only and mangled Shift_JIS pages (R17, 2026-09-04).
+        html = await readBody(response, { maxBytes: appConfig.fetch.maxBodySize });
         pageTitle = this.extractTitleFromHTML(html);
       }
       }

package/src/tools/extract/extractStructured.js

@@ -56,6 +56,8 @@ function mainContentText($, html, url) {
 
 /** Characters of page text the model is shown; about 6k tokens. */
 const SHOWN_TEXT_BUDGET = 24000;
+/** Characters of the whole-page text kept when main content would crowd it out. */
+const PAGE_HEAD_RESERVE = 6000;
 
 /**
  * The text the model reads: main content first, then the whole page when both
@@ -78,11 +80,19 @@ const SHOWN_TEXT_BUDGET = 24000;
  * @param {string} textContent - whole-page text from fetchAndParse
  * @returns {string}
  */
-function shownText($, html, url, textContent) {
+export function shownText($, html, url, textContent) {
   const main = mainContentText($, html, url);
   if (!main) return textContent;
-  if (main.length + textContent.length > SHOWN_TEXT_BUDGET) return main;
-  return `${main}\n\n${textContent}`;
+  if (main.length + textContent.length <= SHOWN_TEXT_BUDGET) return `${main}\n\n${textContent}`;
+  // Over budget: the article still leads, but the head of the whole-page
+  // text keeps its reserved slice. That head is the site chrome — the nav
+  // that reads "Install (6.3.3)" on swift.org while the main content's table
+  // is titled "Previous Releases" and starts at 6.3.2 — and the link-heavy
+  // lists Readability drops, such as the version list on gradle.org/releases,
+  // where main content alone answered nothing at all (R17, 2026-09-04).
+  const head = textContent.slice(0, PAGE_HEAD_RESERVE);
+  const mainCut = main.slice(0, Math.max(0, SHOWN_TEXT_BUDGET - head.length - 2));
+  return `${mainCut}\n\n${head}`;
 }
 
 /**

package/src/tools/scrape/unifiedScrape.js

@@ -16,6 +16,7 @@ import { extractMainContent, isThinMainContent } from './_mainContent.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
 import { stripHiddenFromDom } from '../../utils/hiddenContent.js';
 import { extractBlockText } from '../basic/extractText.js';
+import { pageTitle } from '../../utils/pageTitle.js';
 
 // ── Schema ────────────────────────────────────────────────────────────────────
 
@@ -132,7 +133,7 @@ function extractMetadataFromDom($, pageUrl) {
 
   const title =
     $('meta[property="og:title"]').attr('content') ||
-    $('title').text().trim() ||
+    pageTitle($) ||
     $('h1').first().text().trim() || '';
 
   const ogTags = {};

package/src/utils/challengeDetection.js

@@ -40,6 +40,17 @@ const CHALLENGES = [
     title: /^access denied/i,
     markers: /errors\.edgesuite\.net/i,
     evidence: 'an Akamai access-denied page'
+  },
+  {
+    // Vercel's Attack Challenge Mode answers with HTTP 429, an
+    // x-vercel-mitigated: challenge header and a JavaScript interstitial
+    // titled "Vercel Security Checkpoint" (lesswrong.com, hashicorp.com,
+    // bombas.com, R17 2026-09-04). Chromium solves it and reloads; camoufox
+    // was left on the interstitial, which then read as a successful scrape.
+    vendor: 'vercel',
+    title: /^vercel security checkpoint/i,
+    markers: /vercel\.link\/security-checkpoint|_vercel\/challenge|x-vercel-challenge-token/i,
+    evidence: 'a Vercel Security Checkpoint page'
   }
 ];
 

package/src/utils/pageTitle.js

@@ -0,0 +1,18 @@
+/**
+ * Document title from a cheerio root.
+ *
+ * A bare `$('title')` also matches inline `<svg><title>` elements in the body,
+ * and cheerio's `.text()` concatenates every match: roc-lang.org came back as
+ * "Roc — a fast, friendly, functional languageGitHubYouTubeTwitter..." in the
+ * R17 sweep (2026-09-04), the same defect extract_metadata fixed for MDN on
+ * 2026-08-26. Read the head title first and fall back to the first `<title>`
+ * anywhere for fragments without a head.
+ *
+ * @param {import('cheerio').CheerioAPI} $
+ * @returns {string} trimmed title or ''
+ */
+export function pageTitle($) {
+  return $('head > title').first().text().trim() || $('title').first().text().trim();
+}
+
+export default pageTitle;