crawlforge-mcp-server 6.22.2 → 6.23.0

package/src/core/browser/guardedProxy.js

@@ -0,0 +1,286 @@
+/**
+ * A local forward proxy that every browser context goes out through, so the
+ * SSRF policy of the fetch path (src/utils/ssrfGuard.js) holds for a browser
+ * at connect time, on every connection a page opens.
+ *
+ * A browser follows redirects inside its own network stack, and Playwright's
+ * route() is called neither for a redirect hop nor for a WebSocket (measured
+ * 2026-10-20 on Chromium and camoufox: a 302 to http://127.0.0.1 and a
+ * `new WebSocket('ws://127.0.0.1')` both reached the listener with a
+ * `context.route('**\/*')` handler installed). The browser also resolves the
+ * host itself, after safeGoto's own check, which is the DNS-rebinding window.
+ * A proxy closes all of it: the browser resolves nothing and connects nowhere
+ * but here, and each CONNECT tunnel or plain-HTTP request is checked with the
+ * same pre-flight (assertUrlAllowed) and the same connect-time lookup
+ * (ssrfLookup) as a fetch, then connected to the address that was checked.
+ * Redirect hops, iframes, subresources, fetch/XHR and WebSockets are all just
+ * connections here.
+ *
+ * Both engines send loopback through a context proxy: Playwright gives
+ * Chromium `<-loopback>`, and camoufox's Firefox hijacks localhost (measured).
+ *
+ * An upstream proxy (a caller's proxyRotation, CRAWLFORGE_STEALTH_PROXIES, a
+ * localization proxy) is chained: the browser talks to this proxy, which keeps
+ * the upstream's credentials and opens the tunnel through it (http, https,
+ * socks4, socks5). The target is still checked here first, but by hostname:
+ * the upstream resolves it in its own network, so the exit's geo-DNS answer is
+ * kept, and a rebinding answer there reaches the upstream's network, not this
+ * one. An upstream's `bypass` list is not honoured; nothing may bypass the
+ * check.
+ *
+ * Listens on 127.0.0.1 only, one server per distinct upstream, for the life of
+ * the process (unref'd).
+ */
+import http from 'node:http';
+import https from 'node:https';
+import net from 'node:net';
+import tls from 'node:tls';
+import { config } from '../../constants/config.js';
+import { assertUrlAllowed, isSsrfError, ssrfLookup } from '../../utils/ssrfGuard.js';
+
+/** upstream key → Promise<{ server: string }> */
+const servers = new Map();
+
+const hostForUrl = (host) => (net.isIPv6(host) ? `[${host}]` : host);
+
+/** `host:port` (CONNECT authority, brackets allowed) → { host, port } */
+function splitAuthority(authority) {
+  const url = new URL(`http://${authority}`);
+  return { host: url.hostname.replace(/^\[|\]$/g, ''), port: Number(url.port) || 80 };
+}
+
+/**
+ * The fetch path's pre-flight on a target (protocol aside: no scheme reaches a
+ * proxy). With an upstream, also what the name resolves to here, since the
+ * connection itself is made by the upstream.
+ */
+function checkTarget(host, port, upstream) {
+  return assertUrlAllowed(`http://${hostForUrl(host)}:${port}/`, { resolveDns: !!upstream });
+}
+
+/**
+ * A direct connection, resolved through the fetch path's connect-time lookup:
+ * ssrfLookup refuses a blocked answer and hands back only what it checked, so
+ * nothing resolves the name a second time. IP literals never reach a lookup;
+ * checkTarget has already held them to the same rules.
+ */
+function directConnect(host, port) {
+  const sec = config.security?.ssrfProtection;
+  const lookup = sec && sec.enabled !== false ? { lookup: ssrfLookup } : {};
+  return connected(net.connect({ host, port, autoSelectFamily: true, ...lookup }));
+}
+
+function onceReadable(socket, bytes) {
+  return new Promise((resolve, reject) => {
+    let buf = Buffer.alloc(0);
+    const onData = (chunk) => {
+      buf = Buffer.concat([buf, chunk]);
+      if (buf.length < bytes) return;
+      cleanup();
+      if (buf.length > bytes) socket.unshift(buf.subarray(bytes));
+      resolve(buf.subarray(0, bytes));
+    };
+    const onError = (err) => { cleanup(); reject(err); };
+    const onEnd = () => { cleanup(); reject(new Error('upstream proxy closed the connection')); };
+    // Paused between reads, so nothing that arrives in between is dropped.
+    const cleanup = () => {
+      socket.pause();
+      socket.off('data', onData);
+      socket.off('error', onError);
+      socket.off('end', onEnd);
+    };
+    socket.on('data', onData);
+    socket.on('error', onError);
+    socket.on('end', onEnd);
+    socket.resume();
+  });
+}
+
+function connected(socket, event = 'connect') {
+  return new Promise((resolve, reject) => {
+    socket.once(event, () => { socket.off('error', reject); resolve(socket); });
+    socket.once('error', reject);
+  });
+}
+
+/** A SOCKS5 (RFC 1928/1929) or SOCKS4a CONNECT through `upstream`. */
+async function socksTunnel(upstream, host, port) {
+  const socket = await connected(net.connect(upstream.port, upstream.host));
+  const portBytes = [port >> 8, port & 0xff];
+  if (upstream.protocol === 'socks4:') {
+    const user = Buffer.from(upstream.username || '');
+    socket.write(Buffer.concat([
+      Buffer.from([4, 1, ...portBytes, 0, 0, 0, 1]), user, Buffer.from([0]), Buffer.from(host), Buffer.from([0])
+    ]));
+    const reply = await onceReadable(socket, 8);
+    if (reply[1] !== 0x5a) throw new Error(`socks4 upstream refused the connection (${reply[1]})`);
+    return socket;
+  }
+  const auth = upstream.username ? 2 : 0;
+  socket.write(Buffer.from([5, 1, auth]));
+  const method = await onceReadable(socket, 2);
+  if (method[1] !== auth) throw new Error('socks5 upstream refused the authentication method');
+  if (auth) {
+    const user = Buffer.from(upstream.username);
+    const pass = Buffer.from(upstream.password || '');
+    socket.write(Buffer.concat([Buffer.from([1, user.length]), user, Buffer.from([pass.length]), pass]));
+    const ok = await onceReadable(socket, 2);
+    if (ok[1] !== 0) throw new Error('socks5 upstream refused the credentials');
+  }
+  const name = Buffer.from(host);
+  socket.write(Buffer.concat([Buffer.from([5, 1, 0, 3, name.length]), name, Buffer.from(portBytes)]));
+  const head = await onceReadable(socket, 5);
+  if (head[1] !== 0) throw new Error(`socks5 upstream refused the connection (${head[1]})`);
+  // The bound address that follows: IPv4, a name of head[4] bytes, or IPv6; then the port.
+  const rest = head[3] === 1 ? 5 : head[3] === 4 ? 17 : head[4] + 2;
+  await onceReadable(socket, rest);
+  return socket;
+}
+
+function proxyAuthorization(upstream) {
+  if (!upstream.username) return {};
+  const pair = `${upstream.username}:${upstream.password || ''}`;
+  return { 'Proxy-Authorization': `Basic ${Buffer.from(pair).toString('base64')}` };
+}
+
+/** A checked raw TCP stream to host:port, through the upstream when there is one. */
+async function openTunnel(upstream, host, port) {
+  await checkTarget(host, port, upstream);
+  if (!upstream) return directConnect(host, port);
+  if (upstream.protocol.startsWith('socks')) return socksTunnel(upstream, host, port);
+  const authority = `${hostForUrl(host)}:${port}`;
+  const request = (upstream.protocol === 'https:' ? https : http).request({
+    host: upstream.host,
+    port: upstream.port,
+    method: 'CONNECT',
+    path: authority,
+    headers: { Host: authority, ...proxyAuthorization(upstream) }
+  });
+  request.end();
+  return new Promise((resolve, reject) => {
+    request.once('error', reject);
+    request.once('connect', (res, socket, head) => {
+      if (res.statusCode !== 200) {
+        socket.destroy();
+        return reject(new Error(`upstream proxy answered CONNECT with ${res.statusCode}`));
+      }
+      if (head.length) socket.unshift(head);
+      resolve(socket);
+    });
+  });
+}
+
+function refusalText(err) {
+  return isSsrfError(err) ? (err.cause?.message || err.message) : 'Bad Gateway';
+}
+
+function handleConnect(upstream) {
+  return async (req, client, head) => {
+    client.on('error', () => {});
+    let remote;
+    try {
+      const { host, port } = splitAuthority(req.url);
+      remote = await openTunnel(upstream, host, port);
+    } catch (err) {
+      const status = isSsrfError(err) ? '403 Forbidden' : '502 Bad Gateway';
+      client.end(`HTTP/1.1 ${status}\r\nContent-Type: text/plain\r\nConnection: close\r\n\r\n${refusalText(err)}`);
+      return;
+    }
+    remote.on('error', () => client.destroy());
+    client.on('close', () => remote.destroy());
+    client.write('HTTP/1.1 200 Connection Established\r\n\r\n');
+    if (head.length) remote.write(head);
+    remote.pipe(client);
+    client.pipe(remote);
+  };
+}
+
+function handleRequest(upstream) {
+  return async (req, res) => {
+    const fail = (status, text) => {
+      if (res.headersSent) return res.destroy();
+      res.writeHead(status, { 'Content-Type': 'text/plain', Connection: 'close' });
+      res.end(text);
+    };
+    let target;
+    let options;
+    try {
+      target = new URL(req.url);
+      if (target.protocol !== 'http:') throw new Error(`protocol ${target.protocol} is not proxied`);
+      const port = Number(target.port) || 80;
+      const host = target.hostname.replace(/^\[|\]$/g, '');
+      const headers = { ...req.headers };
+      delete headers['proxy-connection'];
+      delete headers['proxy-authorization'];
+      options = { method: req.method, headers, path: `${target.pathname}${target.search}` };
+      if (upstream && !upstream.protocol.startsWith('socks')) {
+        // An HTTP proxy takes the absolute URI and resolves it itself.
+        await checkTarget(host, port, upstream);
+        Object.assign(options, {
+          host: upstream.host,
+          port: upstream.port,
+          path: target.href,
+          headers: { ...headers, ...proxyAuthorization(upstream) }
+        });
+        options.agent = false;
+      } else {
+        const socket = await openTunnel(upstream, host, port);
+        options.createConnection = () => socket;
+      }
+    } catch (err) {
+      return fail(isSsrfError(err) ? 403 : 502, refusalText(err));
+    }
+    const lib = upstream?.protocol === 'https:' ? https : http;
+    const out = lib.request(options, (upstreamRes) => {
+      res.writeHead(upstreamRes.statusCode, upstreamRes.statusMessage, upstreamRes.rawHeaders);
+      upstreamRes.pipe(res);
+    });
+    out.on('error', () => fail(502, 'Bad Gateway'));
+    req.pipe(out);
+  };
+}
+
+/** A Playwright proxy option ({server, username?, password?}) → the upstream, or null. */
+function upstreamUrl(proxy) {
+  if (!proxy?.server) return null;
+  const raw = /^[a-z][a-z0-9+.-]*:\/\//i.test(proxy.server) ? proxy.server : `http://${proxy.server}`;
+  const url = new URL(raw);
+  return {
+    protocol: url.protocol,
+    host: url.hostname.replace(/^\[|\]$/g, ''),
+    // URL drops a scheme's default port; SOCKS has none of its own there.
+    port: Number(url.port) || { 'http:': 80, 'https:': 443 }[url.protocol] || 1080,
+    // Credentials ride beside the server in Playwright's shape, or in the URL.
+    username: proxy.username ?? (decodeURIComponent(url.username) || undefined),
+    password: proxy.password ?? (decodeURIComponent(url.password) || undefined)
+  };
+}
+
+/**
+ * The proxy option a browser context is created with: the local guarded
+ * proxy, chained to `upstream` when one is given.
+ * @param {{ server: string, username?: string, password?: string }|null} [upstream]
+ *   the proxy the context would otherwise have used, in Playwright's shape
+ * @returns {Promise<{ server: string }>}
+ */
+export function guardedProxyFor(upstream = null) {
+  const up = upstreamUrl(upstream);
+  const key = up ? JSON.stringify(up) : 'direct';
+  if (!servers.has(key)) {
+    const started = new Promise((resolve, reject) => {
+      const server = http.createServer(handleRequest(up));
+      server.on('connect', handleConnect(up));
+      server.on('clientError', (_err, socket) => socket.destroy());
+      server.once('error', reject);
+      server.listen(0, '127.0.0.1', () => {
+        server.unref();
+        resolve({ server: `http://127.0.0.1:${server.address().port}` });
+      });
+    });
+    started.catch(() => servers.delete(key));
+    servers.set(key, started);
+  }
+  return servers.get(key);
+}
+
+export default guardedProxyFor;

package/src/core/browser/snapshot.js

@@ -511,6 +511,28 @@ export function resolveRef(page, selector) {
   return state.refs.get(id).selector;
 }
 
+/**
+ * Point a ref whose element has left the document at the element that took
+ * its place: the one element with the role and name the snapshot recorded.
+ *
+ * A page can swap an element out without navigating — Wikipedia replaces its
+ * search box after load — and the ref then names a detached node that no
+ * action can reach (live sweep 2026-10-20, 3.2). This is not a guess: an
+ * unnamed ref, or a role and name that match none or several elements, is left
+ * as it was and the action fails as before.
+ * @returns {Promise<boolean>} true when the ref now names the replacement
+ */
+export async function reresolveRef(page, selector) {
+  const entry = isRef(selector) ? pageState.get(page)?.refs?.get(selector.slice(1)) : null;
+  if (!entry?.role || !entry.name) return false;
+  // `s`: the name matches exactly and case-sensitively, not as a substring.
+  const candidate = `role=${entry.role}[name=${JSON.stringify(entry.name)}s]`;
+  const count = await page.locator(candidate).count().catch(() => 0);
+  if (count !== 1) return false;
+  entry.selector = candidate;
+  return true;
+}
+
 /** Register the navigation listener that invalidates this page's refs. Idempotent. */
 export function attachRefTracking(page) {
   const state = stateFor(page);

package/src/core/crawlers/BFSCrawler.js

@@ -7,7 +7,7 @@ import { DomainFilter } from '../../utils/domainFilter.js';
 import { LinkAnalyzer } from '../analysis/LinkAnalyzer.js';
 import { normalizeUrl, extractLinks, isValidUrl } from '../../utils/urlNormalizer.js';
 import { Logger } from '../../utils/Logger.js';
-import { safeFetch } from '../../utils/ssrfGuard.js';
+import { safeFetch, isSsrfError } from '../../utils/ssrfGuard.js';
 import { robotsPreflight, redirectGate, RobotsDisallowedError } from '../../utils/robotsGate.js';
 import { throttleHost } from '../../utils/hostRateLimiter.js';
 import { CRAWLFORGE_USER_AGENT, identityHeaders } from '../../utils/fetchIdentity.js';
@@ -169,11 +169,12 @@ export class BFSCrawler {
 
     // Check domain filter (replaces old pattern checking)
     const filterDecision = this.domainFilter.isAllowed(url, { isSeed });
-    this.filterDecisions.push({
+    const decisionEntry = {
       url: normalizedUrl,
       decision: filterDecision,
       timestamp: new Date().toISOString()
-    });
+    };
+    this.filterDecisions.push(decisionEntry);
     
     if (!filterDecision.allowed) {
       logger.debug(`Domain filter blocks: ${normalizedUrl} - ${filterDecision.reason}`);
@@ -349,6 +350,11 @@ export class BFSCrawler {
         }
       }
     } catch (error) {
+      // The domain filter passed this URL, but the SSRF guard refused it: it
+      // was blocked, not allowed (live sweep 2026-10-20).
+      if (isSsrfError(error)) {
+        decisionEntry.decision = { allowed: false, reason: 'SSRF protection' };
+      }
       this.errors.push({
         url,
         depth,
@@ -437,7 +443,7 @@ export class BFSCrawler {
     const $ = load(html);
     
     // Extract title
-    const title = pageTitle($) || $('h1').first().text().trim() || '';
+    const title = (pageTitle($) || $('h1').first().text().trim() || '').replace(/\s+/g, ' ');
     
     // Extract main content. A page's chrome (header, mega-menu, footer)
     // repeats on every page of a crawl and comes first in body order, so the

package/src/core/processing/BrowserProcessor.js

@@ -11,6 +11,7 @@ import HumanBehaviorSimulator from '../../utils/HumanBehaviorSimulator.js';
 import LocalizationManager from '../LocalizationManager.js';
 import { safeGoto, assertNavigationAllowed } from '../../utils/ssrfGuard.js';
 import { browserUserAgent } from '../../utils/fetchIdentity.js';
+import { guardedProxyFor } from '../browser/guardedProxy.js';
 
 const BrowserProcessorSchema = z.object({
   url: z.string().url(),
@@ -622,9 +623,10 @@ export class BrowserProcessor {
     if (options.geolocation) {
       contextOptions.geolocation = options.geolocation;
     }
-    if (options.proxy) {
-      contextOptions.proxy = options.proxy;
-    }
+    // Every connection the page opens — redirect hops, iframes, WebSockets —
+    // goes through the SSRF-guarded proxy, chained to the caller's proxy when
+    // there is one (src/core/browser/guardedProxy.js).
+    contextOptions.proxy = await guardedProxyFor(options.proxy || null);
 
     const context = await this.browser.newContext(contextOptions);
     const page = await context.newPage();

package/src/core/processing/ContentProcessor.js

@@ -8,7 +8,7 @@ import { JSDOM } from 'jsdom';
 import * as cheerio from 'cheerio';
 import { z } from 'zod';
 import { ContentQualityAssessor } from '../../utils/contentUtils.js';
-import { recoverDroppedTables, unwrapMediaWikiHeadings, tooLargeForReadability } from '../../tools/scrape/_mainContent.js';
+import { recoverDroppedTables, unwrapMediaWikiHeadings, keepListItemBylines, tooLargeForReadability } from '../../tools/scrape/_mainContent.js';
 import { pageTitle } from '../../utils/pageTitle.js';
 import { flattenText } from 'crawlforge-extractors';
 
@@ -76,6 +76,7 @@ export class ContentProcessor {
       // A page too large for Readability keeps no `readability` block (R25).
       if (processingOptions.removeBoilerplate && !tooLargeForReadability(html)) {
         unwrapMediaWikiHeadings(document);
+        keepListItemBylines(document);
         const reader = new Readability(document, {
           debug: false,
           maxElemsToDivide: 300,

package/src/core/research/claimFilters.js

@@ -40,6 +40,21 @@ const FRONT_MATTER_PATTERNS = [
   /\bperformance (?:&|and) security by\b/i
 ];
 
+// Lines that are not statements about the topic, all three key findings in the
+// R25 re-check (2026-10-19, "What anti-bot systems do major websites use in
+// 2026"): a question, which is an FAQ heading ("What makes a good anti-bot
+// solution?"); an opening hook addressed to the reader ("If you've ever
+// rage-clicked on a … puzzle, you already understand the problem with
+// CAPTCHAs."); and a scoring rubric's weight ("Detection technology
+// (signatures, ML, rules, session/event models): 25%"). Its live re-run added
+// "What you are experiencing is the cat-and-mouse game…" and the signpost
+// "Here's how Cloudflare, Akamai, Kasada, and PerimeterX each weigh…".
+const NON_STATEMENT_PATTERNS = [
+  /\?["'”’)]*\s*$/,
+  /^(?:if you(?:['’]ve| have) ever|have you ever|ever (?:wondered|tried|noticed)|imagine|picture this|let['’]s|what you(?:['’]re| are)|here['’]s (?:how|what|why|where))\b/i,
+  /^[^.!?]{2,150}:\s*\d{1,3}(?:\.\d+)?\s*%\.?$/
+];
+
 // Tokens that mark an affiliation line rather than a sentence about the topic.
 const AFFILIATION_TOKENS = new Set([
   'university', 'universities', 'institute', 'institut', 'department', 'dept',
@@ -159,10 +174,14 @@ export function properNounDensity(text) {
 // author or affiliation block rather than a sentence about the topic.
 const PROPER_NOUN_DENSITY_MAX = 0.5;
 
-/** A candidate claim is admissible when it is a verb-bearing, non-front-matter sentence. */
+/**
+ * A candidate claim is admissible when it is a verb-bearing, non-front-matter
+ * statement: not a question, a hook addressed to the reader, or a rubric weight.
+ */
 export function isAdmissibleClaim(text) {
   if (typeof text !== 'string' || text.trim().length === 0) return false;
   if (isFrontMatter(text)) return false;
+  if (NON_STATEMENT_PATTERNS.some(pattern => pattern.test(text.trim()))) return false;
   if (!hasFiniteVerb(text)) return false;
   return properNounDensity(text) <= PROPER_NOUN_DENSITY_MAX;
 }

package/src/core/workers/worker.js

@@ -10,6 +10,7 @@ import { Readability } from '@mozilla/readability';
 import compromise from 'compromise';
 import { franc } from 'franc';
 import { pageTitle } from '../../utils/pageTitle.js';
+import { keepListItemBylines } from '../../tools/scrape/_mainContent.js';
 
 // Task handlers
 const taskHandlers = {
@@ -162,6 +163,7 @@ async function handleExtractContent(data) {
 
   // Use Readability for main content extraction
   if (removeBoilerplate) {
+    keepListItemBylines(document);
     const reader = new Readability(document, {
       debug: false,
       maxElemsToDivide: 300,

package/src/schemas/toolOutputSchemas.js

@@ -314,7 +314,7 @@ const searchWebShape = {
   search_time: z.number().optional(),
   offset: z.number().optional(),
   limit: z.number().optional(),
-  next_offset: z.number().optional().describe('The offset to pass for the next page. Duplicates removed from this page are replaced from further down the provider\'s results, so it can be larger than offset + limit'),
+  next_offset: z.number().optional().describe('The offset to pass for the next page. Duplicates removed from this page are replaced from further down the provider\'s results, so it can be larger than offset + limit. For provider "searxng", offsets count items in SearXNG\'s result list'),
   cached: z.boolean().optional(),
   provider: z.object({
     name: z.string().optional(),
@@ -347,7 +347,8 @@ const searchWebShape = {
 const extractStructuredShape = {
   success: z.boolean().optional().describe('False when the extraction errored, or a required field came back missing, empty, or in the wrong shape'),
   url: z.string().optional(),
-  data: z.record(z.unknown()).optional().describe('Extracted fields matching the requested schema'),
+  data: z.record(z.unknown()).optional().describe('Extracted fields matching the requested schema; every requested field is present, null when not found'),
+  missing_fields: z.array(z.string()).optional().describe('Requested schema fields no extraction method found on the page; each is present in data as null'),
   extraction_method: z.string().optional().describe('"llm" | "css_fallback" | "keyword_fallback" | "none"'),
   provider: z.string().optional().describe('LLM provider that produced the data ("ollama" | "openai" | "anthropic"); present when extraction_method is "llm"'),
   model: z.string().optional().describe('Model that produced the data, e.g. "gemma3:12b"; present when extraction_method is "llm"'),

package/src/server/fallbackHints.js

@@ -88,8 +88,12 @@ export const CLASS_HINTS = Object.freeze({
       : 'This tool has no override; get the content from another source.'),
   host_blocked: () =>
     'This host is on CrawlForge\'s permanent blocklist (a site-owner opt-out). No CrawlForge tool will fetch it and there is no override - get the content from another source.',
+  // Names what the active mode refuses (ssrfGuard.js): local Stage 1 lets private networks through.
   ssrf: () =>
-    'The URL points at a private or internal address (localhost, a private network, cloud metadata), which every CrawlForge tool refuses. Do not retry it with this or any other tool; use a public URL.',
+    (process.env.SSRF_STRICT === 'true'
+      ? 'The URL points at a private or internal address (localhost/loopback, link-local, cloud metadata, or a private network), which every CrawlForge tool refuses. '
+      : 'The URL points at an internal address (localhost/loopback, link-local, or cloud metadata), which every CrawlForge tool refuses; private-network addresses are allowed unless SSRF_STRICT=true is set, which refuses them too. ') +
+    'Do not retry it with this or any other tool; use a public URL.',
   not_found: () =>
     'The URL does not exist (HTTP 404, or the host does not resolve). Check it for typos, or find the right one with map_site on the site or search_web. Retrying it, or switching to stealth_mode, will not help.',
   model_not_pulled: () =>

package/src/server/inlineThreshold.js

@@ -31,16 +31,20 @@ export const MAX_INLINE_CHARS_PARAM = {
 const BROWSER_SESSION_CONTENT_OPERATIONS = new Set(['snapshot', 'act', 'read']);
 
 /**
- * The track_changes operations that return a list or a report; the others
- * return one baseline, one diff or one monitor's id and stay whole.
+ * The track_changes operations that return a list, a report or a diff; the
+ * others return one baseline or one monitor's id and stay whole. compare was
+ * left out until a Hacker News compare with max_inline_chars:4000 came back
+ * as about 15 KB inline (live sweep 4.2, 2026-10-20).
  */
-const TRACK_CHANGES_LIST_OPERATIONS = new Set([
-  'get_history', 'export_history', 'list_scheduled_monitors', 'get_dashboard', 'generate_trend_report'
+const TRACK_CHANGES_SHAPED_OPERATIONS = new Set([
+  'compare', 'get_history', 'export_history', 'list_scheduled_monitors', 'get_dashboard', 'generate_trend_report'
 ]);
 
 export const INLINE_THRESHOLD_TOOLS = Object.freeze({
   scrape: { textPaths: ['content.markdown', 'content.text', 'content.html', 'content.rawHtml'], truncate: true },
-  fetch_url: { textPaths: ['body'], truncate: true },
+  // The headers object is what a status/header check asked for; the scalar
+  // filter dropped it from every truncated result (live sweep 6.9).
+  fetch_url: { textPaths: ['body'], truncate: true, keep: ['headers'] },
   extract_content: { textPaths: ['content.markdown', 'content.text', 'content.html', 'content.cleanedHTML'], truncate: true },
   crawl_deep: { textPaths: [], truncate: true },
   batch_scrape: { textPaths: [], truncate: true },
@@ -84,22 +88,26 @@ export const INLINE_THRESHOLD_TOOLS = Object.freeze({
   scrape_template: { textPaths: [], truncate: true },
   map_site: { textPaths: [], truncate: true, keep: ['site_map', 'statistics'] },
   // A prose answer is the text view; a structured or missing one leaves the
-  // JSON view, and the sources stay inline when they fit.
-  agent: { textPaths: ['answer'], truncate: true, keep: ['evidence', 'search_results', 'provenance'] },
+  // JSON view, and the sources stay inline when they fit. `reason` too: a
+  // degraded run's reason runs past the 200 characters kept unasked, and a
+  // degraded:true with no reason says nothing (R25 follow-up, 2026-10-20).
+  agent: { textPaths: ['answer'], truncate: true, keep: ['reason', 'evidence', 'search_results', 'provenance'] },
   // llms-full.txt is the large file and the view; llms.txt stays inline under
-  // `files` when it fits. Neither key can be named in a json_path (the "."),
-  // so the view is the way to read the long one.
+  // `files` when it fits. A json_path names either with brackets, e.g.
+  // files["llms.txt"] (the key holds a ".").
   generate_llms_txt: {
     textPaths: ['files.llms-full.txt', 'files.llms.txt'],
     truncate: true,
     contentKey: 'files',
     keep: ['analysisStats', 'recommendations']
   },
+  // A compare keeps what says whether and how the page changed; the diff
+  // itself (details) is read with json_path. `operation` defaults to compare.
   track_changes: {
     textPaths: ['export.csv'],
     truncate: true,
-    keep: ['pagination', 'timespan'],
-    when: (params) => TRACK_CHANGES_LIST_OPERATIONS.has(params?.operation)
+    keep: ['pagination', 'timespan', 'summary', 'metrics', 'alerts', 'snapshot'],
+    when: (params) => TRACK_CHANGES_SHAPED_OPERATIONS.has(params?.operation ?? 'compare')
   }
 });
 

package/src/skills/agent-skills/crawlforge-batch-automation/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-batch-automation
 description: "Automates large scraping jobs and browser interactions with CrawlForge's batch_scrape, get_batch_results, scrape_with_actions, and generate_llms_txt tools. Use when the user wants to scrape many URLs at once, batch-scrape a list of pages, collect dozens of product, news, or competitor pages, run browser actions (click, type, scroll, wait) before scraping, log in or fill a form before extracting, or generate an llms.txt for a site. Use sync mode for up to about 25 URLs and async with a webhook for large batches; retrieve paginated output with get_batch_results."
 metadata:
-  version: 6.22.2
+  version: 6.23.0
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-browser-sessions/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-browser-sessions
 description: "Keeps one browser page alive across several tool calls with CrawlForge's browser_session tool. Use when the user needs to log in and then read pages behind that login, work through a multi-step flow (search, filter, paginate, fill a wizard), or look at a page before deciding what to click. The loop is: open a session on a URL, snapshot it to list the interactive elements with stable refs (@e1, @e2), act on those refs, read the content, close. Unlike scrape_with_actions, which is one-shot and closes its browser when the call returns, a session survives between calls, so one login is paid for once and a wrong selector costs one call instead of a whole chain."
 metadata:
-  version: 6.22.2
+  version: 6.23.0
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-change-tracking/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-change-tracking
 description: "Monitors web pages for changes over time with CrawlForge's track_changes tool. Use when the user wants to track changes to a page, watch a URL, monitor competitor pricing, detect when content updates, get notified of regulation or product-availability changes, or diff a page against a saved baseline. Workflow: create a baseline with operation create_baseline, then periodically compare with operation compare to get a change percentage and a diff; supports CSS-selector scoping, webhooks, and scheduled monitoring."
 metadata:
-  version: 6.22.2
+  version: 6.23.0
   source: crawlforge-mcp-server
 ---
 
@@ -49,7 +49,7 @@ Use `trackingOptions` to ignore noise and focus on what matters:
   "tool": "track_changes",
   "params": {
     "url": "https://example.com/pricing",
-    "operation": "compare",
+    "operation": "create_baseline",
     "trackingOptions": {
       "granularity": "element",
       "customSelectors": [".price", ".plan-name"],
@@ -64,6 +64,13 @@ Use `trackingOptions` to ignore noise and focus on what matters:
 `trackText`, `trackStructure`, `trackLinks`, `trackImages`. Set
 `significanceThresholds` (`minor`/`moderate`/`major`) to classify change size.
 
+`granularity`, `customSelectors` and `excludeSelectors` are fixed when the
+baseline is created: every `compare` diffs with the baseline's values, so
+pass them on `create_baseline`. A `compare` that asks for different values
+still runs at the baseline's, and its result carries
+`ignoredOptions: [{option, requested, applied}]` plus a `warnings` entry.
+To change them, create a new baseline.
+
 CLI: `crawlforge track https://example.com --selector ".price" --threshold 1`.
 
 ## Scheduled monitoring & notifications

package/src/skills/agent-skills/crawlforge-deep-research/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-deep-research
 description: "Runs multi-source web research and autonomous question-answering with CrawlForge's deep_research, agent, search_web, and reddit_search tools. Use when the user wants to research a topic, do a deep dive, compare competitors, gather facts with citations, answer a question from the web, search the web, or get a synthesized report from many sources, or search Reddit posts, comments, and threads. deep_research expands queries, fetches and dedupes sources, then synthesizes; agent autonomously plans searches and navigation from a plain-English prompt with no URLs needed; search_web returns ranked results; reddit_search searches Reddit via community archives (reddit.com blocks scrapers). Caps costs via max_urls and confirms before expensive runs."
 metadata:
-  version: 6.22.2
+  version: 6.23.0
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-getting-started/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-getting-started
 description: "Orientation and tool-selection guide for the CrawlForge MCP server's 31 web tools. Use when the user is getting started with CrawlForge, asks which CrawlForge tool to use, how to set up the API key, how skills or the CLI work, what a tool costs in credits, or when one tool fails and a fallback is needed. Routes requests to the right specialized skill (web scraping, deep research, stealth, structured extraction, change tracking, batch automation, browser sessions), and explains MCP-tools-vs-CLI, the Ollama-first LLM fallback chain, and per-tool credit costs."
 metadata:
-  version: 6.22.2
+  version: 6.23.0
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-stealth-browsing/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-stealth-browsing
 description: "Bypasses bot detection with CrawlForge's stealth_mode tool and looks up a country's locale settings with localization. Use when a site returns 403 or 429, CAPTCHAs, 'please enable JavaScript', or empty content, or is protected by Cloudflare, DataDome, or PerimeterX, or when the user needs a page in a specific locale, language, or timezone. stealth_mode runs a stealth browser (engine auto by default: camoufox/Firefox for advanced fingerprinting, playwright/Chromium when pinned for speed) and can screenshot; localization returns a country's Accept-Language, timezone and currency for you to pass to fetch_url or stealth_mode - it applies nothing itself and uses no proxy. Explains when to escalate from a normal scrape to stealth, and why a hard block needs a residential proxy."
 metadata:
-  version: 6.22.2
+  version: 6.23.0
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-structured-extraction/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-structured-extraction
 description: "Extracts structured JSON and analyzes content with CrawlForge's extract_structured, extract_with_llm, extract_embedded_state, scrape_structured, scrape_template, process_document, analyze_content, summarize_content, and list_ollama_models tools. Use when the user wants to extract specific fields, pull data into a JSON schema, extract by natural-language prompt, scrape with CSS selectors, read a page's embedded JavaScript state (__NEXT_DATA__, React Server Components, Nuxt, Apollo, Redux), get product, profile, or repo data from known sites (Amazon, LinkedIn, GitHub, YouTube, Reddit, and more), parse a PDF or DOCX, summarize a page, or analyze sentiment, entities, or keywords. Defaults to local Ollama for LLM extraction; OpenAI and Anthropic optional."
 metadata:
-  version: 6.22.2
+  version: 6.23.0
   source: crawlforge-mcp-server
 ---