crawlforge-mcp-server 6.3.1 → 6.5.0

package/README.md

@@ -48,7 +48,7 @@
 | | **CrawlForge MCP** | Firecrawl | Raw scraping API |
 |---|:---:|:---:|:---:|
 | Native MCP server | ✅ 30 tools | ✅ | ❌ |
-| Free tier | ✅ 1,000 credits, rollover | Limited | Varies |
+| Free tier | ✅ 1,000 credits, one-time, never expire | Limited | Varies |
 | Self-hosted / local LLM extraction (Ollama) | ✅ default, $0/token | ❌ | ❌ |
 | Autonomous agent (no URLs needed) | ✅ `agent` | ✅ | ❌ |
 | Deep research with source verification | ✅ `deep_research` | Partial | ❌ |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.3.1",
+  "version": "6.5.0",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 30 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -115,6 +115,7 @@
     "@modelcontextprotocol/node": "^2.0.0",
     "@modelcontextprotocol/server": "^2.0.0",
     "@mozilla/readability": "^0.6.0",
+    "mammoth": "^1.12.2",
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",

package/server.js

@@ -107,7 +107,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.3.1",
+  version: "6.5.0",
   description: "Production-ready MCP server with 30 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -541,7 +541,7 @@ registerToolIfEnabled("serp_rank", {
 
 // Tool: reddit_search — search Reddit posts/comments or read a full thread (via community archives)
 registerToolIfEnabled("reddit_search", {
-  description: "Use this to search Reddit posts or comments, or read a full comment thread - reddit.com blocks direct scraping, so this reads the Arctic Shift community archive instead (free, no Reddit credentials). Modes: 'posts' (default) and 'comments' search; 'thread' returns a post plus its nested comment tree by link_id. A subreddit/author-scoped search queries the archive directly. A keyword search across ALL of Reddit finds posts with a site-restricted web search and then reads those posts from the archive, because Arctic Shift can only keyword-search within a scope; results come back as real archive rows, ordered by search relevance. An unscoped COMMENT search discovers posts the same way and then searches each post's comments for the keywords. A scoped comment search Arctic Shift times out on is retried over narrower windows (7d, 3d, 1d) and reports window_applied. Not for reddit.com URLs via scrape or fetch_url (blocked) - use mode:'thread' with the post's link_id. Cost: 5 credits. Example: reddit_search({query: \"best mechanical keyboard\", subreddit: \"MechanicalKeyboards\", limit: 10})",
+  description: "Use this to search Reddit posts or comments, or read a full comment thread - reddit.com blocks direct scraping, so this reads the Arctic Shift community archive instead (free, no Reddit credentials). Modes: 'posts' (default) and 'comments' search; 'thread' returns a post plus its nested comment tree by link_id. A subreddit/author-scoped search queries the archive directly. A keyword search across ALL of Reddit finds posts with a site-restricted web search and then reads those posts from the archive, because Arctic Shift can only keyword-search within a scope; results come back as real archive rows, ordered by search relevance. An unscoped COMMENT search discovers posts the same way and then searches each post's comments for the keywords. A scoped comment search Arctic Shift times out on is retried over narrower windows (7d, 3d, 1d) and reports window_applied. Arctic Shift is tried first and the PullPush archive second for posts/comments searches (fallback_used says so; PullPush has refused automated clients since August 2026). Not for reddit.com URLs via scrape or fetch_url (blocked) - use mode:'thread' with the post's link_id. Cost: 5 credits. Example: reddit_search({query: \"best mechanical keyboard\", subreddit: \"MechanicalKeyboards\", limit: 10})",
   annotations: { title: "Reddit Search", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     query: z.string().optional().describe("Keyword search. Posts: matches title+selftext; comments: matches body. Supports \"quoted phrases\", OR, -exclusion"),
@@ -680,7 +680,7 @@ registerToolIfEnabled("extract_content", {
 
 // Tool: process_document
 registerToolIfEnabled("process_document", {
-  description: "Use this to extract text from a PDF URL or file - research papers, contracts, reports. Returns structured sections, metadata, and word count. Not for ordinary web pages (scrape), though an HTML URL is accepted. Cost: 2 credits. Example: process_document({source: \"https://example.com/report.pdf\", sourceType: \"pdf_url\"})",
+  description: "Use this to extract text from a PDF or DOCX URL or file - research papers, contracts, reports. The body decides how it is read: a PDF or Word document served under sourceType \"url\" still reaches its parser, and a body this tool cannot read (an image, an archive) is refused by name. Returns structured sections, metadata, and word count. Not for ordinary web pages (scrape), though an HTML URL is accepted. Cost: 2 credits. Example: process_document({source: \"https://example.com/report.pdf\", sourceType: \"pdf_url\"})",
   annotations: { title: "Process Document", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     source: z.string().describe("Document source - URL or file path"),
@@ -876,7 +876,8 @@ registerToolIfEnabled("get_batch_results", {
   inputSchema: {
     batchId: z.string().describe("The batch ID returned by batch_scrape"),
     page: z.number().min(1).default(1).describe("Page number (1-based)"),
-    pageSize: z.number().min(1).max(100).default(25).describe("Number of results per page")
+    pageSize: z.number().min(1).max(100).default(25).describe("Number of results per page"),
+    ...MAX_INLINE_CHARS_PARAM
   }
 }, withAuth("get_batch_results", async ({ batchId, page = 1, pageSize = 25 }) => {
   try {
@@ -990,11 +991,15 @@ registerToolIfEnabled("scrape_with_actions", {
 
     // Publish captured screenshots as crawlforge://screenshot/{actionId}
     // resources (the documented contract) and annotate each with its URI.
+    // The base64 `data` is dropped once stored, as the stealth_mode block
+    // below already does: a failed chain's error screenshot came back inline
+    // as 1.7 MB of base64 beside a 40-char markdown (R21, 2026-09-09).
     if (Array.isArray(result.screenshots)) {
       result.screenshots = result.screenshots.map((shot) => {
         if (shot?.actionId && shot?.data) {
           resourceRegistry.storeScreenshot(shot.actionId, shot.data);
-          return { ...shot, resourceUri: `crawlforge://screenshot/${shot.actionId}` };
+          const { data, ...rest } = shot;
+          return { ...rest, resourceUri: `crawlforge://screenshot/${shot.actionId}` };
         }
         return shot;
       });

package/src/core/ActionExecutor.js

@@ -314,7 +314,15 @@ export class ActionExecutor extends EventEmitter {
               fullPage: true,
               description: 'Error screenshot'
             });
-            executionContext.screenshots.push(errorScreenshot);
+            // An actionId is what lets the server publish the shot as a
+            // crawlforge://screenshot/{actionId} resource and drop the base64
+            // from the result; without one a failed chain shipped 1.7 MB of
+            // PNG inline (R21, 2026-09-09).
+            executionContext.screenshots.push({
+              ...errorScreenshot,
+              actionId: this.generateActionId(),
+              error: true
+            });
           } catch (screenshotError) {
             this.log('warn', 'Failed to capture error screenshot: ' + screenshotError.message);
           }

package/src/core/AgentOrchestrator.js

@@ -333,6 +333,23 @@ export class AgentOrchestrator {
         quotedTerms.every(t => q.toLowerCase().includes(t.toLowerCase())) ? q : `${q} ${quotedTerms.join(' ')}`.trim()
       );
     }
+    // A current-state plan is told to make its first query the bare entity
+    // name, and a small model often stops there. The bare query surfaces the
+    // live front page, which rarely states the fact asked for: "what does
+    // Southwest charge for a first checked bag" fetched southwest.com's home,
+    // booking and careers pages and answered that the fee is not stated
+    // (R20, 2026-09-07). Keep the entity query first — the domain vote and
+    // live-root promotion depend on it — and add the task's own words as a
+    // second query so the page that states the fact is in the queue too.
+    if (currentState && searchQueries.length === 1) {
+      try {
+        const { compactSearchTopic, clampSearchQuery } = await import('./ResearchOrchestrator.js');
+        const factQuery = clampSearchQuery(compactSearchTopic(prompt));
+        if (factQuery && factQuery.toLowerCase() !== searchQueries[0].toLowerCase()) {
+          searchQueries.push(factQuery);
+        }
+      } catch { /* the entity query alone is the pre-R20 behaviour */ }
+    }
 
     // ── GATHER (search) ───────────────────────────────────────────────────────
     const urlQueue = [...seedUrls]; // start with any user-provided seeds
@@ -389,7 +406,11 @@ export class AgentOrchestrator {
     // and never enters evidence.
     if (currentState && searchResults.length > 0) {
       const originCounts = new Map();
+      // Only the bare entity query votes: the fact query added above returns
+      // guides and news sites, which must not outvote the official domain.
+      const entityQuery = searchQueries[0];
       for (const s of searchResults) {
+        if (s.query !== entityQuery) continue;
         try {
           const origin = new URL(s.url).origin;
           originCounts.set(origin, (originCounts.get(origin) || 0) + 1);
@@ -536,7 +557,7 @@ export class AgentOrchestrator {
         // Short and imperative on purpose: the executing model is a small
         // local one (gemma3:4b-class) and ignores hedged phrasing.
         (currentState
-          ? `- The task asks about the CURRENT state. Answer from the FIRST source below (the live page). NEVER present older or dated content as the current answer.\n`
+          ? `- The task asks about the CURRENT state. Answer from the FIRST source below (the live page); if the FIRST source does not state the answer, take it from the next sources in order. NEVER present older or dated content as the current answer.\n`
           : '') +
         `- Answer ONLY from the provided sources; do not use outside knowledge.\n` +
         `- Read the sources carefully before concluding anything is missing from them.\n` +

package/src/core/ChangeTracker.js

@@ -720,14 +720,16 @@ export class ChangeTracker extends EventEmitter {
     significanceScore += Math.min(totalElements * 0.05, 1) * 
       (weights.additions + weights.removals + weights.modifications);
     
-    // Text changes impact
-    if (changeAnalysis.textChanges.length > 0) {
-      const textChangeRatio = changeAnalysis.textChanges.reduce(
-        (sum, change) => sum + (change.added?.length || 0) + (change.removed?.length || 0),
-        0
-      ) / 1000; // Normalize by character count
-      
-      significanceScore += Math.min(textChangeRatio, 1) * weights.textChanges;
+    // Text changes impact. textChanges holds diff GROUPS — {type:'word_diff',
+    // changes:[{added, removed, value}]} — not flat parts, so reading
+    // `change.added.length` off a group was always 0 and this term never
+    // fired: a feed that grew by a whole record scored on similarity alone.
+    // The USGS all-hour earthquake feed gained an event (622 words, 84%
+    // similar) and compare reported hasChanges:false, "No significant
+    // changes detected" (R21, 2026-09-09).
+    const changedChars = this.changedTextChars(changeAnalysis.textChanges);
+    if (changedChars > 0) {
+      significanceScore += Math.min(changedChars / 1000, 1) * weights.textChanges;
     }
     
     // Determine significance level
@@ -1317,6 +1319,21 @@ export class ChangeTracker extends EventEmitter {
     );
   }
 
+  /**
+   * Characters added or removed at the text level, for significance scoring.
+   * Same word-diff-then-line-diff choice as countTextChanges, and for the
+   * same reason: the two describe one edit.
+   */
+  changedTextChars(textChanges = []) {
+    const diff = textChanges.find(c => c.type === 'word_diff')
+      || textChanges.find(c => c.type === 'line_diff');
+    if (!diff) return 0;
+    return diff.changes.reduce(
+      (chars, part) => chars + ((part.added || part.removed) && typeof part.value === 'string' ? part.value.length : 0),
+      0
+    );
+  }
+
   generateChangeSummary(changeAnalysis, significance) {
     const { addedElements, removedElements, modifiedElements, similarity } = changeAnalysis;
 

package/src/core/crawlers/BFSCrawler.js

@@ -12,6 +12,7 @@ import { robotsPreflight, RobotsDisallowedError } from '../../utils/robotsGate.j
 import { throttleHost } from '../../utils/hostRateLimiter.js';
 import { CRAWLFORGE_USER_AGENT, identityHeaders } from '../../utils/fetchIdentity.js';
 import { pageTitle } from '../../utils/pageTitle.js';
+import { extractMainContent, isThinMainContent } from '../../tools/scrape/_mainContent.js';
 
 const logger = new Logger('BFSCrawler');
 
@@ -391,9 +392,27 @@ export class BFSCrawler {
     // Extract title
     const title = pageTitle($) || $('h1').first().text().trim() || '';
     
-    // Extract main content
+    // Extract main content. A page's chrome (header, mega-menu, footer)
+    // repeats on every page of a crawl and comes first in body order, so the
+    // 500-char preview showed the same "Explore Products" menu for all six
+    // Cessna pages, whose menu is plain <div>s outside any <nav> (R20,
+    // 2026-09-07). Use the same main-content pass scrape uses, fall back to
+    // the body minus its landmark chrome when Readability finds nothing or
+    // only a thin fragment, and to the whole body as a last resort.
     $('script, style, noscript').remove();
-    const content = $('body').text().replace(/\s+/g, ' ').trim();
+    const squash = (text) => text.replace(/\s+/g, ' ').trim();
+    let content = '';
+    try {
+      const main = extractMainContent(html, url);
+      if (main.html && !isThinMainContent(main.html, html)) {
+        content = squash(load(main.html)('body').text());
+      }
+    } catch { /* fall through to the body */ }
+    if (!content) {
+      const $body = $('body').clone();
+      $body.find('header, nav, footer, aside, [role="navigation"], [role="banner"], [role="contentinfo"]').remove();
+      content = squash($body.text()) || squash($('body').text());
+    }
     
     // Extract metadata
     const metadata = {

package/src/core/llm/LLMManager.js

@@ -27,6 +27,70 @@ function endsMidJson(text) {
   return !trimmed.endsWith('}') && !trimmed.endsWith(']');
 }
 
+/**
+ * Recover the complete rows of a JSON response the model stopped writing.
+ *
+ * Walks the text tracking strings and the bracket stack, remembers the end of
+ * every complete element written directly inside an array, cuts there and
+ * closes what is still open. Returns null when no element completed, so a
+ * caller can still fall back. A schema asking for a table of rows routinely
+ * overruns a small model's output budget — the ECB key-rates table came back
+ * cut off at 1,800 and again at 3,600 tokens and the whole extraction failed,
+ * although dozens of rows were complete (R21, 2026-09-09).
+ *
+ * @param {string} text
+ * @returns {{ data: unknown, rows: number }|null}
+ */
+export function salvageTruncatedJson(text) {
+  const src = text.trim();
+  if (!src.startsWith('{') && !src.startsWith('[')) return null;
+  const stack = [];
+  let inString = false;
+  let cut = -1;
+  for (let i = 0; i < src.length; i++) {
+    const ch = src[i];
+    if (inString) {
+      if (ch === '\\') i++;
+      else if (ch === '"') {
+        inString = false;
+        // A string written directly into an array is a complete element.
+        if (stack[stack.length - 1] === '[') cut = i + 1;
+      }
+      continue;
+    }
+    if (ch === '"') inString = true;
+    else if (ch === '{' || ch === '[') stack.push(ch);
+    else if (ch === '}' || ch === ']') {
+      stack.pop();
+      if (stack[stack.length - 1] === '[') cut = i + 1;
+    }
+  }
+  if (cut < 0) return null;
+  // Close everything still open at the cut, innermost first.
+  const open = [];
+  inString = false;
+  for (let i = 0; i < cut; i++) {
+    const ch = src[i];
+    if (inString) { if (ch === '\\') i++; else if (ch === '"') inString = false; continue; }
+    if (ch === '"') inString = true;
+    else if (ch === '{' || ch === '[') open.push(ch);
+    else if (ch === '}' || ch === ']') open.pop();
+  }
+  const closers = open.reverse().map((c) => (c === '{' ? '}' : ']')).join('');
+  let data;
+  try {
+    data = JSON.parse(src.slice(0, cut) + closers);
+  } catch {
+    return null;
+  }
+  let rows = 0;
+  (function count(v) {
+    if (Array.isArray(v)) { rows += v.length; v.forEach(count); }
+    else if (v && typeof v === 'object') Object.values(v).forEach(count);
+  })(data);
+  return rows > 0 ? { data, rows } : null;
+}
+
 /**
  * LLM Manager
  * Manages multiple LLM providers and provides unified interface
@@ -987,12 +1051,25 @@ Extract the data and return valid JSON:`;
         try {
           parsed = JSON.parse(cleaned);
         } catch (parseError) {
+          if (!endsMidJson(cleaned)) throw parseError;
           // Name the cause when the JSON simply stops: "unexpected end of
           // input at position 2608" tells a caller nothing they can act on,
           // whereas "cut off at the 1000-token limit" points at the schema.
-          throw endsMidJson(cleaned)
-            ? new Error(`model response was cut off at the ${budget}-token output limit (${cleaned.length} chars) — the schema asks for more rows than fit`)
-            : parseError;
+          const cutOff = `model response was cut off at the ${budget}-token output limit (${cleaned.length} chars) — the schema asks for more rows than fit`;
+          // The doubled budget was the retry; when that is cut off too, the
+          // rows it did complete are worth more than a fallback that has none.
+          const salvaged = attempt > 0 ? salvageTruncatedJson(cleaned) : null;
+          if (!salvaged) throw new Error(cutOff);
+          const validation = this.validateAgainstSchema(salvaged.data, schema);
+          this.logger.warn('LLM structured extraction salvaged a cut-off response', { budget, rows: salvaged.rows });
+          return {
+            data: salvaged.data,
+            method: 'llm',
+            valid: validation.valid,
+            validationErrors: validation.errors,
+            partial: true,
+            warning: `${cutOff}; kept the ${salvaged.rows} complete row(s) it had written. Ask for fewer rows (a prompt naming the rows you need, or a narrower schema) to get the rest.`
+          };
         }
 
         const validation = this.validateAgainstSchema(parsed, schema);
@@ -1016,7 +1093,7 @@ Extract the data and return valid JSON:`;
     // Report which path produced the data. Callers previously labelled this
     // result "llm", so a failed LLM call was returned as a high-confidence
     // LLM extraction.
-    return { ...this.fallbackStructuredExtraction(content, schema), error: lastError.message };
+    return { ...this.fallbackStructuredExtraction(content, schema, lastError.message), error: lastError.message };
   }
 
   /**
@@ -1036,7 +1113,7 @@ Extract the data and return valid JSON:`;
   /**
    * Fallback structured extraction without LLM — keyword/regex matching for primitives
    */
-  fallbackStructuredExtraction(content, schema) {
+  fallbackStructuredExtraction(content, schema, reason = 'no LLM provider available') {
     const extracted = {};
     const properties = schema.properties || {};
 
@@ -1065,7 +1142,10 @@ Extract the data and return valid JSON:`;
       data: extracted,
       method: 'keyword_fallback',
       valid: false,
-      validationErrors: ['Used fallback extraction — no LLM provider available']
+      // Name the real reason: this fallback also runs after an LLM attempt
+      // that failed, and "no LLM provider available" beside a working Ollama
+      // sent a caller looking at the wrong thing (R21, 2026-09-09).
+      validationErrors: [`Used fallback extraction — ${reason}`]
     };
   }
 

package/src/core/processing/PDFProcessor.js

@@ -12,7 +12,9 @@ import { config } from '../../constants/config.js';
 import { identityHeaders } from '../../utils/fetchIdentity.js';
 
 const PDFProcessorSchema = z.object({
-  source: z.string().min(1),
+  // A Buffer is the 'buffer' source: process_document hands over a body it
+  // has already fetched and sniffed as a PDF (R21, 2026-09-09).
+  source: z.union([z.string().min(1), z.instanceof(Buffer)]),
   sourceType: z.enum(['url', 'file', 'buffer']).default('url'),
   options: z.object({
     extractMetadata: z.boolean().default(true),

package/src/server/fallbackHints.js

@@ -80,6 +80,9 @@ export function appendFallbackHint(toolName, result) {
     // plain text
   }
 
+  // An error that already names its next step — a reddit.com refusal points
+  // at reddit_search — keeps it: the tool's generic hint would contradict it.
+  if (parsed?.next_step || /\bNext step:/.test(first.text)) return result;
   const hint = hintFor(toolName, parsed);
   if (!hint || first.text.includes(hint)) return result;
   if (parsed) {

package/src/server/inlineThreshold.js

@@ -30,6 +30,9 @@ export const INLINE_THRESHOLD_TOOLS = Object.freeze({
   extract_content: { textPaths: ['content.markdown', 'content.text', 'content.html', 'content.cleanedHTML'], truncate: true },
   crawl_deep: { textPaths: [], truncate: true },
   batch_scrape: { textPaths: [], truncate: true },
+  // A page of 25 markdown results is the same payload batch_scrape shapes;
+  // an async job's page came back as 111 KB whole (R20, 2026-09-07).
+  get_batch_results: { textPaths: [], truncate: true },
   stealth_mode: { textPaths: ['content.markdown', 'content.text', 'content.html'], truncate: true, when: (params) => params?.operation === 'scrape' },
   scrape_with_actions: { textPaths: ['content.markdown', 'content.text', 'content.html'], truncate: true },
   process_document: { textPaths: ['content.text'], truncate: true },
@@ -69,6 +72,27 @@ export function resultTextView(resultObject, textPaths = []) {
   return { view: 'json', view_path: null, text: JSON.stringify(resultObject, null, 2) };
 }
 
+/**
+ * The non-text fields of `content` small enough to stay inline: everything
+ * except the text views (markdown, text, html, …) up to a quarter of the
+ * inline budget each. Null when there is nothing to keep.
+ */
+export function keepSmallContentFields(content, textPaths = [], maxInline = DEFAULT_MAX_INLINE_CHARS) {
+  if (!content || typeof content !== 'object' || Array.isArray(content)) return null;
+  const textLeaves = new Set(
+    textPaths.filter((p) => p.startsWith('content.')).map((p) => p.slice('content.'.length))
+  );
+  const cap = Math.max(1000, Math.floor(maxInline / 4));
+  const kept = {};
+  for (const [key, value] of Object.entries(content)) {
+    if (textLeaves.has(key) || value === undefined) continue;
+    if (typeof value === 'string' && value.length > cap) continue;
+    if (typeof value === 'object' && value !== null && JSON.stringify(value).length > cap) continue;
+    kept[key] = value;
+  }
+  return Object.keys(kept).length > 0 ? kept : null;
+}
+
 function warningsOf(resultObject) {
   return Array.isArray(resultObject.warnings) ? resultObject.warnings.filter((w) => typeof w === 'string') : [];
 }
@@ -126,7 +150,9 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
   }
 
   const preview = text.slice(0, maxInline);
-  const hint = `Result is ${json.length} chars as JSON, over the inline limit of ${maxInline}; preview holds the first ${preview.length} chars of ${viewDesc} (${text.length} chars in total) and the full result is kept for 1 hour under result_handle ${handle}: ${readWith}.`;
+  const keptFields = keepSmallContentFields(resultObject.content, config.textPaths, maxInline);
+  const keptDesc = keptFields ? `; content.${Object.keys(keptFields).join(', content.')} kept inline` : '';
+  const hint = `Result is ${json.length} chars as JSON, over the inline limit of ${maxInline}; preview holds the first ${preview.length} chars of ${viewDesc} (${text.length} chars in total)${keptDesc} and the full result is kept for 1 hour under result_handle ${handle}: ${readWith}.`;
 
   const shaped = {};
   for (const [key, value] of Object.entries(resultObject)) {
@@ -140,6 +166,13 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
   if (resultObject.redaction && typeof resultObject.redaction === 'object') {
     shaped.redaction = resultObject.redaction;
   }
+  // The query-scoped formats live beside the page text under `content`
+  // (highlights, answer, json, metadata, links). They are the small, exact
+  // answer the caller paid for, and truncating the markdown must not drop
+  // them: an nhs.uk scrape with highlights and a question came back as a
+  // markdown preview and nothing else (R21, 2026-09-09). Keep every
+  // non-text `content` field that fits a quarter of the inline budget.
+  if (keptFields) shaped.content = keptFields;
   Object.assign(shaped, {
     preview,
     result_handle: handle,

package/src/server/requestContext.js

@@ -36,7 +36,7 @@ export function isInternalRequest() {
  * reaches withAuth. It also survives both routes a refusal can take — thrown,
  * or swallowed into an isError result.
  *
- * @param {string} code 'ROBOTS_DISALLOWED' | 'HOST_BLOCKED'
+ * @param {string} code 'ROBOTS_DISALLOWED' | 'HOST_BLOCKED' | 'USE_REDDIT_SEARCH'
  */
 export function markPreflightRefusal(code) {
   const store = requestContext.getStore();

package/src/skills/agent-skills/crawlforge-batch-automation/SKILL.md

@@ -73,6 +73,8 @@ completed or in-progress job. Cheap (1 credit) because the batch was already
 paid for. Completed jobs are also exposed as `crawlforge://job/{jobId}`
 resources. Stored batch results share the local 1-hour result store that
 `read_result` reads, with the same eviction, so page through a batch within the hour.
+Like `batch_scrape`, it takes `max_inline_chars` (default 40,000): a page over the
+limit comes back as a `preview` plus a `result_handle` for `read_result`.
 
 ## scrape_with_actions — interact, then scrape (cost: 5)
 

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -13,6 +13,7 @@ import ExtractContentTool from '../extract/extractContent.js';
 import { stealthDocumentVerdict } from '../../utils/stealthVerdict.js';
 import { elementText } from '../../utils/elementText.js';
 import { pageTitle } from '../../utils/pageTitle.js';
+import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
 
 // Recording / replay helpers
 import {
@@ -726,6 +727,26 @@ export class ScrapeWithActionsTool extends EventEmitter {
         }
       }
 
+      // extractContent hands back text alone when Readability finds no
+      // article. support.southwest.com's help centre is a JS shell: after an
+      // 8 s wait the body text was there, `html` came back empty and
+      // `markdown` was the "Content not available" placeholder, reported as
+      // success (R20, 2026-09-07). The post-action DOM is in hand, so serve
+      // the requested formats from it instead of a placeholder.
+      if (chainResult?.finalHtml) {
+        extractResult.content = extractResult.content || {};
+        if (params.formats?.includes('markdown') && !extractResult.content.markdown) {
+          const bodyMarkdown = htmlToMarkdown(chainResult.finalHtml);
+          if (bodyMarkdown) {
+            extractResult.content.markdown = bodyMarkdown;
+            extractResult.content.markdownSource = 'body';
+          }
+        }
+        if (params.formats?.includes('html') && !extractResult.content.html) {
+          extractResult.content.html = chainResult.finalHtml;
+        }
+      }
+
       return extractResult;
     } catch (error) {
       this.log('warn', `Final content extraction failed: ${error.message}`);

package/src/tools/basic/extractLinks.js

@@ -42,7 +42,9 @@ export async function extractLinksHandler({ url, filter_external, base_url, user
       const href = $(element).attr('href');
       const text = $(element).text().trim();
 
-      if (!href) return;
+      // A javascript: pseudo-link ("Cookie Settings") is a button, not a
+      // link; it was counted as an external link on boeing.com (R20).
+      if (!href || /^\s*javascript:/i.test(href)) return;
 
       try {
         const absoluteUrl = new URL(href, baseUrl).toString();