crawlforge-mcp-server 5.6.11 → 5.9.0

package/src/schemas/toolOutputSchemas.js

@@ -15,7 +15,7 @@
  */
 
 import { z } from 'zod';
-import { SCRAPE_STRING_FORMATS } from '../tools/scrape/formats.js';
+import { SCRAPE_STRING_FORMATS, SCRAPE_OBJECT_FORMATS } from '../tools/scrape/formats.js';
 
 // `_cost` is injected by withAuth into the legacy JSON-text copy of the
 // result (never into structuredContent directly) — included here anyway so
@@ -28,6 +28,20 @@ const costShape = z.object({
   projection_note: z.string().optional().describe('Human-readable note about how the cost was projected')
 }).passthrough().optional().describe('Cost-transparency metadata (D3.5), present when injected into the text copy of the result');
 
+// ── result handles (Phase 2) ────────────────────────────────────────────────
+
+// A result over max_inline_chars comes back as a preview plus a handle for
+// read_result; these fields describe that shape wherever it can appear.
+const resultHandleShape = {
+  preview: z.string().optional().describe('The first max_inline_chars characters of the view named by view_path (or of the pretty-printed JSON)'),
+  result_handle: z.string().optional().describe('Handle for read_result; the full result is kept 1 hour'),
+  total_chars: z.number().optional().describe('Length of the full view in characters'),
+  view: z.enum(['text', 'json']).optional().describe('Whether preview and read_result offsets index a text field or the pretty-printed JSON'),
+  view_path: z.string().nullable().optional().describe('Dotted path of the text field the view was cut from; null for the JSON view'),
+  truncated: z.boolean().optional().describe('True when the inline result is a preview'),
+  expires_at: z.string().optional().describe('When the stored result is dropped (ISO 8601)')
+};
+
 // ── scrape ──────────────────────────────────────────────────────────────────
 
 const scrapeLinkShape = z.object({
@@ -59,11 +73,23 @@ const scrapeMetadataShape = z.object({
   url: z.string().optional()
 }).passthrough();
 
+// A sentence, table row or fenced code block returned verbatim by the
+// highlights and question formats; `offset`/`length` index the `markdown`
+// format of the same call.
+const scrapeHighlightUnitShape = z.object({
+  text: z.string().optional().describe('Verbatim page text: markdown.slice(offset, offset + length) === text'),
+  kind: z.enum(['sentence', 'table_row', 'code_block']).optional(),
+  offset: z.number().optional().describe('JS string index into the markdown format of this call'),
+  length: z.number().optional(),
+  score: z.number().optional().describe('BM25 relevance to the query, higher is better')
+}).passthrough();
+
 // One content key per string format `scrape` accepts (the "screenshot"
-// format fills `screenshots`), plus `json` for the {type:"json"} object
-// format. Checked against the format list the input schema is built from,
-// so a format added to formats.js without a shape here fails at load
-// instead of shipping a schema that does not describe the result (0.3).
+// format fills `screenshots`), plus one per object format: `json` for
+// {type:"json"}, `highlights` for {type:"highlights"}, `answer` for
+// {type:"question"}. Checked against the format lists the input schema is
+// built from, so a format added to formats.js without a shape here fails at
+// load instead of shipping a schema that does not describe the result (0.3).
 const scrapeFormatShapes = {
   markdown: z.string().optional(),
   html: z.string().optional(),
@@ -72,10 +98,17 @@ const scrapeFormatShapes = {
   links: scrapeLinksShape.optional(),
   metadata: scrapeMetadataShape.optional(),
   branding: z.record(z.unknown()).optional().describe('Static design tokens: colors, fonts, logo'),
-  screenshots: z.array(z.object({}).passthrough()).optional().describe('Present for the "screenshot" format; each item carries a resourceUri once published')
+  screenshots: z.array(z.object({}).passthrough()).optional().describe('Present for the "screenshot" format; each item carries a resourceUri once published'),
+  json: z.unknown().optional().describe('Result of the {type:"json"} format (LLM-structured extraction)'),
+  highlights: z.array(scrapeHighlightUnitShape).optional().describe('Result of the {type:"highlights"} format: the units matching the query, best first, verbatim with offsets'),
+  answer: z.object({
+    text: z.string().optional().describe('Extractive mode: the evidence texts joined; model mode: the model\'s answer'),
+    grounded: z.boolean().optional().describe('True when every number and proper noun in text appears in the evidence or the question; always true in extractive mode'),
+    evidence: z.array(scrapeHighlightUnitShape).optional().describe('The units the answer rests on, verbatim with offsets')
+  }).passthrough().optional().describe('Result of the {type:"question"} format')
 };
-const scrapeContentKey = (format) => (format === 'screenshot' ? 'screenshots' : format);
-for (const format of SCRAPE_STRING_FORMATS) {
+const scrapeContentKey = (format) => ({ screenshot: 'screenshots', question: 'answer' }[format] ?? format);
+for (const format of [...SCRAPE_STRING_FORMATS, ...SCRAPE_OBJECT_FORMATS]) {
   if (!scrapeFormatShapes[scrapeContentKey(format)]) {
     throw new Error(`toolOutputSchemas: scrape format "${format}" has no output shape`);
   }
@@ -91,11 +124,14 @@ const scrapeShape = {
     vendor: z.string().optional(),
     evidence: z.string().optional()
   }).passthrough().optional().describe('Present when a bot-defence vendor served a challenge page; the fallback hint names the tool to try next'),
-  content: z.object({
-    ...scrapeFormatShapes,
-    json: z.unknown().optional().describe('Result of the {type:"json"} format (LLM-structured extraction)')
-  }).passthrough().optional().describe('One key per requested format'),
+  escalated: z.boolean().optional().describe('Present only when escalate:true was passed: whether the blocked plain fetch was retried in the stealth browser. False means the plain fetch sufficed and the call is charged at the base price'),
+  stealth: z.object({
+    engine: z.string().optional(),
+    vendor_detected: z.string().nullable().optional()
+  }).passthrough().optional().describe('Present when escalated is true: the stealth engine that ran, and the bot-defence vendor the plain fetch hit (null when the block named none)'),
+  content: z.object(scrapeFormatShapes).passthrough().optional().describe('One key per requested format'),
   warnings: z.array(z.string()).optional().describe('Per-format warnings; partial success never fails the whole call'),
+  ...resultHandleShape,
   _cost: costShape
 };
 
@@ -344,6 +380,45 @@ const crawlDeepShape = {
   }).passthrough().optional(),
   crawled_at: z.string().optional().describe('When the pages were actually fetched (ISO 8601)'),
   cached: z.boolean().optional().describe('True when this response was replayed from an earlier crawl rather than crawled now; crawled_at gives its age'),
+  ...resultHandleShape,
+  _cost: costShape
+};
+
+// ── read_result ───────────────────────────────────────────────────────────────
+
+const readResultMatchShape = z.object({
+  offset: z.number().optional().describe('Index of the match in the view'),
+  length: z.number().optional(),
+  context_offset: z.number().optional().describe('Index of the first character of context in the view'),
+  context: z.string().optional().describe('Verbatim text around the match: view.slice(context_offset, context_offset + context.length)')
+}).passthrough();
+
+const readResultShape = {
+  handle: z.string().optional(),
+  tool: z.string().optional().describe('The tool that produced the stored result'),
+  operation: z.enum(['slice', 'search', 'lines', 'json_path']).optional(),
+  view: z.enum(['text', 'json']).optional(),
+  view_path: z.string().nullable().optional(),
+  total_chars: z.number().optional().describe('Length of the full view'),
+  expires_at: z.string().optional(),
+  offset: z.number().optional().describe('slice: first character returned'),
+  length: z.number().optional().describe('slice: characters returned'),
+  text: z.string().optional().describe('slice: verbatim view.slice(offset, offset + length)'),
+  has_more: z.boolean().optional().describe('slice/lines: more follows the returned range'),
+  query: z.string().optional(),
+  matches: z.array(readResultMatchShape).optional().describe('search: matches with 200 chars of context each side'),
+  total_matches: z.number().optional(),
+  first_line: z.number().optional(),
+  line_count: z.number().optional(),
+  total_lines: z.number().optional(),
+  char_offset: z.number().optional().describe('lines: view offset of the first returned line'),
+  lines: z.array(z.string()).optional(),
+  path: z.string().optional(),
+  value: z.unknown().optional().describe('json_path: the subtree; null with a preview when it is over max_inline_chars'),
+  value_chars: z.number().optional(),
+  preview: z.string().optional(),
+  truncated: z.boolean().optional().describe('search: more matches than returned; json_path: value replaced by a preview'),
+  warnings: z.array(z.string()).optional(),
   _cost: costShape
 };
 
@@ -356,7 +431,8 @@ export const OUTPUT_SCHEMAS = {
   reddit_search: redditSearchShape,
   search_web: searchWebShape,
   extract_structured: extractStructuredShape,
-  crawl_deep: crawlDeepShape
+  crawl_deep: crawlDeepShape,
+  read_result: readResultShape
 };
 
 export default OUTPUT_SCHEMAS;

package/src/server/fallbackHints.js

@@ -24,6 +24,7 @@ export const FALLBACK_HINTS = Object.freeze({
   map_site: 'Use crawl_deep with extract_content:false to discover URLs by following links when there is no sitemap.',
   batch_scrape: 'For an async job poll get_batch_results with the batchId; scrape one of the URLs alone to diagnose a per-URL failure.',
   get_batch_results: 'An unknown or expired batchId cannot be recovered - run batch_scrape again only if the results are still needed.',
+  read_result: 'An unknown or expired handle cannot be recovered - results are kept 1 hour; run the original tool again only if the result is still needed.',
   process_document: 'For an HTML page use scrape formats:["markdown"]; sourceType:"pdf_url" needs a URL that serves a PDF.',
   summarize_content: 'Pass the text itself (e.g. markdown from a scrape result), not a URL.',
   analyze_content: 'Pass the text itself (e.g. markdown from a scrape result), not a URL.',
@@ -41,26 +42,51 @@ export const FALLBACK_HINTS = Object.freeze({
   scrape_template: 'Use template:"list" to see valid template ids, template:"auto" to pick from the URL, or scrape for a site without a template.'
 });
 
+/**
+ * The second-stage hint (Phase 3, 3.4). A `scrape` that came back
+ * `escalated: true` has ALREADY run the stealth browser on this URL, so the
+ * ordinary hint would send the model back to a tool that has just failed.
+ * What is left is a region-specific block, which `localization` can change,
+ * or a block that is final: an Akamai-style TLS-level wall is beaten with
+ * residential proxies, which CrawlForge does not offer.
+ */
+export const SCRAPE_ESCALATED_HINT =
+  'The stealth browser has already run on this URL (escalate:true) - do not call stealth_mode or repeat this call. ' +
+  'If the block is regional, set a country with localization and try once more; otherwise the block is final - ' +
+  'a TLS-level wall needs residential proxies, which CrawlForge does not offer. Get the content from another source.';
+
+/** The hint for this result: the tool's, unless a scrape already escalated. */
+function hintFor(toolName, parsed) {
+  if (toolName === 'scrape' && parsed?.escalated === true) return SCRAPE_ESCALATED_HINT;
+  return FALLBACK_HINTS[toolName];
+}
+
 /**
  * Append the hint for `toolName` to an error result in place. No-op for
  * success results, unknown tools, non-text content, or a hint already present.
  */
 export function appendFallbackHint(toolName, result) {
-  const hint = FALLBACK_HINTS[toolName];
-  if (!hint || result?.isError !== true || !Array.isArray(result.content)) return result;
+  if (result?.isError !== true || !Array.isArray(result.content)) return result;
   const first = result.content[0];
   if (first?.type !== 'text' || typeof first.text !== 'string') return result;
-  if (first.text.includes(hint)) return result;
+
+  // Parsed first: which hint a `scrape` failure gets depends on whether the
+  // body says the escalation already ran.
+  let parsed = null;
   try {
-    const parsed = JSON.parse(first.text);
-    if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
-      parsed.next_step = hint;
-      first.text = JSON.stringify(parsed, null, 2);
-      return result;
-    }
+    const candidate = JSON.parse(first.text);
+    if (candidate && typeof candidate === 'object' && !Array.isArray(candidate)) parsed = candidate;
   } catch {
     // plain text
   }
+
+  const hint = hintFor(toolName, parsed);
+  if (!hint || first.text.includes(hint)) return result;
+  if (parsed) {
+    parsed.next_step = hint;
+    first.text = JSON.stringify(parsed, null, 2);
+    return result;
+  }
   first.text = `${first.text}\nNext step: ${hint}`;
   return result;
 }

package/src/server/inlineThreshold.js

@@ -0,0 +1,149 @@
+/**
+ * inlineThreshold — the one declaration (G5) of `max_inline_chars` and of
+ * the rule that turns an oversized tool result into a preview plus a
+ * result_handle for read_result (Phase 2).
+ *
+ * Applied by withAuth after a successful handler run and before `_cost`
+ * injection, for the tools listed in INLINE_THRESHOLD_TOOLS only. Internal
+ * (website REST proxy) requests are never shaped here — the website applies
+ * its own threshold with its own store.
+ */
+
+import { z } from 'zod';
+
+export const DEFAULT_MAX_INLINE_CHARS = 40000;
+
+export const MAX_INLINE_CHARS_PARAM = {
+  max_inline_chars: z.number().int().min(1000).max(10_000_000).optional().describe("Largest result to return inline, in characters of its JSON. Over it, the call returns a preview plus a result_handle for read_result instead of the whole result (default 40,000; env CRAWLFORGE_MAX_INLINE_CHARS)")
+};
+
+/**
+ * Per-tool rule. `textPaths` are dotted paths tried in order; the first one
+ * holding a string becomes the text view the preview and read_result work
+ * on. An empty list means the pretty-printed JSON is the view. `truncate:
+ * false` keeps the whole result inline and only adds the handle
+ * (extract_embedded_state's never-truncate rule). `when` gates on params.
+ */
+export const INLINE_THRESHOLD_TOOLS = Object.freeze({
+  scrape: { textPaths: ['content.markdown', 'content.text', 'content.html', 'content.rawHtml'], truncate: true },
+  fetch_url: { textPaths: ['body'], truncate: true },
+  extract_content: { textPaths: ['content.markdown', 'content.text', 'content.html', 'content.cleanedHTML'], truncate: true },
+  crawl_deep: { textPaths: [], truncate: true },
+  batch_scrape: { textPaths: [], truncate: true },
+  stealth_mode: { textPaths: ['content.markdown', 'content.text', 'content.html'], truncate: true, when: (params) => params?.operation === 'scrape' },
+  scrape_with_actions: { textPaths: ['content.markdown', 'content.text', 'content.html'], truncate: true },
+  process_document: { textPaths: ['content.text'], truncate: true },
+  deep_research: { textPaths: [], truncate: true },
+  extract_embedded_state: { textPaths: [], truncate: false }
+});
+
+/** param -> env (an int of at least 1,000) -> default. */
+export function resolveMaxInlineChars(params, env = process.env) {
+  const fromParam = params?.max_inline_chars;
+  if (Number.isInteger(fromParam) && fromParam >= 1000) return fromParam;
+  const fromEnv = Number.parseInt(env?.CRAWLFORGE_MAX_INLINE_CHARS ?? '', 10);
+  if (Number.isInteger(fromEnv) && fromEnv >= 1000) return fromEnv;
+  return DEFAULT_MAX_INLINE_CHARS;
+}
+
+/** Read a dotted path ("content.markdown") off an object; undefined when absent. */
+export function readDottedPath(object, dotted) {
+  let current = object;
+  for (const key of dotted.split('.')) {
+    if (current === null || typeof current !== 'object') return undefined;
+    current = current[key];
+  }
+  return current;
+}
+
+/**
+ * The text a preview and read_result operate on: the first configured path
+ * holding a string, else the pretty-printed JSON of the whole result.
+ * @returns {{ view: 'text'|'json', view_path: string|null, text: string }}
+ */
+export function resultTextView(resultObject, textPaths = []) {
+  for (const path of textPaths) {
+    const value = readDottedPath(resultObject, path);
+    if (typeof value === 'string') return { view: 'text', view_path: path, text: value };
+  }
+  return { view: 'json', view_path: null, text: JSON.stringify(resultObject, null, 2) };
+}
+
+function warningsOf(resultObject) {
+  return Array.isArray(resultObject.warnings) ? resultObject.warnings.filter((w) => typeof w === 'string') : [];
+}
+
+/**
+ * @param {string} toolName
+ * @param {object} resultObject — the parsed JSON result
+ * @param {object} params — the tool params (max_inline_chars, url, operation)
+ * @param {{ store: import('../core/ResultStore.js').ResultStore, env?: object }} deps
+ * @returns {{ result: object, stored: boolean }}
+ */
+export function applyInlineThreshold(toolName, resultObject, params, { store, env = process.env }) {
+  const unchanged = { result: resultObject, stored: false };
+  const config = INLINE_THRESHOLD_TOOLS[toolName];
+  if (!config || (config.when && !config.when(params))) return unchanged;
+  if (!resultObject || typeof resultObject !== 'object' || Array.isArray(resultObject)) return unchanged;
+
+  const maxInline = resolveMaxInlineChars(params, env);
+  const json = JSON.stringify(resultObject);
+  if (json.length <= maxInline) return unchanged;
+
+  const { view, view_path, text } = resultTextView(resultObject, config.textPaths);
+
+  let handle;
+  try {
+    handle = store.put(toolName, resultObject, {
+      meta: { view, view_path, url: resultObject.url ?? params?.url ?? null }
+    });
+  } catch {
+    return {
+      result: { ...resultObject, warnings: [...warningsOf(resultObject), 'result could not be stored; returned inline'] },
+      stored: false
+    };
+  }
+
+  const expires_at = new Date(Date.now() + store.ttlMs).toISOString();
+  const viewDesc = view === 'text' ? `the ${view_path} text` : 'the pretty-printed JSON';
+  const readWith = 'read it with read_result (1 credit) - operation "search" (query), "slice" (offset, length), "lines" or "json_path" (path) - and do not fetch the page again';
+
+  if (!config.truncate) {
+    const hint = `Result is ${json.length} chars as JSON, over the inline limit of ${maxInline}; it is returned whole (this tool never truncates) and is also kept for 1 hour under result_handle ${handle}: ${readWith}.`;
+    return {
+      result: {
+        ...resultObject,
+        result_handle: handle,
+        total_chars: text.length,
+        view,
+        view_path,
+        truncated: false,
+        expires_at,
+        warnings: [...warningsOf(resultObject), hint]
+      },
+      stored: true
+    };
+  }
+
+  const preview = text.slice(0, maxInline);
+  const hint = `Result is ${json.length} chars as JSON, over the inline limit of ${maxInline}; preview holds the first ${preview.length} chars of ${viewDesc} (${text.length} chars in total) and the full result is kept for 1 hour under result_handle ${handle}: ${readWith}.`;
+
+  const shaped = {};
+  for (const [key, value] of Object.entries(resultObject)) {
+    if (key === 'warnings') continue;
+    if (value === null || typeof value === 'number' || typeof value === 'boolean' || (typeof value === 'string' && value.length <= 200)) {
+      shaped[key] = value;
+    }
+  }
+  Object.assign(shaped, {
+    preview,
+    result_handle: handle,
+    total_chars: text.length,
+    view,
+    view_path,
+    truncated: true,
+    expires_at,
+    warnings: [...warningsOf(resultObject), hint]
+  });
+  return { result: shaped, stored: true };
+}

package/src/server/toolFilter.js

@@ -1,7 +1,7 @@
 /**
  * toolFilter — client-side tool selection (Phase 6).
  *
- * Lets an MCP client load a subset of the 29 registered tools via env vars,
+ * Lets an MCP client load a subset of the 30 registered tools via env vars,
  * cutting context bloat (mirrors Bright Data / Exa's TOOLS / GROUPS pattern).
  *
  * Pure module: no I/O, no logging; process.env is only read via
@@ -9,12 +9,14 @@
  * tests.
  */
 
+import { INLINE_THRESHOLD_TOOLS } from './inlineThreshold.js';
+
 // The full set of tool names server.js registers, grouped by category.
 export const TOOL_GROUPS = {
-  basic: ['fetch_url', 'extract_text', 'extract_links', 'extract_metadata', 'scrape_structured'],
-  search: ['search_web', 'serp_rank'],
+  basic: ['fetch_url', 'extract_text', 'extract_links', 'extract_metadata', 'scrape_structured', 'read_result'],
+  search: ['search_web', 'serp_rank', 'reddit_search'],
   crawl: ['crawl_deep', 'map_site'],
-  extract: ['extract_content', 'process_document', 'summarize_content', 'analyze_content', 'extract_structured', 'extract_with_llm', 'list_ollama_models'],
+  extract: ['extract_content', 'process_document', 'summarize_content', 'analyze_content', 'extract_structured', 'extract_with_llm', 'list_ollama_models', 'extract_embedded_state'],
   batch: ['batch_scrape', 'get_batch_results', 'scrape_with_actions'],
   research: ['deep_research'],
   tracking: ['track_changes'],
@@ -84,6 +86,11 @@ export function createToolFilter(env = process.env) {
   if (enabled.has('batch_scrape')) {
     enabled.add('get_batch_results');
   }
+  // Likewise a tool whose large results come back as a result_handle names
+  // read_result in its hint, so that hint must never point at an unregistered tool.
+  if (Object.keys(INLINE_THRESHOLD_TOOLS).some((toolName) => enabled.has(toolName))) {
+    enabled.add('read_result');
+  }
 
   return {
     isEnabled(toolName) {

package/src/server/withAuth.js

@@ -19,6 +19,22 @@ import { createHash } from 'node:crypto';
 import { recordToolInvocation } from '../observability/tracing.js';
 import { isInternalRequest, preflightRefusal, reportedActualCost, requestContext } from './requestContext.js';
 import { appendFallbackHint } from './fallbackHints.js';
+import { INLINE_THRESHOLD_TOOLS, applyInlineThreshold } from './inlineThreshold.js';
+import { getResultStore } from '../core/ResultStore.js';
+
+/**
+ * Replace an over-threshold JSON text result (and its structuredContent, if
+ * any) with the shaped preview + handle. Non-JSON text is left alone.
+ */
+function shapeLargeResult(toolName, result, params) {
+  if (!result || !Array.isArray(result.content) || result.content[0]?.type !== 'text') return;
+  let parsed;
+  try { parsed = JSON.parse(result.content[0].text); } catch { return; }
+  const { result: shaped, stored } = applyInlineThreshold(toolName, parsed, params, { store: getResultStore() });
+  if (!stored) return;
+  result.content[0].text = JSON.stringify(shaped, null, 2);
+  if (result.structuredContent) result.structuredContent = shaped;
+}
 
 export function hashParams(params) {
   try {
@@ -99,6 +115,15 @@ export function makeWithAuth({ authManager, logger, metrics = null }) {
         // multi-URL tool that skipped one disallowed URL and still returned a
         // result did real work and bills for it.
         const refused = isErrorResult && preflightRefusal() !== null;
+
+        // Phase 2: a result over max_inline_chars is stored and returned as
+        // a preview plus a result_handle for read_result. Internal (website
+        // REST proxy) requests are left whole — the website applies its own
+        // threshold with its own store.
+        if (!isErrorResult && !internal && toolName in INLINE_THRESHOLD_TOOLS) {
+          try { shapeLargeResult(toolName, result, params); } catch { /* shaping must never break the request path */ }
+        }
+
         const base = billable();
         const charge = creditCost === 0 || refused
           ? 0

package/src/skills/agent-skills/crawlforge-batch-automation/SKILL.md

@@ -71,7 +71,8 @@ CLI: `crawlforge batch urls.txt --format markdown --concurrency 10`.
 Use the `batchId` from `batch_scrape` to retrieve paginated results for a
 completed or in-progress job. Cheap (1 credit) because the batch was already
 paid for. Completed jobs are also exposed as `crawlforge://job/{jobId}`
-resources.
+resources. Stored batch results share the local 1-hour result store that
+`read_result` reads, with the same eviction, so page through a batch within the hour.
 
 ## scrape_with_actions — interact, then scrape (cost: 5)
 

package/src/skills/agent-skills/crawlforge-getting-started/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: crawlforge-getting-started
-description: "Orientation and tool-selection guide for the CrawlForge MCP server's 29 web tools. Use when the user is getting started with CrawlForge, asks which CrawlForge tool to use, how to set up the API key, how skills or the CLI work, what a tool costs in credits, or when one tool fails and a fallback is needed. Routes requests to the right specialized skill (web scraping, deep research, stealth, structured extraction, change tracking, batch automation), and explains MCP-tools-vs-CLI, the Ollama-first LLM fallback chain, and per-tool credit costs."
+description: "Orientation and tool-selection guide for the CrawlForge MCP server's 30 web tools. Use when the user is getting started with CrawlForge, asks which CrawlForge tool to use, how to set up the API key, how skills or the CLI work, what a tool costs in credits, or when one tool fails and a fallback is needed. Routes requests to the right specialized skill (web scraping, deep research, stealth, structured extraction, change tracking, batch automation), and explains MCP-tools-vs-CLI, the Ollama-first LLM fallback chain, and per-tool credit costs."
 metadata:
   version: 5.6.6
   source: crawlforge-mcp-server
@@ -8,7 +8,7 @@ metadata:
 
 # CrawlForge: Getting Started
 
-CrawlForge is an MCP server with **29 tools** for web scraping, crawling,
+CrawlForge is an MCP server with **30 tools** for web scraping, crawling,
 extraction, research, change tracking, and AI-compliance. This skill orients you
 and routes each request to the right specialized skill.
 
@@ -52,9 +52,9 @@ stored at `~/.crawlforge/config.json`.
 | Watch a page for changes / monitor pricing | **crawlforge-change-tracking** |
 | Scrape many URLs, run browser actions, generate llms.txt | **crawlforge-batch-automation** |
 
-## The 29 tools at a glance
+## The 30 tools at a glance
 
-- **Basic (5):** fetch_url, extract_text, extract_links, extract_metadata, scrape_structured
+- **Basic (6):** fetch_url, extract_text, extract_links, extract_metadata, scrape_structured, read_result
 - **Unified (1):** scrape (multi-format single fetch)
 - **Search & research (5):** search_web, serp_rank, reddit_search, deep_research, agent
 - **Crawl (2):** crawl_deep, map_site
@@ -90,11 +90,13 @@ Do not suggest adding API keys — local Ollama is the intended zero-cost defaul
 | Symptom | Try next |
 |---------|----------|
 | `scrape` / `fetch_url` returns 403, 429, CAPTCHA, or empty JS shell | `stealth_mode` (crawlforge-stealth-browsing) |
+| A site you already know blocks | `scrape` with `escalate: true` — plain fetch first, stealth browser only if it is walled; never `stealth_mode` first |
 | No template for a known site | `scrape_structured` → `extract_structured` → `extract_with_llm` |
 | LLM extraction unavailable (no Ollama/keys) | `scrape_structured` with CSS selectors |
 | Single page too slow / many pages | `batch_scrape` (async + webhook) |
 | Wrong region / currency shown | `localization` |
 | Need a big report but cost is high | lower `maxUrls` on `deep_research` |
+| Result came back `truncated: true` with a `result_handle` | `read_result` (search, slice, lines, json_path) — never fetch the page again |
 
 ## Credits
 

package/src/skills/agent-skills/crawlforge-getting-started/references/credits.md

@@ -14,12 +14,13 @@ metered; there is no free tier. Tools marked "scales" cost more as work grows.
 | `scrape_template` | Pre-built site extractor. |
 | `list_ollama_models` | List local LLMs. |
 | `get_batch_results` | Retrieve an already-paid batch. |
+| `read_result` | Search, slice, lines or a JSON path over a stored result (`result_handle`). |
 
 ## 2 credits
 
 | Tool | Notes |
 |------|-------|
-| `scrape` | Unified multi-format single fetch. |
+| `scrape` | Unified multi-format single fetch. A `highlights` or `question` format adds 1 once per call; `mode:"model"` on one of them adds 3 once. `escalate:true` adds 5 as the projected ceiling; the actual charge drops back to the base when the plain fetch succeeded and no escalation ran. |
 | `scrape_structured` | CSS-selector extraction. |
 | `extract_embedded_state` | Embedded JS state (`__NEXT_DATA__`, RSC, Nuxt, Apollo, Redux). |
 | `extract_content` | Readability-cleaned article. |
@@ -74,4 +75,10 @@ metered; there is no free tier. Tools marked "scales" cost more as work grows.
 - Cap dynamic tools: `deep_research`/`agent` via `maxUrls`, `crawl_deep` via
   `max_pages`, `batch_scrape` via the URL list size.
 - `get_batch_results` (1) is cheap — submit a batch once, page through results.
+- A result over `max_inline_chars` costs nothing extra; read the stored copy with
+  `read_result` (1) instead of fetching again.
+- On a site that blocks, one `scrape` with `escalate:true` costs at most 7 and
+  only 2 when the plain fetch works; `scrape` then `stealth_mode` costs 7 either
+  way. Do not set it on a site that reads fine — it raises the projection your
+  balance is checked against.
 - Errors are charged at half the tool cost; creator mode is unlimited.

package/src/skills/agent-skills/crawlforge-stealth-browsing/SKILL.md

@@ -24,6 +24,11 @@ Escalate from a normal `scrape` / `fetch_url` (see crawlforge-web-scraping) when
 `stealth_mode` drives a real browser with randomized fingerprints, human
 behavior simulation, and WebRTC/canvas/WebGL spoofing.
 
+When you already know the site blocks, `scrape` with `escalate: true` does both
+steps in one call: the plain fetch first, then the same stealth browser only if
+that fetch is walled. Projected at 7, charged 2 when the plain fetch worked.
+Still never reach for `stealth_mode` first.
+
 ## stealth_mode (cost: 5)
 
 `stealth_mode` is operation-based. Typical flow: create a context, then create a

package/src/skills/agent-skills/crawlforge-web-scraping/SKILL.md

@@ -22,6 +22,7 @@ site discovery (sitemaps and URL maps), and whole-site crawling.
 - "Get the title / meta / Open Graph / SEO tags" → `extract_metadata`
 - "Map all the URLs on this site" / "generate a sitemap" → `map_site`
 - "Crawl the whole docs site" / "index every page" → `crawl_deep`
+- "Read more of that truncated result" → `read_result`
 
 ## Tool selection (fastest path first)
 
@@ -35,7 +36,9 @@ site discovery (sitemaps and URL maps), and whole-site crawling.
 5. **`crawl_deep`** to walk an entire site and (optionally) extract content.
 
 If a page returns 403/429, a CAPTCHA, or empty "enable JavaScript" content,
-switch to the **crawlforge-stealth-browsing** skill (`stealth_mode`).
+switch to the **crawlforge-stealth-browsing** skill (`stealth_mode`). When the
+site is *known* to block, use `scrape` with `escalate: true` instead of two
+calls — still never `stealth_mode` first.
 
 ## scrape — unified multi-format (cost: 2)
 
@@ -58,6 +61,62 @@ Get markdown + links + metadata in a single call:
 a failing format adds a `warnings[]` entry instead of failing the whole call.
 `onlyMainContent` (default `true`) strips boilerplate via Readability.
 
+### escalate — one call for a site that blocks (projected 7, charged 2 when it doesn't)
+
+`escalate: true` keeps the plain fetch first and retries in the stealth browser
+only if that fetch comes back walled (403/429, a challenge page, an empty
+shell). The escalated page goes through the same formats, so `markdown`,
+`highlights` and the rest work exactly as they do on a plain read.
+
+```json
+{
+  "tool": "scrape",
+  "params": {
+    "url": "https://www.producthunt.com/",
+    "formats": ["markdown"],
+    "escalate": true,
+    "escalate_engine": "playwright"
+  }
+}
+```
+
+The result adds `escalated: true|false`, and `stealth: { engine,
+vendor_detected }` when the browser ran. The projection is `2 + 5`; the charge
+falls back to the base when the plain fetch succeeded and nothing escalated.
+`escalate_engine` is `"playwright"` (default) or `"camoufox"`. robots.txt is
+respected on the escalated path too, and a second call to a host that blocked
+within the last 24 hours skips the doomed plain fetch and says so in
+`warnings[]`.
+
+Query-scoped formats return only the parts of the page that match, verbatim,
+with offsets into the `markdown` of the same call:
+
+```json
+{
+  "tool": "scrape",
+  "params": {
+    "url": "https://example.com/pricing",
+    "formats": [
+      "markdown",
+      { "type": "highlights", "query": "professional plan price", "max_highlights": 5 },
+      { "type": "question", "question": "How much is the professional plan per month?" }
+    ]
+  }
+}
+```
+
+`{ "type": "highlights", "query", "max_highlights"?, "mode"? }` returns
+`content.highlights: [{ text, kind, offset, length, score }]`, best first;
+`kind` is `sentence`, `table_row` or `code_block`, and
+`markdown.slice(offset, offset + length) === text`. `{ "type": "question",
+"question", "mode"? }` returns `content.answer: { text, grounded, evidence }`,
+with `text` the evidence joined and `grounded: true`. Either adds 1 credit once
+per call and calls no model. `"mode": "model"` adds 3 once per call: the model
+chooses the highlights (never rewrites them) or writes the answer, and a
+grounding check sets `grounded: false` when the answer holds a number or a name
+the evidence does not. Ask for `"markdown"` in the same call to quote with a
+locator.
+
 CLI equivalent:
 
 ```bash
@@ -131,6 +190,17 @@ BFS crawl up to depth 5 / 1000 pages. Use `include_patterns` /
 (on by default). Crawls projected over ~500 pages trigger a confirmation
 prompt (elicitation). CLI: `crawlforge crawl https://docs.example.com --depth 3 --max-pages 200`.
 
+## Large results — `max_inline_chars` and `read_result` (cost: 1)
+
+`scrape`, `fetch_url`, `extract_content` and `crawl_deep` accept
+`max_inline_chars` (default 40,000). A larger result comes back as its scalar
+fields plus `preview` (the first `max_inline_chars` characters of its text
+view), `result_handle`, `total_chars`, `truncated: true` and `expires_at`; the
+full result stays 1 hour on the local machine, never uploaded. Read it with
+`read_result` instead of fetching again: `operation:"search"` with a `query`
+finds offsets, then `operation:"slice"` with `offset`/`length` reads around one;
+`operation:"json_path"` with a `path` picks one field (`"lines"` reads by line).
+
 ## Cost note
 
 `scrape` (2 credits) is the default page read: one fetch returns every format
@@ -138,6 +208,7 @@ you ask for. `fetch_url`, `extract_text`, `extract_links`, `extract_metadata`
 (1 each) are cheaper only when one of them is the whole job; `fetch_url` followed
 by an `extract_*` call on the same URL is a double fetch that costs more than one
 `scrape`. `extract_content`, `map_site` = 2 · `crawl_deep` = 4 (grows with
-`max_pages`). Never re-fetch a page whose content is already in the conversation.
+`max_pages`) · `read_result` = 1 (reads a stored result, no fetch). Never re-fetch
+a page whose content is already in the conversation.
 
 See [tool reference](references/tool-reference.md) for the full parameter list.