crawlforge-mcp-server 5.6.11 → 5.8.0

package/src/schemas/toolOutputSchemas.js

@@ -15,7 +15,7 @@
  */
 
 import { z } from 'zod';
-import { SCRAPE_STRING_FORMATS } from '../tools/scrape/formats.js';
+import { SCRAPE_STRING_FORMATS, SCRAPE_OBJECT_FORMATS } from '../tools/scrape/formats.js';
 
 // `_cost` is injected by withAuth into the legacy JSON-text copy of the
 // result (never into structuredContent directly) — included here anyway so
@@ -28,6 +28,20 @@ const costShape = z.object({
   projection_note: z.string().optional().describe('Human-readable note about how the cost was projected')
 }).passthrough().optional().describe('Cost-transparency metadata (D3.5), present when injected into the text copy of the result');
 
+// ── result handles (Phase 2) ────────────────────────────────────────────────
+
+// A result over max_inline_chars comes back as a preview plus a handle for
+// read_result; these fields describe that shape wherever it can appear.
+const resultHandleShape = {
+  preview: z.string().optional().describe('The first max_inline_chars characters of the view named by view_path (or of the pretty-printed JSON)'),
+  result_handle: z.string().optional().describe('Handle for read_result; the full result is kept 1 hour'),
+  total_chars: z.number().optional().describe('Length of the full view in characters'),
+  view: z.enum(['text', 'json']).optional().describe('Whether preview and read_result offsets index a text field or the pretty-printed JSON'),
+  view_path: z.string().nullable().optional().describe('Dotted path of the text field the view was cut from; null for the JSON view'),
+  truncated: z.boolean().optional().describe('True when the inline result is a preview'),
+  expires_at: z.string().optional().describe('When the stored result is dropped (ISO 8601)')
+};
+
 // ── scrape ──────────────────────────────────────────────────────────────────
 
 const scrapeLinkShape = z.object({
@@ -59,11 +73,23 @@ const scrapeMetadataShape = z.object({
   url: z.string().optional()
 }).passthrough();
 
+// A sentence, table row or fenced code block returned verbatim by the
+// highlights and question formats; `offset`/`length` index the `markdown`
+// format of the same call.
+const scrapeHighlightUnitShape = z.object({
+  text: z.string().optional().describe('Verbatim page text: markdown.slice(offset, offset + length) === text'),
+  kind: z.enum(['sentence', 'table_row', 'code_block']).optional(),
+  offset: z.number().optional().describe('JS string index into the markdown format of this call'),
+  length: z.number().optional(),
+  score: z.number().optional().describe('BM25 relevance to the query, higher is better')
+}).passthrough();
+
 // One content key per string format `scrape` accepts (the "screenshot"
-// format fills `screenshots`), plus `json` for the {type:"json"} object
-// format. Checked against the format list the input schema is built from,
-// so a format added to formats.js without a shape here fails at load
-// instead of shipping a schema that does not describe the result (0.3).
+// format fills `screenshots`), plus one per object format: `json` for
+// {type:"json"}, `highlights` for {type:"highlights"}, `answer` for
+// {type:"question"}. Checked against the format lists the input schema is
+// built from, so a format added to formats.js without a shape here fails at
+// load instead of shipping a schema that does not describe the result (0.3).
 const scrapeFormatShapes = {
   markdown: z.string().optional(),
   html: z.string().optional(),
@@ -72,10 +98,17 @@ const scrapeFormatShapes = {
   links: scrapeLinksShape.optional(),
   metadata: scrapeMetadataShape.optional(),
   branding: z.record(z.unknown()).optional().describe('Static design tokens: colors, fonts, logo'),
-  screenshots: z.array(z.object({}).passthrough()).optional().describe('Present for the "screenshot" format; each item carries a resourceUri once published')
+  screenshots: z.array(z.object({}).passthrough()).optional().describe('Present for the "screenshot" format; each item carries a resourceUri once published'),
+  json: z.unknown().optional().describe('Result of the {type:"json"} format (LLM-structured extraction)'),
+  highlights: z.array(scrapeHighlightUnitShape).optional().describe('Result of the {type:"highlights"} format: the units matching the query, best first, verbatim with offsets'),
+  answer: z.object({
+    text: z.string().optional().describe('Extractive mode: the evidence texts joined; model mode: the model\'s answer'),
+    grounded: z.boolean().optional().describe('True when every number and proper noun in text appears in the evidence or the question; always true in extractive mode'),
+    evidence: z.array(scrapeHighlightUnitShape).optional().describe('The units the answer rests on, verbatim with offsets')
+  }).passthrough().optional().describe('Result of the {type:"question"} format')
 };
-const scrapeContentKey = (format) => (format === 'screenshot' ? 'screenshots' : format);
-for (const format of SCRAPE_STRING_FORMATS) {
+const scrapeContentKey = (format) => ({ screenshot: 'screenshots', question: 'answer' }[format] ?? format);
+for (const format of [...SCRAPE_STRING_FORMATS, ...SCRAPE_OBJECT_FORMATS]) {
   if (!scrapeFormatShapes[scrapeContentKey(format)]) {
     throw new Error(`toolOutputSchemas: scrape format "${format}" has no output shape`);
   }
@@ -91,11 +124,9 @@ const scrapeShape = {
     vendor: z.string().optional(),
     evidence: z.string().optional()
   }).passthrough().optional().describe('Present when a bot-defence vendor served a challenge page; the fallback hint names the tool to try next'),
-  content: z.object({
-    ...scrapeFormatShapes,
-    json: z.unknown().optional().describe('Result of the {type:"json"} format (LLM-structured extraction)')
-  }).passthrough().optional().describe('One key per requested format'),
+  content: z.object(scrapeFormatShapes).passthrough().optional().describe('One key per requested format'),
   warnings: z.array(z.string()).optional().describe('Per-format warnings; partial success never fails the whole call'),
+  ...resultHandleShape,
   _cost: costShape
 };
 
@@ -344,6 +375,45 @@ const crawlDeepShape = {
   }).passthrough().optional(),
   crawled_at: z.string().optional().describe('When the pages were actually fetched (ISO 8601)'),
   cached: z.boolean().optional().describe('True when this response was replayed from an earlier crawl rather than crawled now; crawled_at gives its age'),
+  ...resultHandleShape,
+  _cost: costShape
+};
+
+// ── read_result ───────────────────────────────────────────────────────────────
+
+const readResultMatchShape = z.object({
+  offset: z.number().optional().describe('Index of the match in the view'),
+  length: z.number().optional(),
+  context_offset: z.number().optional().describe('Index of the first character of context in the view'),
+  context: z.string().optional().describe('Verbatim text around the match: view.slice(context_offset, context_offset + context.length)')
+}).passthrough();
+
+const readResultShape = {
+  handle: z.string().optional(),
+  tool: z.string().optional().describe('The tool that produced the stored result'),
+  operation: z.enum(['slice', 'search', 'lines', 'json_path']).optional(),
+  view: z.enum(['text', 'json']).optional(),
+  view_path: z.string().nullable().optional(),
+  total_chars: z.number().optional().describe('Length of the full view'),
+  expires_at: z.string().optional(),
+  offset: z.number().optional().describe('slice: first character returned'),
+  length: z.number().optional().describe('slice: characters returned'),
+  text: z.string().optional().describe('slice: verbatim view.slice(offset, offset + length)'),
+  has_more: z.boolean().optional().describe('slice/lines: more follows the returned range'),
+  query: z.string().optional(),
+  matches: z.array(readResultMatchShape).optional().describe('search: matches with 200 chars of context each side'),
+  total_matches: z.number().optional(),
+  first_line: z.number().optional(),
+  line_count: z.number().optional(),
+  total_lines: z.number().optional(),
+  char_offset: z.number().optional().describe('lines: view offset of the first returned line'),
+  lines: z.array(z.string()).optional(),
+  path: z.string().optional(),
+  value: z.unknown().optional().describe('json_path: the subtree; null with a preview when it is over max_inline_chars'),
+  value_chars: z.number().optional(),
+  preview: z.string().optional(),
+  truncated: z.boolean().optional().describe('search: more matches than returned; json_path: value replaced by a preview'),
+  warnings: z.array(z.string()).optional(),
   _cost: costShape
 };
 
@@ -356,7 +426,8 @@ export const OUTPUT_SCHEMAS = {
   reddit_search: redditSearchShape,
   search_web: searchWebShape,
   extract_structured: extractStructuredShape,
-  crawl_deep: crawlDeepShape
+  crawl_deep: crawlDeepShape,
+  read_result: readResultShape
 };
 
 export default OUTPUT_SCHEMAS;

package/src/server/fallbackHints.js

@@ -24,6 +24,7 @@ export const FALLBACK_HINTS = Object.freeze({
   map_site: 'Use crawl_deep with extract_content:false to discover URLs by following links when there is no sitemap.',
   batch_scrape: 'For an async job poll get_batch_results with the batchId; scrape one of the URLs alone to diagnose a per-URL failure.',
   get_batch_results: 'An unknown or expired batchId cannot be recovered - run batch_scrape again only if the results are still needed.',
+  read_result: 'An unknown or expired handle cannot be recovered - results are kept 1 hour; run the original tool again only if the result is still needed.',
   process_document: 'For an HTML page use scrape formats:["markdown"]; sourceType:"pdf_url" needs a URL that serves a PDF.',
   summarize_content: 'Pass the text itself (e.g. markdown from a scrape result), not a URL.',
   analyze_content: 'Pass the text itself (e.g. markdown from a scrape result), not a URL.',

package/src/server/inlineThreshold.js

@@ -0,0 +1,149 @@
+/**
+ * inlineThreshold — the one declaration (G5) of `max_inline_chars` and of
+ * the rule that turns an oversized tool result into a preview plus a
+ * result_handle for read_result (Phase 2).
+ *
+ * Applied by withAuth after a successful handler run and before `_cost`
+ * injection, for the tools listed in INLINE_THRESHOLD_TOOLS only. Internal
+ * (website REST proxy) requests are never shaped here — the website applies
+ * its own threshold with its own store.
+ */
+
+import { z } from 'zod';
+
+export const DEFAULT_MAX_INLINE_CHARS = 40000;
+
+export const MAX_INLINE_CHARS_PARAM = {
+  max_inline_chars: z.number().int().min(1000).max(10_000_000).optional().describe("Largest result to return inline, in characters of its JSON. Over it, the call returns a preview plus a result_handle for read_result instead of the whole result (default 40,000; env CRAWLFORGE_MAX_INLINE_CHARS)")
+};
+
+/**
+ * Per-tool rule. `textPaths` are dotted paths tried in order; the first one
+ * holding a string becomes the text view the preview and read_result work
+ * on. An empty list means the pretty-printed JSON is the view. `truncate:
+ * false` keeps the whole result inline and only adds the handle
+ * (extract_embedded_state's never-truncate rule). `when` gates on params.
+ */
+export const INLINE_THRESHOLD_TOOLS = Object.freeze({
+  scrape: { textPaths: ['content.markdown', 'content.text', 'content.html', 'content.rawHtml'], truncate: true },
+  fetch_url: { textPaths: ['body'], truncate: true },
+  extract_content: { textPaths: ['content.markdown', 'content.text', 'content.html', 'content.cleanedHTML'], truncate: true },
+  crawl_deep: { textPaths: [], truncate: true },
+  batch_scrape: { textPaths: [], truncate: true },
+  stealth_mode: { textPaths: ['content.markdown', 'content.text', 'content.html'], truncate: true, when: (params) => params?.operation === 'scrape' },
+  scrape_with_actions: { textPaths: ['content.markdown', 'content.text', 'content.html'], truncate: true },
+  process_document: { textPaths: ['content.text'], truncate: true },
+  deep_research: { textPaths: [], truncate: true },
+  extract_embedded_state: { textPaths: [], truncate: false }
+});
+
+/** param -> env (an int of at least 1,000) -> default. */
+export function resolveMaxInlineChars(params, env = process.env) {
+  const fromParam = params?.max_inline_chars;
+  if (Number.isInteger(fromParam) && fromParam >= 1000) return fromParam;
+  const fromEnv = Number.parseInt(env?.CRAWLFORGE_MAX_INLINE_CHARS ?? '', 10);
+  if (Number.isInteger(fromEnv) && fromEnv >= 1000) return fromEnv;
+  return DEFAULT_MAX_INLINE_CHARS;
+}
+
+/** Read a dotted path ("content.markdown") off an object; undefined when absent. */
+export function readDottedPath(object, dotted) {
+  let current = object;
+  for (const key of dotted.split('.')) {
+    if (current === null || typeof current !== 'object') return undefined;
+    current = current[key];
+  }
+  return current;
+}
+
+/**
+ * The text a preview and read_result operate on: the first configured path
+ * holding a string, else the pretty-printed JSON of the whole result.
+ * @returns {{ view: 'text'|'json', view_path: string|null, text: string }}
+ */
+export function resultTextView(resultObject, textPaths = []) {
+  for (const path of textPaths) {
+    const value = readDottedPath(resultObject, path);
+    if (typeof value === 'string') return { view: 'text', view_path: path, text: value };
+  }
+  return { view: 'json', view_path: null, text: JSON.stringify(resultObject, null, 2) };
+}
+
+function warningsOf(resultObject) {
+  return Array.isArray(resultObject.warnings) ? resultObject.warnings.filter((w) => typeof w === 'string') : [];
+}
+
+/**
+ * @param {string} toolName
+ * @param {object} resultObject — the parsed JSON result
+ * @param {object} params — the tool params (max_inline_chars, url, operation)
+ * @param {{ store: import('../core/ResultStore.js').ResultStore, env?: object }} deps
+ * @returns {{ result: object, stored: boolean }}
+ */
+export function applyInlineThreshold(toolName, resultObject, params, { store, env = process.env }) {
+  const unchanged = { result: resultObject, stored: false };
+  const config = INLINE_THRESHOLD_TOOLS[toolName];
+  if (!config || (config.when && !config.when(params))) return unchanged;
+  if (!resultObject || typeof resultObject !== 'object' || Array.isArray(resultObject)) return unchanged;
+
+  const maxInline = resolveMaxInlineChars(params, env);
+  const json = JSON.stringify(resultObject);
+  if (json.length <= maxInline) return unchanged;
+
+  const { view, view_path, text } = resultTextView(resultObject, config.textPaths);
+
+  let handle;
+  try {
+    handle = store.put(toolName, resultObject, {
+      meta: { view, view_path, url: resultObject.url ?? params?.url ?? null }
+    });
+  } catch {
+    return {
+      result: { ...resultObject, warnings: [...warningsOf(resultObject), 'result could not be stored; returned inline'] },
+      stored: false
+    };
+  }
+
+  const expires_at = new Date(Date.now() + store.ttlMs).toISOString();
+  const viewDesc = view === 'text' ? `the ${view_path} text` : 'the pretty-printed JSON';
+  const readWith = 'read it with read_result (1 credit) - operation "search" (query), "slice" (offset, length), "lines" or "json_path" (path) - and do not fetch the page again';
+
+  if (!config.truncate) {
+    const hint = `Result is ${json.length} chars as JSON, over the inline limit of ${maxInline}; it is returned whole (this tool never truncates) and is also kept for 1 hour under result_handle ${handle}: ${readWith}.`;
+    return {
+      result: {
+        ...resultObject,
+        result_handle: handle,
+        total_chars: text.length,
+        view,
+        view_path,
+        truncated: false,
+        expires_at,
+        warnings: [...warningsOf(resultObject), hint]
+      },
+      stored: true
+    };
+  }
+
+  const preview = text.slice(0, maxInline);
+  const hint = `Result is ${json.length} chars as JSON, over the inline limit of ${maxInline}; preview holds the first ${preview.length} chars of ${viewDesc} (${text.length} chars in total) and the full result is kept for 1 hour under result_handle ${handle}: ${readWith}.`;
+
+  const shaped = {};
+  for (const [key, value] of Object.entries(resultObject)) {
+    if (key === 'warnings') continue;
+    if (value === null || typeof value === 'number' || typeof value === 'boolean' || (typeof value === 'string' && value.length <= 200)) {
+      shaped[key] = value;
+    }
+  }
+  Object.assign(shaped, {
+    preview,
+    result_handle: handle,
+    total_chars: text.length,
+    view,
+    view_path,
+    truncated: true,
+    expires_at,
+    warnings: [...warningsOf(resultObject), hint]
+  });
+  return { result: shaped, stored: true };
+}

package/src/server/toolFilter.js

@@ -1,7 +1,7 @@
 /**
  * toolFilter — client-side tool selection (Phase 6).
  *
- * Lets an MCP client load a subset of the 29 registered tools via env vars,
+ * Lets an MCP client load a subset of the 30 registered tools via env vars,
  * cutting context bloat (mirrors Bright Data / Exa's TOOLS / GROUPS pattern).
  *
  * Pure module: no I/O, no logging; process.env is only read via
@@ -9,12 +9,14 @@
  * tests.
  */
 
+import { INLINE_THRESHOLD_TOOLS } from './inlineThreshold.js';
+
 // The full set of tool names server.js registers, grouped by category.
 export const TOOL_GROUPS = {
-  basic: ['fetch_url', 'extract_text', 'extract_links', 'extract_metadata', 'scrape_structured'],
-  search: ['search_web', 'serp_rank'],
+  basic: ['fetch_url', 'extract_text', 'extract_links', 'extract_metadata', 'scrape_structured', 'read_result'],
+  search: ['search_web', 'serp_rank', 'reddit_search'],
   crawl: ['crawl_deep', 'map_site'],
-  extract: ['extract_content', 'process_document', 'summarize_content', 'analyze_content', 'extract_structured', 'extract_with_llm', 'list_ollama_models'],
+  extract: ['extract_content', 'process_document', 'summarize_content', 'analyze_content', 'extract_structured', 'extract_with_llm', 'list_ollama_models', 'extract_embedded_state'],
   batch: ['batch_scrape', 'get_batch_results', 'scrape_with_actions'],
   research: ['deep_research'],
   tracking: ['track_changes'],
@@ -84,6 +86,11 @@ export function createToolFilter(env = process.env) {
   if (enabled.has('batch_scrape')) {
     enabled.add('get_batch_results');
   }
+  // Likewise a tool whose large results come back as a result_handle names
+  // read_result in its hint, so that hint must never point at an unregistered tool.
+  if (Object.keys(INLINE_THRESHOLD_TOOLS).some((toolName) => enabled.has(toolName))) {
+    enabled.add('read_result');
+  }
 
   return {
     isEnabled(toolName) {

package/src/server/withAuth.js

@@ -19,6 +19,22 @@ import { createHash } from 'node:crypto';
 import { recordToolInvocation } from '../observability/tracing.js';
 import { isInternalRequest, preflightRefusal, reportedActualCost, requestContext } from './requestContext.js';
 import { appendFallbackHint } from './fallbackHints.js';
+import { INLINE_THRESHOLD_TOOLS, applyInlineThreshold } from './inlineThreshold.js';
+import { getResultStore } from '../core/ResultStore.js';
+
+/**
+ * Replace an over-threshold JSON text result (and its structuredContent, if
+ * any) with the shaped preview + handle. Non-JSON text is left alone.
+ */
+function shapeLargeResult(toolName, result, params) {
+  if (!result || !Array.isArray(result.content) || result.content[0]?.type !== 'text') return;
+  let parsed;
+  try { parsed = JSON.parse(result.content[0].text); } catch { return; }
+  const { result: shaped, stored } = applyInlineThreshold(toolName, parsed, params, { store: getResultStore() });
+  if (!stored) return;
+  result.content[0].text = JSON.stringify(shaped, null, 2);
+  if (result.structuredContent) result.structuredContent = shaped;
+}
 
 export function hashParams(params) {
   try {
@@ -99,6 +115,15 @@ export function makeWithAuth({ authManager, logger, metrics = null }) {
         // multi-URL tool that skipped one disallowed URL and still returned a
         // result did real work and bills for it.
         const refused = isErrorResult && preflightRefusal() !== null;
+
+        // Phase 2: a result over max_inline_chars is stored and returned as
+        // a preview plus a result_handle for read_result. Internal (website
+        // REST proxy) requests are left whole — the website applies its own
+        // threshold with its own store.
+        if (!isErrorResult && !internal && toolName in INLINE_THRESHOLD_TOOLS) {
+          try { shapeLargeResult(toolName, result, params); } catch { /* shaping must never break the request path */ }
+        }
+
         const base = billable();
         const charge = creditCost === 0 || refused
           ? 0

package/src/skills/agent-skills/crawlforge-batch-automation/SKILL.md

@@ -71,7 +71,8 @@ CLI: `crawlforge batch urls.txt --format markdown --concurrency 10`.
 Use the `batchId` from `batch_scrape` to retrieve paginated results for a
 completed or in-progress job. Cheap (1 credit) because the batch was already
 paid for. Completed jobs are also exposed as `crawlforge://job/{jobId}`
-resources.
+resources. Stored batch results share the local 1-hour result store that
+`read_result` reads, with the same eviction, so page through a batch within the hour.
 
 ## scrape_with_actions — interact, then scrape (cost: 5)
 

package/src/skills/agent-skills/crawlforge-getting-started/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: crawlforge-getting-started
-description: "Orientation and tool-selection guide for the CrawlForge MCP server's 29 web tools. Use when the user is getting started with CrawlForge, asks which CrawlForge tool to use, how to set up the API key, how skills or the CLI work, what a tool costs in credits, or when one tool fails and a fallback is needed. Routes requests to the right specialized skill (web scraping, deep research, stealth, structured extraction, change tracking, batch automation), and explains MCP-tools-vs-CLI, the Ollama-first LLM fallback chain, and per-tool credit costs."
+description: "Orientation and tool-selection guide for the CrawlForge MCP server's 30 web tools. Use when the user is getting started with CrawlForge, asks which CrawlForge tool to use, how to set up the API key, how skills or the CLI work, what a tool costs in credits, or when one tool fails and a fallback is needed. Routes requests to the right specialized skill (web scraping, deep research, stealth, structured extraction, change tracking, batch automation), and explains MCP-tools-vs-CLI, the Ollama-first LLM fallback chain, and per-tool credit costs."
 metadata:
   version: 5.6.6
   source: crawlforge-mcp-server
@@ -8,7 +8,7 @@ metadata:
 
 # CrawlForge: Getting Started
 
-CrawlForge is an MCP server with **29 tools** for web scraping, crawling,
+CrawlForge is an MCP server with **30 tools** for web scraping, crawling,
 extraction, research, change tracking, and AI-compliance. This skill orients you
 and routes each request to the right specialized skill.
 
@@ -52,9 +52,9 @@ stored at `~/.crawlforge/config.json`.
 | Watch a page for changes / monitor pricing | **crawlforge-change-tracking** |
 | Scrape many URLs, run browser actions, generate llms.txt | **crawlforge-batch-automation** |
 
-## The 29 tools at a glance
+## The 30 tools at a glance
 
-- **Basic (5):** fetch_url, extract_text, extract_links, extract_metadata, scrape_structured
+- **Basic (6):** fetch_url, extract_text, extract_links, extract_metadata, scrape_structured, read_result
 - **Unified (1):** scrape (multi-format single fetch)
 - **Search & research (5):** search_web, serp_rank, reddit_search, deep_research, agent
 - **Crawl (2):** crawl_deep, map_site
@@ -95,6 +95,7 @@ Do not suggest adding API keys — local Ollama is the intended zero-cost defaul
 | Single page too slow / many pages | `batch_scrape` (async + webhook) |
 | Wrong region / currency shown | `localization` |
 | Need a big report but cost is high | lower `maxUrls` on `deep_research` |
+| Result came back `truncated: true` with a `result_handle` | `read_result` (search, slice, lines, json_path) — never fetch the page again |
 
 ## Credits
 

package/src/skills/agent-skills/crawlforge-getting-started/references/credits.md

@@ -14,12 +14,13 @@ metered; there is no free tier. Tools marked "scales" cost more as work grows.
 | `scrape_template` | Pre-built site extractor. |
 | `list_ollama_models` | List local LLMs. |
 | `get_batch_results` | Retrieve an already-paid batch. |
+| `read_result` | Search, slice, lines or a JSON path over a stored result (`result_handle`). |
 
 ## 2 credits
 
 | Tool | Notes |
 |------|-------|
-| `scrape` | Unified multi-format single fetch. |
+| `scrape` | Unified multi-format single fetch. A `highlights` or `question` format adds 1 once per call; `mode:"model"` on one of them adds 3 once. |
 | `scrape_structured` | CSS-selector extraction. |
 | `extract_embedded_state` | Embedded JS state (`__NEXT_DATA__`, RSC, Nuxt, Apollo, Redux). |
 | `extract_content` | Readability-cleaned article. |
@@ -74,4 +75,6 @@ metered; there is no free tier. Tools marked "scales" cost more as work grows.
 - Cap dynamic tools: `deep_research`/`agent` via `maxUrls`, `crawl_deep` via
   `max_pages`, `batch_scrape` via the URL list size.
 - `get_batch_results` (1) is cheap — submit a batch once, page through results.
+- A result over `max_inline_chars` costs nothing extra; read the stored copy with
+  `read_result` (1) instead of fetching again.
 - Errors are charged at half the tool cost; creator mode is unlimited.

package/src/skills/agent-skills/crawlforge-web-scraping/SKILL.md

@@ -22,6 +22,7 @@ site discovery (sitemaps and URL maps), and whole-site crawling.
 - "Get the title / meta / Open Graph / SEO tags" → `extract_metadata`
 - "Map all the URLs on this site" / "generate a sitemap" → `map_site`
 - "Crawl the whole docs site" / "index every page" → `crawl_deep`
+- "Read more of that truncated result" → `read_result`
 
 ## Tool selection (fastest path first)
 
@@ -58,6 +59,35 @@ Get markdown + links + metadata in a single call:
 a failing format adds a `warnings[]` entry instead of failing the whole call.
 `onlyMainContent` (default `true`) strips boilerplate via Readability.
 
+Query-scoped formats return only the parts of the page that match, verbatim,
+with offsets into the `markdown` of the same call:
+
+```json
+{
+  "tool": "scrape",
+  "params": {
+    "url": "https://example.com/pricing",
+    "formats": [
+      "markdown",
+      { "type": "highlights", "query": "professional plan price", "max_highlights": 5 },
+      { "type": "question", "question": "How much is the professional plan per month?" }
+    ]
+  }
+}
+```
+
+`{ "type": "highlights", "query", "max_highlights"?, "mode"? }` returns
+`content.highlights: [{ text, kind, offset, length, score }]`, best first;
+`kind` is `sentence`, `table_row` or `code_block`, and
+`markdown.slice(offset, offset + length) === text`. `{ "type": "question",
+"question", "mode"? }` returns `content.answer: { text, grounded, evidence }`,
+with `text` the evidence joined and `grounded: true`. Either adds 1 credit once
+per call and calls no model. `"mode": "model"` adds 3 once per call: the model
+chooses the highlights (never rewrites them) or writes the answer, and a
+grounding check sets `grounded: false` when the answer holds a number or a name
+the evidence does not. Ask for `"markdown"` in the same call to quote with a
+locator.
+
 CLI equivalent:
 
 ```bash
@@ -131,6 +161,17 @@ BFS crawl up to depth 5 / 1000 pages. Use `include_patterns` /
 (on by default). Crawls projected over ~500 pages trigger a confirmation
 prompt (elicitation). CLI: `crawlforge crawl https://docs.example.com --depth 3 --max-pages 200`.
 
+## Large results — `max_inline_chars` and `read_result` (cost: 1)
+
+`scrape`, `fetch_url`, `extract_content` and `crawl_deep` accept
+`max_inline_chars` (default 40,000). A larger result comes back as its scalar
+fields plus `preview` (the first `max_inline_chars` characters of its text
+view), `result_handle`, `total_chars`, `truncated: true` and `expires_at`; the
+full result stays 1 hour on the local machine, never uploaded. Read it with
+`read_result` instead of fetching again: `operation:"search"` with a `query`
+finds offsets, then `operation:"slice"` with `offset`/`length` reads around one;
+`operation:"json_path"` with a `path` picks one field (`"lines"` reads by line).
+
 ## Cost note
 
 `scrape` (2 credits) is the default page read: one fetch returns every format
@@ -138,6 +179,7 @@ you ask for. `fetch_url`, `extract_text`, `extract_links`, `extract_metadata`
 (1 each) are cheaper only when one of them is the whole job; `fetch_url` followed
 by an `extract_*` call on the same URL is a double fetch that costs more than one
 `scrape`. `extract_content`, `map_site` = 2 · `crawl_deep` = 4 (grows with
-`max_pages`). Never re-fetch a page whose content is already in the conversation.
+`max_pages`) · `read_result` = 1 (reads a stored result, no fetch). Never re-fetch
+a page whose content is already in the conversation.
 
 See [tool reference](references/tool-reference.md) for the full parameter list.

package/src/skills/agent-skills/crawlforge-web-scraping/references/tool-reference.md

@@ -8,9 +8,10 @@ CrawlForge credits.
 | Param | Type | Default | Notes |
 |-------|------|---------|-------|
 | `url` | string (URL) | — | Required. |
-| `formats` | array | `["markdown"]` | Any of `markdown`, `html`, `rawHtml`, `text`, `links`, `metadata`, `screenshot`, or `{type:"json", schema?, prompt?}`. |
+| `formats` | array | `["markdown"]` | Any of `markdown`, `html`, `rawHtml`, `text`, `links`, `metadata`, `branding`, `screenshot`, `{type:"json", schema?, prompt?}`, `{type:"highlights", query, max_highlights?, mode?}` (the matching sentences, table rows and code blocks, verbatim, with offsets into the markdown; +1 credit once per call) or `{type:"question", question, mode?}` (an answer assembled from the evidence units; same +1). `mode:"model"` adds 3 credits and needs an LLM route. |
 | `onlyMainContent` | boolean | `true` | Strip boilerplate via Readability. |
 | `timeoutMs` | number | `15000` | 1000–60000. |
+| `max_inline_chars` | number | `40000` | 1,000–10,000,000. A larger result returns `preview` + `result_handle` for `read_result` (below). |
 
 Partial success: a format that fails produces a `warnings[]` entry rather than
 failing the whole call. `{type:"json"}` may incur external LLM cost (billed by
@@ -23,6 +24,7 @@ your provider).
 | `url` | string (URL) | — | Required. |
 | `headers` | object | — | Custom HTTP headers (e.g. auth tokens). |
 | `timeout` | number | `10000` | 1000–30000 ms. |
+| `max_inline_chars` | number | `40000` | 1,000–10,000,000. A larger result returns `preview` + `result_handle` for `read_result` (below). |
 
 ## extract_text (cost: 1)
 
@@ -53,6 +55,7 @@ your provider).
 |-------|------|-------|
 | `url` | string (URL) | Required. Readability-cleaned article body (markdown). |
 | `options` | object | Additional extraction options. |
+| `max_inline_chars` | number | Default 40000 (1,000–10,000,000). A larger result returns `preview` + `result_handle` for `read_result` (below). |
 
 ## map_site (cost: 2)
 
@@ -82,10 +85,33 @@ your provider).
 | `concurrency` | number | — | 1–20 concurrent requests. |
 | `enable_link_analysis` | boolean | — | Compute PageRank over crawled pages. |
 | `session` | object | — | Shared cookie jar for login-then-crawl. |
+| `max_inline_chars` | number | `40000` | 1,000–10,000,000. A larger result returns `preview` + `result_handle` for `read_result` (below). |
 
 *Server default honors `RESPECT_ROBOTS_TXT`. Crawls projected over ~500 pages
 trigger an elicitation confirmation.
 
+## read_result (cost: 1)
+
+Reads a result a tool returned with `truncated: true` and a `result_handle`
+(kept 1 hour, on the local machine; an unknown or expired handle is an error).
+
+| Param | Type | Default | Notes |
+|-------|------|---------|-------|
+| `handle` | string | — | Required. The `result_handle` (`res_…`). |
+| `operation` | enum | — | Required. `slice`, `search`, `lines` or `json_path`. |
+| `offset` | number | — | `slice`: character offset; `lines`: first line (default 0). |
+| `length` | number | — | `slice`: characters (default 10,000); `lines`: lines (default 200, at most 5,000). |
+| `query` | string | — | `search`: case-insensitive literal substring. |
+| `max_matches` | number | `20` | `search`: 1–100. |
+| `path` | string | — | `json_path`: path into the stored result object (or the parsed body when the stored text is JSON, e.g. a `fetch_url` body). |
+| `max_inline_chars` | number | `40000` | 1,000–10,000,000. |
+
+Every response carries `handle`, `tool`, `operation`, `view` (`text` or `json`),
+`view_path`, `total_chars` and `expires_at`. `slice` returns `text` + `has_more`;
+`search` returns `matches[{offset, length, context_offset, context}]` (200
+characters of context each side) + `total_matches`; `lines` returns `lines[]`
++ `total_lines`, `char_offset`, `has_more`; `json_path` returns `value`.
+
 ## CLI quick map
 
 | Tool | CLI |