crawlforge-mcp-server 5.6.10 → 5.8.0

package/src/skills/agent-skills/crawlforge-web-scraping/references/tool-reference.md

@@ -8,9 +8,10 @@ CrawlForge credits.
 | Param | Type | Default | Notes |
 |-------|------|---------|-------|
 | `url` | string (URL) | — | Required. |
-| `formats` | array | `["markdown"]` | Any of `markdown`, `html`, `rawHtml`, `text`, `links`, `metadata`, `screenshot`, or `{type:"json", schema?, prompt?}`. |
+| `formats` | array | `["markdown"]` | Any of `markdown`, `html`, `rawHtml`, `text`, `links`, `metadata`, `branding`, `screenshot`, `{type:"json", schema?, prompt?}`, `{type:"highlights", query, max_highlights?, mode?}` (the matching sentences, table rows and code blocks, verbatim, with offsets into the markdown; +1 credit once per call) or `{type:"question", question, mode?}` (an answer assembled from the evidence units; same +1). `mode:"model"` adds 3 credits and needs an LLM route. |
 | `onlyMainContent` | boolean | `true` | Strip boilerplate via Readability. |
 | `timeoutMs` | number | `15000` | 1000–60000. |
+| `max_inline_chars` | number | `40000` | 1,000–10,000,000. A larger result returns `preview` + `result_handle` for `read_result` (below). |
 
 Partial success: a format that fails produces a `warnings[]` entry rather than
 failing the whole call. `{type:"json"}` may incur external LLM cost (billed by
@@ -23,6 +24,7 @@ your provider).
 | `url` | string (URL) | — | Required. |
 | `headers` | object | — | Custom HTTP headers (e.g. auth tokens). |
 | `timeout` | number | `10000` | 1000–30000 ms. |
+| `max_inline_chars` | number | `40000` | 1,000–10,000,000. A larger result returns `preview` + `result_handle` for `read_result` (below). |
 
 ## extract_text (cost: 1)
 
@@ -53,6 +55,7 @@ your provider).
 |-------|------|-------|
 | `url` | string (URL) | Required. Readability-cleaned article body (markdown). |
 | `options` | object | Additional extraction options. |
+| `max_inline_chars` | number | Default 40000 (1,000–10,000,000). A larger result returns `preview` + `result_handle` for `read_result` (below). |
 
 ## map_site (cost: 2)
 
@@ -82,10 +85,33 @@ your provider).
 | `concurrency` | number | — | 1–20 concurrent requests. |
 | `enable_link_analysis` | boolean | — | Compute PageRank over crawled pages. |
 | `session` | object | — | Shared cookie jar for login-then-crawl. |
+| `max_inline_chars` | number | `40000` | 1,000–10,000,000. A larger result returns `preview` + `result_handle` for `read_result` (below). |
 
 *Server default honors `RESPECT_ROBOTS_TXT`. Crawls projected over ~500 pages
 trigger an elicitation confirmation.
 
+## read_result (cost: 1)
+
+Reads a result a tool returned with `truncated: true` and a `result_handle`
+(kept 1 hour, on the local machine; an unknown or expired handle is an error).
+
+| Param | Type | Default | Notes |
+|-------|------|---------|-------|
+| `handle` | string | — | Required. The `result_handle` (`res_…`). |
+| `operation` | enum | — | Required. `slice`, `search`, `lines` or `json_path`. |
+| `offset` | number | — | `slice`: character offset; `lines`: first line (default 0). |
+| `length` | number | — | `slice`: characters (default 10,000); `lines`: lines (default 200, at most 5,000). |
+| `query` | string | — | `search`: case-insensitive literal substring. |
+| `max_matches` | number | `20` | `search`: 1–100. |
+| `path` | string | — | `json_path`: path into the stored result object (or the parsed body when the stored text is JSON, e.g. a `fetch_url` body). |
+| `max_inline_chars` | number | `40000` | 1,000–10,000,000. |
+
+Every response carries `handle`, `tool`, `operation`, `view` (`text` or `json`),
+`view_path`, `total_chars` and `expires_at`. `slice` returns `text` + `has_more`;
+`search` returns `matches[{offset, length, context_offset, context}]` (200
+characters of context each side) + `total_matches`; `lines` returns `lines[]`
++ `total_lines`, `char_offset`, `has_more`; `json_path` returns `value`.
+
 ## CLI quick map
 
 | Tool | CLI |

package/src/tools/advanced/batchScrape/index.js

@@ -18,6 +18,7 @@ import { EventEmitter } from 'events';
 import { ElicitationHelper } from '../../../core/ElicitationHelper.js'; // D1.4
 import JobManager from '../../../core/JobManager.js';
 import WebhookDispatcher from '../../../core/WebhookDispatcher.js';
+import { getResultStore } from '../../../core/ResultStore.js';
 import { BatchScrapeSchema } from './schema.js';
 import { scrapeUrlsBatch, processResults, paginateResults } from './queue.js';
 import { sendWebhookNotification } from './reporter.js';
@@ -35,7 +36,7 @@ export class BatchScrapeTool extends EventEmitter {
       maxBatchSize = 50,
       enableResultCaching = true,
       enableLogging = true,
-      maxCachedBatches = 20
+      resultStore = null
     } = options;
 
     this.jobManager = jobManager || new JobManager({
@@ -54,21 +55,12 @@ export class BatchScrapeTool extends EventEmitter {
     this.enableWebhookNotifications = enableWebhookNotifications;
 
     this.activeBatches = new Map();
-    this.batchResults = new Map();
-    this.maxCachedBatches = maxCachedBatches;
-    this.resultCacheTtl = 3600000; // 1 hour — matches the ttl stored per cache entry
+    // Phase 2: completed batches live in the shared ResultStore (keyed by
+    // batchId) so batch and single results share one eviction budget and TTL.
+    this.resultStore = resultStore || getResultStore();
     // D1.4: Elicitation helper (set mcpServer after instantiation if desired)
     this._elicitation = new ElicitationHelper({});
 
-    // Bound batchResults' lifetime: sweep expired entries periodically (in
-    // addition to the on-read eviction in getBatchResults) so cached results —
-    // including full HTML bodies when formats includes 'html' — don't
-    // accumulate in memory for the life of the process.
-    // .unref() so this timer never blocks process exit on its own — matches
-    // SnapshotManager's cleanupTimer.
-    this._resultsSweepTimer = setInterval(() => this._sweepBatchResults(), 10 * 60 * 1000);
-    if (typeof this._resultsSweepTimer.unref === 'function') this._resultsSweepTimer.unref();
-
     this.stats = {
       totalBatches: 0,
       completedBatches: 0,
@@ -232,20 +224,18 @@ export class BatchScrapeTool extends EventEmitter {
   }
 
   async getBatchResults(batchId, page = 1, pageSize = 25) {
-    const cached = this.batchResults.get(batchId);
+    // An expired or unknown id reads as null (the store evicts on read) and
+    // falls through to the activeBatches/jobManager lookups below.
+    const cached = this.resultStore.get(batchId);
     if (cached) {
-      if (Date.now() - cached.timestamp < cached.ttl) {
-        const offset = (page - 1) * pageSize;
-        return {
-          batchId, success: true, status: 'completed', ...(cached.mode ? { mode: cached.mode } : {}),
-          results: paginateResults(cached.results, offset, pageSize),
-          pagination: { page, pageSize, totalResults: cached.results.length, totalPages: Math.ceil(cached.results.length / pageSize) },
-          cached: true, timestamp: cached.timestamp
-        };
-      }
-      // Expired — evict now instead of waiting for the periodic sweep, and
-      // fall through to the activeBatches/jobManager lookups below.
-      this.batchResults.delete(batchId);
+      const { results, mode } = cached.payload;
+      const offset = (page - 1) * pageSize;
+      return {
+        batchId, success: true, status: 'completed', ...(mode ? { mode } : {}),
+        results: paginateResults(results, offset, pageSize),
+        pagination: { page, pageSize, totalResults: results.length, totalPages: Math.ceil(results.length / pageSize) },
+        cached: true, timestamp: cached.createdAt
+      };
     }
 
     const active = this.activeBatches.get(batchId);
@@ -321,7 +311,7 @@ export class BatchScrapeTool extends EventEmitter {
     return {
       ...this.stats,
       activeBatches: this.activeBatches.size,
-      cachedResults: this.batchResults.size,
+      cachedResults: this._cachedBatchIds().length,
       jobManagerStats: this.jobManager ? this.jobManager.getStats() : null,
       webhookStats: this.webhookDispatcher ? this.webhookDispatcher.getStats() : null
     };
@@ -332,8 +322,7 @@ export class BatchScrapeTool extends EventEmitter {
       try { await this.cancelBatch(batchId); } catch (e) { this._log('warn', `Failed to cancel batch ${batchId}: ${e.message}`); }
     }
     this.activeBatches.clear();
-    this.batchResults.clear();
-    if (this._resultsSweepTimer) clearInterval(this._resultsSweepTimer);
+    for (const batchId of this._cachedBatchIds()) this.resultStore.delete(batchId);
     this.jobManager?.destroy();
     this.webhookDispatcher?.destroy();
     this.removeAllListeners();
@@ -365,23 +354,14 @@ export class BatchScrapeTool extends EventEmitter {
     return this.webhookDispatcher.registerWebhook(webhookConfig.url, config);
   }
 
-  /**
-   * Cache a batch's results, capped to maxCachedBatches. Map preserves
-   * insertion order, so the oldest entry is evicted first (LRU by write time).
-   */
+  /** Cache a batch's results in the shared store under its batchId. */
   _cacheBatchResult(batchId, results, mode) {
-    this.batchResults.set(batchId, { results, mode, timestamp: Date.now(), ttl: this.resultCacheTtl });
-    while (this.batchResults.size > this.maxCachedBatches) {
-      const oldestKey = this.batchResults.keys().next().value;
-      this.batchResults.delete(oldestKey);
-    }
+    this.resultStore.put('batch_scrape', { results, mode }, { key: batchId });
   }
 
-  _sweepBatchResults() {
-    const now = Date.now();
-    for (const [id, entry] of this.batchResults) {
-      if (now - entry.timestamp >= entry.ttl) this.batchResults.delete(id);
-    }
+  /** Live store entries this tool wrote. */
+  _cachedBatchIds() {
+    return this.resultStore.list().filter((entry) => entry.toolName === 'batch_scrape').map((entry) => entry.handle);
   }
 
   _updateAverageBatchTime(batchTime) {

package/src/tools/extract/_fetchAndParse.js

@@ -6,8 +6,8 @@
  *   extractContent.js      (uses native fetch directly but can adopt this)
  *   processDocument.js     (URL sources)
  *
- * Returns { html, $, textContent, finalUrl, warnings } so callers don't repeat
- * the same fetch/cheerio/cleanup boilerplate.
+ * Returns { html, $, textContent, finalUrl, warnings, status } so callers don't
+ * repeat the same fetch/cheerio/cleanup boilerplate.
  */
 
 import { load } from 'cheerio';
@@ -124,7 +124,8 @@ export function flattenBodyText($) {
  * @param {string}   [options.apiKey]         — hashed into the audit row
  * @param {number}   [options.timeoutMs]
  * @param {string[]} [options.stripTags]   — additional tags to strip (default: script, style, noscript, iframe, svg)
- * @returns {Promise<{ html: string, $: import('cheerio').CheerioAPI, textContent: string, finalUrl: string, warnings: string[] }>}
+ * @param {boolean}  [options.errorDocuments] — return a non-2xx HTML/text body with its status instead of throwing
+ * @returns {Promise<{ html: string, $: import('cheerio').CheerioAPI, textContent: string, finalUrl: string, warnings: string[], status: number }>}
  */
 export async function fetchAndParse(url, options = {}) {
   const {
@@ -133,7 +134,8 @@ export async function fetchAndParse(url, options = {}) {
     tool,
     apiKey,
     timeoutMs = DEFAULT_TIMEOUT_MS,
-    stripTags = ['script', 'style', 'noscript', 'iframe', 'svg']
+    stripTags = ['script', 'style', 'noscript', 'iframe', 'svg'],
+    errorDocuments = false
   } = options;
 
   // robots.txt / blocklist gate + per-host throttle. Throws if it refuses.
@@ -148,17 +150,24 @@ export async function fetchAndParse(url, options = {}) {
     signal: AbortSignal.timeout(timeoutMs)
   });
 
+  const contentType = response.headers?.get?.('content-type') || null;
+  const classification = classifyContentType(contentType);
+
   if (!response.ok) {
     // Honour a back-off the host asked for before giving up on this request.
     if (response.status === 429 || response.status === 503) {
       noteRetryAfter(url, response.headers?.get?.('retry-after'));
     }
-    throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+    // A real challenge page reaches a plain fetch as a 403 with the wall in
+    // the body (producthunt.com, travel.state.gov, edmunds.com, 2026-09-05).
+    // scrape opts in to read that body so its verdict can name the vendor
+    // (Phase 0, 0.1); every other caller keeps the throw, and a binary error
+    // body still throws.
+    if (!errorDocuments || classification === 'binary') {
+      throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+    }
   }
 
-  const contentType = response.headers?.get?.('content-type') || null;
-  const classification = classifyContentType(contentType);
-
   if (classification === 'binary') {
     throw new Error(
       `Unsupported content type "${contentType}" — this looks like binary content, not HTML/text. Use process_document for PDFs/documents/binary files.`
@@ -171,7 +180,7 @@ export async function fetchAndParse(url, options = {}) {
   // HTML parser risks misinterpreting substrings (e.g. a "<script>" value
   // inside a JSON string) as real tags and stripping/mangling content.
   if (classification === 'text') {
-    return { html, $: load(''), textContent: html.trim(), finalUrl: response.url, warnings };
+    return { html, $: load(''), textContent: html.trim(), finalUrl: response.url, warnings, status: response.status };
   }
 
   const $ = load(html);
@@ -182,5 +191,5 @@ export async function fetchAndParse(url, options = {}) {
 
   const textContent = flattenBodyText($);
 
-  return { html, $, textContent, finalUrl: response.url, warnings };
+  return { html, $, textContent, finalUrl: response.url, warnings, status: response.status };
 }

package/src/tools/result/readResult.js

@@ -0,0 +1,147 @@
+/**
+ * read_result — read a stored result by its handle (Phase 2).
+ *
+ * A tool whose result passed max_inline_chars returned a preview and a
+ * result_handle; the whole result is in the ResultStore for an hour. This
+ * tool slices, searches, paginates by line or reads a JSON path out of it,
+ * so the caller never has to fetch the page again. Extractive only: every
+ * operation returns verbatim text with offsets into the same view the
+ * preview was cut from (G3).
+ */
+
+import { z } from 'zod';
+import { selectJsonPath } from 'crawlforge-extractors';
+import { getResultStore, RESULT_HANDLE_PATTERN } from '../../core/ResultStore.js';
+import { MAX_INLINE_CHARS_PARAM, resolveMaxInlineChars, resultTextView } from '../../server/inlineThreshold.js';
+import { dualOutput } from '../../server/registerTool.js';
+
+const CONTEXT_CHARS = 200;
+const DEFAULT_SLICE_LENGTH = 10000;
+const DEFAULT_LINE_COUNT = 200;
+const MAX_LINE_COUNT = 5000;
+
+export const READ_RESULT_INPUT_SHAPE = {
+  handle: z.string().regex(RESULT_HANDLE_PATTERN).describe("The result_handle a truncated result returned (res_… or a batch id)"),
+  operation: z.enum(['slice', 'search', 'lines', 'json_path']).describe("slice: characters from offset; search: case-insensitive literal query with context and offsets; lines: a page of lines; json_path: one subtree of a JSON result"),
+  offset: z.number().int().min(0).optional().describe("slice: first character (default 0); lines: first line index (default 0)"),
+  length: z.number().int().min(1).max(200000).optional().describe("slice: characters to return (default 10,000); lines: lines to return (default 200, max 5,000)"),
+  query: z.string().min(1).max(500).optional().describe("search: the text to find, matched literally, case-insensitive"),
+  max_matches: z.number().int().min(1).max(100).optional().default(20).describe("search: matches to return (default 20)"),
+  path: z.string().optional().describe("json_path: dotted keys and array indexes, e.g. \"results[3].content\" — not JSONPath"),
+  ...MAX_INLINE_CHARS_PARAM
+};
+
+const fail = (text) => ({ content: [{ type: 'text', text }], isError: true });
+// read_result declares an outputSchema, so a success carries structuredContent too.
+const ok = (object) => dualOutput(object);
+
+/**
+ * @param {{ handle: string, operation: string, offset?: number, length?: number, query?: string, max_matches?: number, path?: string, max_inline_chars?: number }} params
+ */
+export async function readResultHandler(params) {
+  const { handle, operation } = params;
+  const entry = getResultStore().get(handle);
+  if (!entry) return fail('Unknown or expired result handle (results are kept 1 hour)');
+
+  const maxChars = resolveMaxInlineChars(params);
+  const { view, view_path, text } = resultTextView(entry.payload, entry.meta?.view_path ? [entry.meta.view_path] : []);
+  const base = {
+    handle,
+    tool: entry.toolName,
+    operation,
+    view,
+    view_path,
+    total_chars: text.length,
+    expires_at: new Date(entry.expiresAt).toISOString()
+  };
+
+  switch (operation) {
+    case 'slice': {
+      const offset = params.offset ?? 0;
+      const length = Math.min(params.length ?? DEFAULT_SLICE_LENGTH, maxChars);
+      const slice = text.slice(offset, offset + length);
+      return ok({ ...base, offset, length: slice.length, text: slice, has_more: offset + slice.length < text.length });
+    }
+
+    case 'search': {
+      const { query } = params;
+      if (!query) return fail('query is required for operation "search"');
+      const maxMatches = params.max_matches ?? 20;
+      // A literal (escaped) case-insensitive regex over the ORIGINAL text:
+      // offsets must index the verbatim string, and lowercasing a copy can
+      // change its length (Turkish İ lowercases to two code units).
+      const literal = new RegExp(query.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'gi');
+      const matches = [];
+      let total = 0;
+      let budget = maxChars;
+      for (let m = literal.exec(text); m !== null; m = literal.exec(text)) {
+        if (m[0].length === 0) literal.lastIndex++;
+        total++;
+        if (matches.length >= maxMatches) continue;
+        const contextOffset = Math.max(0, m.index - CONTEXT_CHARS);
+        const context = text.slice(contextOffset, m.index + m[0].length + CONTEXT_CHARS);
+        if (context.length > budget) continue;
+        budget -= context.length;
+        matches.push({ offset: m.index, length: m[0].length, context_offset: contextOffset, context });
+      }
+      return ok({ ...base, query, matches, total_matches: total, truncated: total > matches.length });
+    }
+
+    case 'lines': {
+      const all = text.split('\n');
+      const first = params.offset ?? 0;
+      const count = Math.min(params.length ?? DEFAULT_LINE_COUNT, MAX_LINE_COUNT);
+      let charOffset = 0;
+      for (let i = 0; i < Math.min(first, all.length); i++) charOffset += all[i].length + 1;
+      const lines = [];
+      let chars = 0;
+      for (const line of all.slice(first, first + count)) {
+        if (chars + line.length + 1 > maxChars && lines.length > 0) break;
+        lines.push(line);
+        chars += line.length + 1;
+      }
+      return ok({
+        ...base,
+        first_line: first,
+        line_count: lines.length,
+        total_lines: all.length,
+        char_offset: charOffset,
+        lines,
+        has_more: first + lines.length < all.length
+      });
+    }
+
+    case 'json_path': {
+      const { path } = params;
+      if (!path) return fail('path is required for operation "json_path"');
+      // A text view whose text is itself JSON (a fetch_url body) is read as
+      // that JSON; otherwise the stored result object is the subject.
+      let subject = entry.payload;
+      if (view === 'text') {
+        try { subject = JSON.parse(text); } catch { /* not JSON: address the result object */ }
+      }
+      let value;
+      try {
+        value = selectJsonPath(subject, path);
+      } catch (error) {
+        return fail(error.message);
+      }
+      const valueJson = JSON.stringify(value) ?? 'null';
+      if (valueJson.length > maxChars) {
+        return ok({
+          ...base,
+          path,
+          value: null,
+          value_chars: valueJson.length,
+          preview: JSON.stringify(value, null, 2).slice(0, maxChars),
+          truncated: true,
+          warnings: [`Value at "${path}" is ${valueJson.length} chars, over max_inline_chars ${maxChars}; narrow the path to a smaller subtree.`]
+        });
+      }
+      return ok({ ...base, path, value, value_chars: valueJson.length });
+    }
+
+    default:
+      return fail(`Unknown operation "${operation}"`);
+  }
+}

package/src/tools/scrape/_highlights.js

@@ -0,0 +1,79 @@
+/**
+ * _highlights.js — the pure pieces of `scrape`'s query-scoped formats
+ * (Phase 1): the public unit shape, the parser for a model's excerpt
+ * choice, and the grounding check a model-mode answer must pass.
+ *
+ * Nothing here touches the network or a model, so every rule is testable
+ * without one.
+ */
+
+/**
+ * The unit shape a caller sees. `heading` is a ranking input
+ * (crawlforge-extractors counts its terms at half weight), not an output.
+ * @param {{ text: string, kind: string, offset: number, length: number, score: number }} unit
+ */
+export function toPublicUnit({ text, kind, offset, length, score }) {
+  return { text, kind, offset, length, score };
+}
+
+/**
+ * The candidate indexes a model chose, from a reply such as "3, 0, 7" or
+ * "[3] and [7]". Out-of-range and repeated numbers are dropped; at most
+ * `limit` survive, in the model's order. Empty when nothing parses, so the
+ * caller can fall back to the extractive order.
+ *
+ * @param {string} reply
+ * @param {number} count how many candidates were offered (indexes 0..count-1)
+ * @param {number} limit
+ * @returns {number[]}
+ */
+export function parseChosenIndexes(reply, count, limit) {
+  const chosen = [];
+  for (const match of String(reply ?? '').matchAll(/\d+/g)) {
+    const index = Number(match[0]);
+    if (index < count && !chosen.includes(index)) chosen.push(index);
+    if (chosen.length >= limit) break;
+  }
+  return chosen;
+}
+
+const NUMBER = /\d[\d.,:%]*/g;
+// A capitalised word of two or more letters. Letters only: "Node.js" is
+// "Node" (checked) and "js" (not capitalised).
+const WORD = /\p{L}{2,}/gu;
+// What precedes the first word of a sentence: nothing, or a terminator.
+const SENTENCE_START = /(?:^|[.!?]|\n)[\s"'“‘(\[*_]*$/;
+
+/**
+ * Whether a model-written answer stays inside its evidence. Every number
+ * and every proper noun in `answer` — a capitalised word of at least two
+ * letters that is not the first word of a sentence — must appear,
+ * case-insensitively, in the evidence or in the question. A price the page
+ * never states or a vendor it never names is what "grounded: false" is for.
+ *
+ * @param {string} answer
+ * @param {string} evidence the evidence units' text
+ * @param {string} question
+ * @returns {{ grounded: boolean, unbacked: string[] }}
+ */
+export function groundingCheck(answer, evidence, question) {
+  const text = String(answer ?? '');
+  const haystack = `${evidence ?? ''}\n${question ?? ''}`.toLowerCase();
+  const unbacked = [];
+  const check = (token) => {
+    if (!haystack.includes(token.toLowerCase()) && !unbacked.includes(token)) unbacked.push(token);
+  };
+
+  for (const match of text.matchAll(NUMBER)) {
+    // "$49." at the end of a sentence is the number 49.
+    check(match[0].replace(/[.,:]+$/, ''));
+  }
+  for (const match of text.matchAll(WORD)) {
+    const word = match[0];
+    if (word[0] !== word[0].toUpperCase() || word[0] === word[0].toLowerCase()) continue;
+    if (SENTENCE_START.test(text.slice(0, match.index))) continue;
+    check(word);
+  }
+
+  return { grounded: unbacked.length === 0, unbacked };
+}

package/src/tools/scrape/formats.js

@@ -0,0 +1,69 @@
+/**
+ * formats.js — the one declaration of what `scrape` can return.
+ *
+ * The formats enum used to be spelled in server.js, unifiedScrape.js and
+ * toolOutputSchemas.js separately, so a format added to one copy shipped a
+ * schema that did not describe the tool (Phase 0, 0.3). server.js and the
+ * output schema import from here; the tool module re-exports.
+ *
+ * The query-scoped formats (Phase 1) carry their price rule here too, so
+ * AuthManager.getToolCost and the tool read the same one.
+ */
+
+import { z } from 'zod';
+
+export const SCRAPE_STRING_FORMATS = ['markdown', 'html', 'rawHtml', 'text', 'links', 'metadata', 'screenshot', 'branding'];
+
+export const JsonFormatSchema = z.object({
+  type: z.literal('json'),
+  schema: z.record(z.any()).optional().describe('JSON schema for extraction'),
+  prompt: z.string().optional().describe('Extraction instruction for the LLM')
+});
+
+// "extractive" returns page text verbatim and calls no model; "model" lets a
+// model choose among (highlights) or answer from (question) the extractive
+// units, and is priced separately.
+const QueryModeSchema = z.enum(['extractive', 'model']).optional().default('extractive')
+  .describe('"extractive" (default) returns verbatim page text, no model; "model" adds an LLM step (+3 credits)');
+
+export const HighlightsFormatSchema = z.object({
+  type: z.literal('highlights'),
+  query: z.string().min(1).max(500).describe('What to look for; the matching sentences, table rows and code blocks come back verbatim with offsets into the markdown'),
+  max_highlights: z.number().int().min(1).max(50).optional().default(10).describe('How many units to return (default 10)'),
+  mode: QueryModeSchema
+});
+
+export const QuestionFormatSchema = z.object({
+  type: z.literal('question'),
+  question: z.string().min(1).max(500).describe('The question to answer from the page; the evidence units come back verbatim with offsets'),
+  mode: QueryModeSchema
+});
+
+const OBJECT_FORMAT_SCHEMAS = [JsonFormatSchema, HighlightsFormatSchema, QuestionFormatSchema];
+
+/** The `type` of each object format, in schema order. */
+export const SCRAPE_OBJECT_FORMATS = OBJECT_FORMAT_SCHEMAS.map((schema) => schema.shape.type.value);
+
+export const FormatSchema = z.union([
+  z.enum(SCRAPE_STRING_FORMATS),
+  ...OBJECT_FORMAT_SCHEMAS
+]);
+
+const QUERY_FORMATS = new Set(['highlights', 'question']);
+
+/**
+ * What the query-scoped formats add to `scrape`'s base price: 1 credit once
+ * per call when any `highlights` or `question` format is present, 3 more
+ * once when any of them asks for `mode: "model"`. Reads the raw params
+ * (getToolCost runs before validation), so anything that is not an array
+ * of formats prices as none.
+ *
+ * @param {unknown} formats
+ * @returns {{ query: 0 | 1, model: 0 | 3 }}
+ */
+export function scrapeFormatSurcharge(formats) {
+  if (!Array.isArray(formats)) return { query: 0, model: 0 };
+  const queryFormats = formats.filter((fmt) => fmt && typeof fmt === 'object' && QUERY_FORMATS.has(fmt.type));
+  if (queryFormats.length === 0) return { query: 0, model: 0 };
+  return { query: 1, model: queryFormats.some((fmt) => fmt.mode === 'model') ? 3 : 0 };
+}