crawlforge-mcp-server 5.6.11 → 5.8.0

package/src/tools/advanced/batchScrape/index.js

@@ -18,6 +18,7 @@ import { EventEmitter } from 'events';
 import { ElicitationHelper } from '../../../core/ElicitationHelper.js'; // D1.4
 import JobManager from '../../../core/JobManager.js';
 import WebhookDispatcher from '../../../core/WebhookDispatcher.js';
+import { getResultStore } from '../../../core/ResultStore.js';
 import { BatchScrapeSchema } from './schema.js';
 import { scrapeUrlsBatch, processResults, paginateResults } from './queue.js';
 import { sendWebhookNotification } from './reporter.js';
@@ -35,7 +36,7 @@ export class BatchScrapeTool extends EventEmitter {
       maxBatchSize = 50,
       enableResultCaching = true,
       enableLogging = true,
-      maxCachedBatches = 20
+      resultStore = null
     } = options;
 
     this.jobManager = jobManager || new JobManager({
@@ -54,21 +55,12 @@ export class BatchScrapeTool extends EventEmitter {
     this.enableWebhookNotifications = enableWebhookNotifications;
 
     this.activeBatches = new Map();
-    this.batchResults = new Map();
-    this.maxCachedBatches = maxCachedBatches;
-    this.resultCacheTtl = 3600000; // 1 hour — matches the ttl stored per cache entry
+    // Phase 2: completed batches live in the shared ResultStore (keyed by
+    // batchId) so batch and single results share one eviction budget and TTL.
+    this.resultStore = resultStore || getResultStore();
     // D1.4: Elicitation helper (set mcpServer after instantiation if desired)
     this._elicitation = new ElicitationHelper({});
 
-    // Bound batchResults' lifetime: sweep expired entries periodically (in
-    // addition to the on-read eviction in getBatchResults) so cached results —
-    // including full HTML bodies when formats includes 'html' — don't
-    // accumulate in memory for the life of the process.
-    // .unref() so this timer never blocks process exit on its own — matches
-    // SnapshotManager's cleanupTimer.
-    this._resultsSweepTimer = setInterval(() => this._sweepBatchResults(), 10 * 60 * 1000);
-    if (typeof this._resultsSweepTimer.unref === 'function') this._resultsSweepTimer.unref();
-
     this.stats = {
       totalBatches: 0,
       completedBatches: 0,
@@ -232,20 +224,18 @@ export class BatchScrapeTool extends EventEmitter {
   }
 
   async getBatchResults(batchId, page = 1, pageSize = 25) {
-    const cached = this.batchResults.get(batchId);
+    // An expired or unknown id reads as null (the store evicts on read) and
+    // falls through to the activeBatches/jobManager lookups below.
+    const cached = this.resultStore.get(batchId);
     if (cached) {
-      if (Date.now() - cached.timestamp < cached.ttl) {
-        const offset = (page - 1) * pageSize;
-        return {
-          batchId, success: true, status: 'completed', ...(cached.mode ? { mode: cached.mode } : {}),
-          results: paginateResults(cached.results, offset, pageSize),
-          pagination: { page, pageSize, totalResults: cached.results.length, totalPages: Math.ceil(cached.results.length / pageSize) },
-          cached: true, timestamp: cached.timestamp
-        };
-      }
-      // Expired — evict now instead of waiting for the periodic sweep, and
-      // fall through to the activeBatches/jobManager lookups below.
-      this.batchResults.delete(batchId);
+      const { results, mode } = cached.payload;
+      const offset = (page - 1) * pageSize;
+      return {
+        batchId, success: true, status: 'completed', ...(mode ? { mode } : {}),
+        results: paginateResults(results, offset, pageSize),
+        pagination: { page, pageSize, totalResults: results.length, totalPages: Math.ceil(results.length / pageSize) },
+        cached: true, timestamp: cached.createdAt
+      };
     }
 
     const active = this.activeBatches.get(batchId);
@@ -321,7 +311,7 @@ export class BatchScrapeTool extends EventEmitter {
     return {
       ...this.stats,
       activeBatches: this.activeBatches.size,
-      cachedResults: this.batchResults.size,
+      cachedResults: this._cachedBatchIds().length,
       jobManagerStats: this.jobManager ? this.jobManager.getStats() : null,
       webhookStats: this.webhookDispatcher ? this.webhookDispatcher.getStats() : null
     };
@@ -332,8 +322,7 @@ export class BatchScrapeTool extends EventEmitter {
       try { await this.cancelBatch(batchId); } catch (e) { this._log('warn', `Failed to cancel batch ${batchId}: ${e.message}`); }
     }
     this.activeBatches.clear();
-    this.batchResults.clear();
-    if (this._resultsSweepTimer) clearInterval(this._resultsSweepTimer);
+    for (const batchId of this._cachedBatchIds()) this.resultStore.delete(batchId);
     this.jobManager?.destroy();
     this.webhookDispatcher?.destroy();
     this.removeAllListeners();
@@ -365,23 +354,14 @@ export class BatchScrapeTool extends EventEmitter {
     return this.webhookDispatcher.registerWebhook(webhookConfig.url, config);
   }
 
-  /**
-   * Cache a batch's results, capped to maxCachedBatches. Map preserves
-   * insertion order, so the oldest entry is evicted first (LRU by write time).
-   */
+  /** Cache a batch's results in the shared store under its batchId. */
   _cacheBatchResult(batchId, results, mode) {
-    this.batchResults.set(batchId, { results, mode, timestamp: Date.now(), ttl: this.resultCacheTtl });
-    while (this.batchResults.size > this.maxCachedBatches) {
-      const oldestKey = this.batchResults.keys().next().value;
-      this.batchResults.delete(oldestKey);
-    }
+    this.resultStore.put('batch_scrape', { results, mode }, { key: batchId });
   }
 
-  _sweepBatchResults() {
-    const now = Date.now();
-    for (const [id, entry] of this.batchResults) {
-      if (now - entry.timestamp >= entry.ttl) this.batchResults.delete(id);
-    }
+  /** Live store entries this tool wrote. */
+  _cachedBatchIds() {
+    return this.resultStore.list().filter((entry) => entry.toolName === 'batch_scrape').map((entry) => entry.handle);
   }
 
   _updateAverageBatchTime(batchTime) {

package/src/tools/result/readResult.js

@@ -0,0 +1,147 @@
+/**
+ * read_result — read a stored result by its handle (Phase 2).
+ *
+ * A tool whose result passed max_inline_chars returned a preview and a
+ * result_handle; the whole result is in the ResultStore for an hour. This
+ * tool slices, searches, paginates by line or reads a JSON path out of it,
+ * so the caller never has to fetch the page again. Extractive only: every
+ * operation returns verbatim text with offsets into the same view the
+ * preview was cut from (G3).
+ */
+
+import { z } from 'zod';
+import { selectJsonPath } from 'crawlforge-extractors';
+import { getResultStore, RESULT_HANDLE_PATTERN } from '../../core/ResultStore.js';
+import { MAX_INLINE_CHARS_PARAM, resolveMaxInlineChars, resultTextView } from '../../server/inlineThreshold.js';
+import { dualOutput } from '../../server/registerTool.js';
+
+const CONTEXT_CHARS = 200;
+const DEFAULT_SLICE_LENGTH = 10000;
+const DEFAULT_LINE_COUNT = 200;
+const MAX_LINE_COUNT = 5000;
+
+export const READ_RESULT_INPUT_SHAPE = {
+  handle: z.string().regex(RESULT_HANDLE_PATTERN).describe("The result_handle a truncated result returned (res_… or a batch id)"),
+  operation: z.enum(['slice', 'search', 'lines', 'json_path']).describe("slice: characters from offset; search: case-insensitive literal query with context and offsets; lines: a page of lines; json_path: one subtree of a JSON result"),
+  offset: z.number().int().min(0).optional().describe("slice: first character (default 0); lines: first line index (default 0)"),
+  length: z.number().int().min(1).max(200000).optional().describe("slice: characters to return (default 10,000); lines: lines to return (default 200, max 5,000)"),
+  query: z.string().min(1).max(500).optional().describe("search: the text to find, matched literally, case-insensitive"),
+  max_matches: z.number().int().min(1).max(100).optional().default(20).describe("search: matches to return (default 20)"),
+  path: z.string().optional().describe("json_path: dotted keys and array indexes, e.g. \"results[3].content\" — not JSONPath"),
+  ...MAX_INLINE_CHARS_PARAM
+};
+
+const fail = (text) => ({ content: [{ type: 'text', text }], isError: true });
+// read_result declares an outputSchema, so a success carries structuredContent too.
+const ok = (object) => dualOutput(object);
+
+/**
+ * @param {{ handle: string, operation: string, offset?: number, length?: number, query?: string, max_matches?: number, path?: string, max_inline_chars?: number }} params
+ */
+export async function readResultHandler(params) {
+  const { handle, operation } = params;
+  const entry = getResultStore().get(handle);
+  if (!entry) return fail('Unknown or expired result handle (results are kept 1 hour)');
+
+  const maxChars = resolveMaxInlineChars(params);
+  const { view, view_path, text } = resultTextView(entry.payload, entry.meta?.view_path ? [entry.meta.view_path] : []);
+  const base = {
+    handle,
+    tool: entry.toolName,
+    operation,
+    view,
+    view_path,
+    total_chars: text.length,
+    expires_at: new Date(entry.expiresAt).toISOString()
+  };
+
+  switch (operation) {
+    case 'slice': {
+      const offset = params.offset ?? 0;
+      const length = Math.min(params.length ?? DEFAULT_SLICE_LENGTH, maxChars);
+      const slice = text.slice(offset, offset + length);
+      return ok({ ...base, offset, length: slice.length, text: slice, has_more: offset + slice.length < text.length });
+    }
+
+    case 'search': {
+      const { query } = params;
+      if (!query) return fail('query is required for operation "search"');
+      const maxMatches = params.max_matches ?? 20;
+      // A literal (escaped) case-insensitive regex over the ORIGINAL text:
+      // offsets must index the verbatim string, and lowercasing a copy can
+      // change its length (Turkish İ lowercases to two code units).
+      const literal = new RegExp(query.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'gi');
+      const matches = [];
+      let total = 0;
+      let budget = maxChars;
+      for (let m = literal.exec(text); m !== null; m = literal.exec(text)) {
+        if (m[0].length === 0) literal.lastIndex++;
+        total++;
+        if (matches.length >= maxMatches) continue;
+        const contextOffset = Math.max(0, m.index - CONTEXT_CHARS);
+        const context = text.slice(contextOffset, m.index + m[0].length + CONTEXT_CHARS);
+        if (context.length > budget) continue;
+        budget -= context.length;
+        matches.push({ offset: m.index, length: m[0].length, context_offset: contextOffset, context });
+      }
+      return ok({ ...base, query, matches, total_matches: total, truncated: total > matches.length });
+    }
+
+    case 'lines': {
+      const all = text.split('\n');
+      const first = params.offset ?? 0;
+      const count = Math.min(params.length ?? DEFAULT_LINE_COUNT, MAX_LINE_COUNT);
+      let charOffset = 0;
+      for (let i = 0; i < Math.min(first, all.length); i++) charOffset += all[i].length + 1;
+      const lines = [];
+      let chars = 0;
+      for (const line of all.slice(first, first + count)) {
+        if (chars + line.length + 1 > maxChars && lines.length > 0) break;
+        lines.push(line);
+        chars += line.length + 1;
+      }
+      return ok({
+        ...base,
+        first_line: first,
+        line_count: lines.length,
+        total_lines: all.length,
+        char_offset: charOffset,
+        lines,
+        has_more: first + lines.length < all.length
+      });
+    }
+
+    case 'json_path': {
+      const { path } = params;
+      if (!path) return fail('path is required for operation "json_path"');
+      // A text view whose text is itself JSON (a fetch_url body) is read as
+      // that JSON; otherwise the stored result object is the subject.
+      let subject = entry.payload;
+      if (view === 'text') {
+        try { subject = JSON.parse(text); } catch { /* not JSON: address the result object */ }
+      }
+      let value;
+      try {
+        value = selectJsonPath(subject, path);
+      } catch (error) {
+        return fail(error.message);
+      }
+      const valueJson = JSON.stringify(value) ?? 'null';
+      if (valueJson.length > maxChars) {
+        return ok({
+          ...base,
+          path,
+          value: null,
+          value_chars: valueJson.length,
+          preview: JSON.stringify(value, null, 2).slice(0, maxChars),
+          truncated: true,
+          warnings: [`Value at "${path}" is ${valueJson.length} chars, over max_inline_chars ${maxChars}; narrow the path to a smaller subtree.`]
+        });
+      }
+      return ok({ ...base, path, value, value_chars: valueJson.length });
+    }
+
+    default:
+      return fail(`Unknown operation "${operation}"`);
+  }
+}

package/src/tools/scrape/_highlights.js

@@ -0,0 +1,79 @@
+/**
+ * _highlights.js — the pure pieces of `scrape`'s query-scoped formats
+ * (Phase 1): the public unit shape, the parser for a model's excerpt
+ * choice, and the grounding check a model-mode answer must pass.
+ *
+ * Nothing here touches the network or a model, so every rule is testable
+ * without one.
+ */
+
+/**
+ * The unit shape a caller sees. `heading` is a ranking input
+ * (crawlforge-extractors counts its terms at half weight), not an output.
+ * @param {{ text: string, kind: string, offset: number, length: number, score: number }} unit
+ */
+export function toPublicUnit({ text, kind, offset, length, score }) {
+  return { text, kind, offset, length, score };
+}
+
+/**
+ * The candidate indexes a model chose, from a reply such as "3, 0, 7" or
+ * "[3] and [7]". Out-of-range and repeated numbers are dropped; at most
+ * `limit` survive, in the model's order. Empty when nothing parses, so the
+ * caller can fall back to the extractive order.
+ *
+ * @param {string} reply
+ * @param {number} count how many candidates were offered (indexes 0..count-1)
+ * @param {number} limit
+ * @returns {number[]}
+ */
+export function parseChosenIndexes(reply, count, limit) {
+  const chosen = [];
+  for (const match of String(reply ?? '').matchAll(/\d+/g)) {
+    const index = Number(match[0]);
+    if (index < count && !chosen.includes(index)) chosen.push(index);
+    if (chosen.length >= limit) break;
+  }
+  return chosen;
+}
+
+const NUMBER = /\d[\d.,:%]*/g;
+// A capitalised word of two or more letters. Letters only: "Node.js" is
+// "Node" (checked) and "js" (not capitalised).
+const WORD = /\p{L}{2,}/gu;
+// What precedes the first word of a sentence: nothing, or a terminator.
+const SENTENCE_START = /(?:^|[.!?]|\n)[\s"'“‘(\[*_]*$/;
+
+/**
+ * Whether a model-written answer stays inside its evidence. Every number
+ * and every proper noun in `answer` — a capitalised word of at least two
+ * letters that is not the first word of a sentence — must appear,
+ * case-insensitively, in the evidence or in the question. A price the page
+ * never states or a vendor it never names is what "grounded: false" is for.
+ *
+ * @param {string} answer
+ * @param {string} evidence the evidence units' text
+ * @param {string} question
+ * @returns {{ grounded: boolean, unbacked: string[] }}
+ */
+export function groundingCheck(answer, evidence, question) {
+  const text = String(answer ?? '');
+  const haystack = `${evidence ?? ''}\n${question ?? ''}`.toLowerCase();
+  const unbacked = [];
+  const check = (token) => {
+    if (!haystack.includes(token.toLowerCase()) && !unbacked.includes(token)) unbacked.push(token);
+  };
+
+  for (const match of text.matchAll(NUMBER)) {
+    // "$49." at the end of a sentence is the number 49.
+    check(match[0].replace(/[.,:]+$/, ''));
+  }
+  for (const match of text.matchAll(WORD)) {
+    const word = match[0];
+    if (word[0] !== word[0].toUpperCase() || word[0] === word[0].toLowerCase()) continue;
+    if (SENTENCE_START.test(text.slice(0, match.index))) continue;
+    check(word);
+  }
+
+  return { grounded: unbacked.length === 0, unbacked };
+}

package/src/tools/scrape/formats.js

@@ -5,6 +5,9 @@
  * toolOutputSchemas.js separately, so a format added to one copy shipped a
  * schema that did not describe the tool (Phase 0, 0.3). server.js and the
  * output schema import from here; the tool module re-exports.
+ *
+ * The query-scoped formats (Phase 1) carry their price rule here too, so
+ * AuthManager.getToolCost and the tool read the same one.
  */
 
 import { z } from 'zod';
@@ -17,7 +20,50 @@ export const JsonFormatSchema = z.object({
   prompt: z.string().optional().describe('Extraction instruction for the LLM')
 });
 
+// "extractive" returns page text verbatim and calls no model; "model" lets a
+// model choose among (highlights) or answer from (question) the extractive
+// units, and is priced separately.
+const QueryModeSchema = z.enum(['extractive', 'model']).optional().default('extractive')
+  .describe('"extractive" (default) returns verbatim page text, no model; "model" adds an LLM step (+3 credits)');
+
+export const HighlightsFormatSchema = z.object({
+  type: z.literal('highlights'),
+  query: z.string().min(1).max(500).describe('What to look for; the matching sentences, table rows and code blocks come back verbatim with offsets into the markdown'),
+  max_highlights: z.number().int().min(1).max(50).optional().default(10).describe('How many units to return (default 10)'),
+  mode: QueryModeSchema
+});
+
+export const QuestionFormatSchema = z.object({
+  type: z.literal('question'),
+  question: z.string().min(1).max(500).describe('The question to answer from the page; the evidence units come back verbatim with offsets'),
+  mode: QueryModeSchema
+});
+
+const OBJECT_FORMAT_SCHEMAS = [JsonFormatSchema, HighlightsFormatSchema, QuestionFormatSchema];
+
+/** The `type` of each object format, in schema order. */
+export const SCRAPE_OBJECT_FORMATS = OBJECT_FORMAT_SCHEMAS.map((schema) => schema.shape.type.value);
+
 export const FormatSchema = z.union([
   z.enum(SCRAPE_STRING_FORMATS),
-  JsonFormatSchema
+  ...OBJECT_FORMAT_SCHEMAS
 ]);
+
+const QUERY_FORMATS = new Set(['highlights', 'question']);
+
+/**
+ * What the query-scoped formats add to `scrape`'s base price: 1 credit once
+ * per call when any `highlights` or `question` format is present, 3 more
+ * once when any of them asks for `mode: "model"`. Reads the raw params
+ * (getToolCost runs before validation), so anything that is not an array
+ * of formats prices as none.
+ *
+ * @param {unknown} formats
+ * @returns {{ query: 0 | 1, model: 0 | 3 }}
+ */
+export function scrapeFormatSurcharge(formats) {
+  if (!Array.isArray(formats)) return { query: 0, model: 0 };
+  const queryFormats = formats.filter((fmt) => fmt && typeof fmt === 'object' && QUERY_FORMATS.has(fmt.type));
+  if (queryFormats.length === 0) return { query: 0, model: 0 };
+  return { query: 1, model: queryFormats.some((fmt) => fmt.mode === 'model') ? 3 : 0 };
+}

package/src/tools/scrape/unifiedScrape.js

@@ -5,6 +5,9 @@
  * Mirrors the output shape of ScrapeWithActionsTool.generateFormats():
  *   content.html, content.rawHtml, content.text, content.markdown,
  *   content.links, content.metadata, content.screenshots, content.json
+ * plus the query-scoped formats (Phase 1): content.highlights for
+ * {type:"highlights"} and content.answer for {type:"question"}, both built
+ * from the markdown this same call produces, verbatim with offsets into it.
  *
  * onlyMainContent maps to Readability boilerplate removal (same as extractContent).
  * Partial success: per-format warnings[] never fail the whole call.
@@ -12,8 +15,14 @@
 
 import { z } from 'zod';
 import { load } from 'cheerio';
-import { documentVerdict } from 'crawlforge-extractors';
-import { SCRAPE_STRING_FORMATS, JsonFormatSchema, FormatSchema } from './formats.js';
+import { documentVerdict, segmentUnits, rankUnits } from 'crawlforge-extractors';
+import {
+  SCRAPE_STRING_FORMATS, JsonFormatSchema, HighlightsFormatSchema, QuestionFormatSchema, FormatSchema,
+  scrapeFormatSurcharge
+} from './formats.js';
+import { toPublicUnit, parseChosenIndexes, groundingCheck } from './_highlights.js';
+import { setActualCost } from '../../server/requestContext.js';
+import { fenceUntrusted } from '../../utils/untrustedContent.js';
 import { fetchAndParse } from '../extract/_fetchAndParse.js';
 import { extractMainContent, isThinMainContent } from './_mainContent.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
@@ -24,7 +33,15 @@ import { noteHostBlocked, clearHostBlocked } from '../../utils/hostRateLimiter.j
 
 // ── Schema ────────────────────────────────────────────────────────────────────
 
-export { SCRAPE_STRING_FORMATS, JsonFormatSchema, FormatSchema };
+export { SCRAPE_STRING_FORMATS, JsonFormatSchema, HighlightsFormatSchema, QuestionFormatSchema, FormatSchema };
+
+// Above this, a markdown result is the whole page in the context window;
+// the warning names the format that returns only the matching units (1.4).
+const OVERSIZED_MARKDOWN_CHARS = 40000;
+// How many extractive units a question's answer rests on.
+const QUESTION_EVIDENCE_UNITS = 5;
+
+const isQueryFormat = (fmt) => Boolean(fmt) && typeof fmt === 'object' && (fmt.type === 'highlights' || fmt.type === 'question');
 
 // The six public fields in tools/list order, each carrying the description
 // the client sees. server.js spreads this with COMPLIANCE_PARAMS instead of
@@ -177,6 +194,61 @@ export class UnifiedScrapeTool {
     // Optional shared ActionExecutor (injected from server.js so we reuse the
     // existing browser pool rather than spinning up a second one).
     this._actionExecutor = options.actionExecutor || null;
+    this._mcpServer = null;
+  }
+
+  /** Wire the MCP server so the mode:"model" step can fall back to client sampling. */
+  setMcpServer(mcpServer) {
+    this._mcpServer = mcpServer;
+  }
+
+  /**
+   * One completion through the SamplingClient chain (Ollama → server keys →
+   * MCP sampling); throws when no route exists. Page text must already be
+   * fenced by the caller.
+   */
+  async _complete(prompt, options) {
+    const { SamplingClient } = await import('../../core/SamplingClient.js');
+    return new SamplingClient({ mcpServer: this._mcpServer }).complete(prompt, options);
+  }
+
+  /**
+   * mode:"model" for highlights: the model picks which of the extractive
+   * candidates to keep; it never writes text. Returns the chosen units in
+   * extractive order, or the extractive top `limit` when the reply does not
+   * name any candidate.
+   */
+  async _chooseHighlights(candidates, query, limit, warnings) {
+    const listing = candidates.map((unit, index) => `[${index}] ${unit.text}`).join('\n');
+    const { text } = await this._complete(
+      `${fenceUntrusted(listing, 'numbered page excerpts')}\nQuery: ${query}\n` +
+      `Reply with the numbers of the ${limit} excerpts most relevant to the query.`,
+      {
+        maxTokens: Math.max(64, limit * 4),
+        systemPrompt: 'You choose which numbered excerpts from a web page answer a query. ' +
+          'Reply with the numbers only, comma-separated, most relevant first. Do not write anything else.'
+      }
+    );
+    const chosen = parseChosenIndexes(text, candidates.length, limit);
+    if (chosen.length === 0) {
+      warnings.push('highlights: the model named no candidate; the extractive order is returned');
+      return candidates.slice(0, limit);
+    }
+    return chosen.sort((a, b) => a - b).map((index) => candidates[index]);
+  }
+
+  /** mode:"model" for question: an answer written from the fenced evidence alone. */
+  async _answerQuestion(evidence, question) {
+    const { text } = await this._complete(
+      `${fenceUntrusted(evidence.map((unit) => unit.text).join('\n'), 'page evidence')}\nQuestion: ${question}\n` +
+      'Answer from the evidence only; if the evidence does not say, say so. Reply with the answer only.',
+      {
+        maxTokens: 256,
+        systemPrompt: 'You answer a question from evidence quoted from a web page. Use only the evidence. ' +
+          'If the evidence does not contain the answer, say that it does not. Reply with the answer only, no preamble.'
+      }
+    );
+    return text.trim();
   }
 
   /** Lazy-load ExtractWithLlm to avoid pulling in heavy deps unless needed. */
@@ -293,6 +365,25 @@ export class UnifiedScrapeTool {
       return mainHtml;
     }
 
+    // The markdown format and the query-scoped formats read the same string,
+    // produced once, so a highlight's offset always indexes what `markdown`
+    // returned at this call's onlyMainContent.
+    let markdown = null;
+    function getMarkdown() {
+      if (markdown === null) {
+        markdown = onlyMainContent ? htmlToMarkdown(getMainHtml()) : htmlToMarkdown($.html('body') || html);
+      }
+      return markdown;
+    }
+    let units = null;
+    function getUnits() {
+      if (units === null) units = segmentUnits(getMarkdown());
+      return units;
+    }
+    // Whether a model completed for a mode:"model" format; the model
+    // surcharge is charged only then.
+    let modelUsed = false;
+
     const content = {};
     // The gate's warnings (e.g. a respect_robots override) travel with the
     // per-format ones, so the caller sees the decision in the response.
@@ -383,13 +474,66 @@ export class UnifiedScrapeTool {
         continue;
       }
 
+      // Query-scoped formats (Phase 1): verbatim units of this call's
+      // markdown, ranked against the query. mode:"model" lets a model choose
+      // among (highlights) or answer from (question) those units; when no
+      // LLM route exists the extractive result stands and the model
+      // surcharge is dropped below. An empty match is a warning, not an error.
+      if (isQueryFormat(fmt)) {
+        const modelUnavailable = (label) =>
+          warnings.push(`${label}: mode "model" was unavailable (no Ollama, API key or client sampling); the extractive result is returned at the extractive price`);
+        try {
+          if (fmt.type === 'highlights') {
+            let chosen = rankUnits(getUnits(), fmt.query, { maxUnits: fmt.max_highlights });
+            if (fmt.mode === 'model' && chosen.length > 0) {
+              const candidates = rankUnits(getUnits(), fmt.query, { maxUnits: fmt.max_highlights * 3 });
+              if (candidates.length > fmt.max_highlights) {
+                try {
+                  chosen = await this._chooseHighlights(candidates, fmt.query, fmt.max_highlights, warnings);
+                  modelUsed = true;
+                } catch {
+                  modelUnavailable('highlights');
+                }
+              } else {
+                warnings.push('highlights: no more candidates than max_highlights, so the model had nothing to choose; charged at the extractive price');
+              }
+            }
+            content.highlights = chosen.map(toPublicUnit);
+            if (chosen.length === 0) warnings.push(`highlights: no sentence, table row or code block matched "${fmt.query}"`);
+          } else {
+            const evidence = rankUnits(getUnits(), fmt.question, { maxUnits: QUESTION_EVIDENCE_UNITS });
+            const evidenceText = evidence.map((unit) => unit.text).join('\n');
+            let text = evidenceText;
+            let grounded = true;
+            if (fmt.mode === 'model' && evidence.length > 0) {
+              try {
+                text = await this._answerQuestion(evidence, fmt.question);
+                modelUsed = true;
+                const check = groundingCheck(text, evidenceText, fmt.question);
+                grounded = check.grounded;
+                if (!grounded) {
+                  warnings.push(`question: the model's answer has ${check.unbacked.length} token(s) not found in the evidence or the question (${check.unbacked.join(', ')}); grounded: false`);
+                }
+              } catch {
+                modelUnavailable('question');
+              }
+            }
+            content.answer = { text, grounded, evidence: evidence.map(toPublicUnit) };
+            if (evidence.length === 0) warnings.push(`question: no sentence, table row or code block matched "${fmt.question}"`);
+          }
+        } catch (err) {
+          if (fmt.type === 'highlights') content.highlights = [];
+          else content.answer = { text: '', grounded: true, evidence: [] };
+          warnings.push(`${fmt.type}: ${err.message}`);
+        }
+        continue;
+      }
+
       // String formats
       switch (fmt) {
         case 'markdown':
           try {
-            content.markdown = onlyMainContent
-              ? htmlToMarkdown(getMainHtml())
-              : htmlToMarkdown($.html('body') || html);
+            content.markdown = getMarkdown();
           } catch (err) {
             content.markdown = '';
             warnings.push(`markdown: ${err.message}`);
@@ -501,6 +645,19 @@ export class UnifiedScrapeTool {
       }
     }
 
+    const queryScoped = formats.some(isQueryFormat);
+    if (queryScoped && !formats.includes('markdown')) {
+      warnings.push('offsets index the "markdown" format of this call (same onlyMainContent); add "markdown" to formats to quote with a locator');
+    }
+    if (!queryScoped && typeof content.markdown === 'string' && content.markdown.length > OVERSIZED_MARKDOWN_CHARS) {
+      warnings.push(`markdown: ${content.markdown.length} characters; ask for {type:"highlights", query} (1 extra credit, no model) to get only the matching sentences, table rows and code blocks`);
+    }
+    // The projection charged the model step; nothing ran, so the charge
+    // drops to the extractive price (2 is scrape's base in AuthManager;
+    // withAuth clamps to the projection either way).
+    const surcharge = scrapeFormatSurcharge(formats);
+    if (surcharge.model > 0 && !modelUsed) setActualCost(2 + surcharge.query);
+
     return {
       success: true,
       url: finalUrl,