crawlforge-mcp-server 6.22.3 → 6.23.0

package/src/tools/extract/processDocument.js

@@ -15,6 +15,7 @@ import { safeFetch } from '../../utils/ssrfGuard.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
 import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
 import { isRemoteTransport } from '../../utils/remoteMode.js';
+import { assertLocalFileAllowed } from '../../utils/localFilePolicy.js';
 import { flattenText } from 'crawlforge-extractors';
 import { load } from 'cheerio';
 
@@ -178,6 +179,38 @@ export function decodeTextBody(buffer, contentType = '') {
   }
 }
 
+// Lines that open a PDF's text layer without being its title: arXiv's margin
+// stamp ("arXiv:1706.03762v7 [cs.CL] 2 Aug 2023") and bare page numbers.
+const NON_TITLE_LINE_RE = /^(arXiv:\S+.*|\d+)$/i;
+const MAX_TITLE_CHARS = 200;
+
+/**
+ * A title for a document with none in its metadata: the first non-empty line
+ * of its text when that line is short enough to be one, else null. arXiv
+ * stamps and page numbers before it are skipped.
+ * @param {string} text
+ * @returns {string|null}
+ */
+export function titleFromText(text) {
+  for (const line of String(text || '').split(/\r?\n/)) {
+    const candidate = line.replace(/\s+/g, ' ').trim();
+    if (candidate === '' || NON_TITLE_LINE_RE.test(candidate)) continue;
+    return candidate.length <= MAX_TITLE_CHARS ? candidate : null;
+  }
+  return null;
+}
+
+/**
+ * The text of the first h1-h6 in HTML, or null when it has none.
+ * @param {string} html
+ * @returns {string|null}
+ */
+export function firstHeadingFromHtml(html) {
+  const $ = load(html || '');
+  const text = $('h1, h2, h3, h4, h5, h6').first().text().replace(/\s+/g, ' ').trim();
+  return text || null;
+}
+
 export class ProcessDocumentTool {
   constructor() {
     this.pdfProcessor = new PDFProcessor();
@@ -221,6 +254,12 @@ export class ProcessDocumentTool {
         );
       }
 
+      // Locally, a read is still held to the document extensions and kept out
+      // of dot-directories (or held to CRAWLFORGE_ALLOWED_DIRS): see localFilePolicy.js.
+      const readFrom = sourceType === 'file' || sourceType === 'pdf_file'
+        ? await assertLocalFileAllowed(source)
+        : source;
+
       const result = {
         source,
         sourceType,
@@ -232,10 +271,10 @@ export class ProcessDocumentTool {
       // Determine document type and processing method
       if (sourceType.includes('pdf')) {
         result.documentType = 'pdf';
-        await this.processPDFDocument(result, source, sourceType, options, { respect_robots, user_agent });
+        await this.processPDFDocument(result, readFrom, sourceType, options, { respect_robots, user_agent });
       } else if (sourceType === 'file') {
         result.documentType = 'file';
-        await this.processLocalFileDocument(result, source, options);
+        await this.processLocalFileDocument(result, readFrom, options);
       } else {
         result.documentType = 'web';
         await this.processWebDocument(result, source, options, { respect_robots, user_agent });
@@ -362,8 +401,9 @@ export class ProcessDocumentTool {
       result.pagesRead = { ...pdfResult.extractedPages, totalPages: pdfResult.pageCount };
     }
 
-    // Set title
-    result.title = pdfResult.metadata?.title || null;
+    // Set title: the metadata title, else the first short line of the text
+    // (an arXiv PDF carries no /Title, live sweep 6.11).
+    result.title = pdfResult.metadata?.title || titleFromText(result.content.text);
 
     // Set metadata
     if (pdfResult.metadata) {
@@ -481,7 +521,10 @@ export class ProcessDocumentTool {
       const md = await mammoth.convertToMarkdown({ buffer });
       result.content.markdown = md.value || '';
     }
-    result.title = null;
+    // A .docx has no title field mammoth reads: use the first heading, else
+    // the first short line of the text (live sweep 6.11).
+    const html = await mammoth.convertToHtml({ buffer });
+    result.title = firstHeadingFromHtml(html.value) || titleFromText(result.content.text);
     const notes = (raw.messages || []).map((m) => m.message).filter(Boolean);
     if (notes.length > 0) result.warnings = [...(result.warnings || []), ...notes.slice(0, 5)];
   }

package/src/tools/extract/summarizeContent.js

@@ -21,7 +21,9 @@ const SummarizeContentSchema = z.object({
   text: z.string().min(10),
   options: z.object({
     summaryLength: z.enum(['short', 'medium', 'long']).default('medium'),
-    summaryType: z.enum(['extractive', 'abstractive']).default('extractive'),
+    // No default: omitted with summaryLength 'medium', abstractive is tried
+    // and falls back to extractive without flagging it (see execute).
+    summaryType: z.enum(['extractive', 'abstractive']).optional(),
     includeKeypoints: z.boolean().default(true),
     includeKeywords: z.boolean().default(true),
     includeStatistics: z.boolean().default(true),
@@ -93,6 +95,26 @@ const NAVIGATION_PHRASES = new Set([
   'de wikipedia, la enciclopedia libre'
 ]);
 
+// Whole lines that are page chrome wherever they appear: consent buttons and
+// account links. The live sweep (2026-10-20) found "all cookies manage
+// preferences" and "register subscribe" in summarize_content key points and
+// keywords. Matched exactly, like NAVIGATION_PHRASES, so a sentence that
+// merely contains one of these words is never touched.
+const CHROME_LINES = new Set([
+  'accept', 'accept all', 'accept all cookies', 'accept cookies', 'reject all',
+  'reject all cookies', 'manage preferences', 'manage cookies', 'manage settings',
+  'cookie settings', 'cookie preferences', 'cookies', 'cookie policy', 'i agree',
+  'register', 'subscribe', 'sign in', 'sign up', 'sign up now', 'log in', 'login',
+  'menu', 'search', 'close', 'skip to content', 'accessibility help', 'privacy policy',
+  'terms of use', 'terms of service'
+]);
+
+// A line about cookies that also speaks the language of consent is a banner,
+// not content; "Bake the cookies for 12 minutes" has no consent vocabulary.
+const COOKIE_WORD = /\bcookies?\b/i;
+const CONSENT_WORD = /\b(accept|agree|consent|manage|preferences|settings|policy|privacy|reject|we use|partners)\b/i;
+const SENTENCE_PUNCTUATION = /[.!?…。！？]/;
+
 export class SummarizeContentTool {
   constructor() {
     this.contentAnalyzer = new ContentAnalyzer();
@@ -131,7 +153,10 @@ export class SummarizeContentTool {
       // Page text often opens with navigation chrome ("Jump to content",
       // "From Wikipedia, the free encyclopedia") that otherwise leads the
       // summary and key points — strip it before summarizing.
-      const text = this.stripLeadingBoilerplate(rawText);
+      // The rest of the page's chrome (menus, consent banners, repeated
+      // labels) is spread through the text, and its unpunctuated lines glue
+      // whole runs of headlines into one "sentence" — strip and split those too.
+      const text = this.terminateLines(this.stripPageBoilerplate(this.stripLeadingBoilerplate(rawText)));
 
       const result = {
         originalText: rawText.substring(0, 500) + (rawText.length > 500 ? '...' : ''),
@@ -148,7 +173,7 @@ export class SummarizeContentTool {
           extractKeywords: options.includeKeywords,
           detectLanguage: true,
           summaryLength: options.summaryLength,
-          summaryType: options.summaryType,
+          summaryType: options.summaryType ?? 'extractive',
           maxKeywords: options.maxKeywords,
           extractTopics: false,
           extractEntities: false,
@@ -167,7 +192,18 @@ export class SummarizeContentTool {
       // D1.3: If abstractive mode requested, attempt sampling-based enhancement.
       // If it can't run (no LLM/sampling available), fall back to the extractive
       // result but flag it explicitly rather than silently masking.
-      if (options.summaryType === 'abstractive') {
+      // An omitted summaryType at the default 'medium' length prefers an
+      // abstractive summary when an LLM is reachable, and otherwise keeps the
+      // extractive one silently: the caller asked for no particular type, so
+      // nothing was degraded. Text too short to compress (ratio 1) is left as is.
+      const preferAbstractive = options.summaryType === undefined &&
+        options.summaryLength === 'medium' && analysisResult.summary.compressionRatio < 1;
+      if (preferAbstractive) {
+        const abstractive = await this._abstractiveSummaryViaSampling(
+          text, analysisResult.summary, options.summaryLength, analysisResult.language?.name
+        );
+        if (abstractive) result.summary = abstractive;
+      } else if (options.summaryType === 'abstractive') {
         const abstractive = await this._abstractiveSummaryViaSampling(
           text, analysisResult.summary, options.summaryLength, analysisResult.language?.name
         );
@@ -201,7 +237,7 @@ export class SummarizeContentTool {
       // Step 6: Set metadata
       result.metadata = {
         language: analysisResult.language?.code || options.language || 'unknown',
-        processingMethod: options.summaryType,
+        processingMethod: options.summaryType ?? result.summary.type,
         confidenceScore: this.calculateConfidenceScore(text, result.summary.text)
       };
 
@@ -361,6 +397,88 @@ export class SummarizeContentTool {
     return strippedLines > 0 ? lines.slice(index).join('\n') : text;
   }
 
+  /**
+   * Strip navigation and consent chrome from anywhere in extracted page text.
+   * A line is dropped when it is:
+   *   - a CHROME_LINES / NAVIGATION_PHRASES entry, or a cookie line that uses
+   *     consent vocabulary;
+   *   - a pipe/bullet menu ("Home | News | Sport"): 3+ segments of ≤ 4 words;
+   *   - repeated on the page: a short label (≤ 4 words, unpunctuated) loses
+   *     every copy — "In stock", "Add to basket", section names — while a
+   *     longer repeated line keeps its first copy;
+   *   - part of a nav run: 4+ consecutive lines that are each ≤ 30 chars,
+   *     ≤ 4 words, unpunctuated and digit-free. The digit rule keeps listings
+   *     whose short titles alternate with prices or counts.
+   * If almost nothing survives (< 10 chars) the text is returned untouched.
+   * @param {string} text - Text to clean
+   * @returns {string} - Text without page chrome
+   */
+  stripPageBoilerplate(text) {
+    const lines = text.split('\n');
+    const key = (line) => line.trim().replace(/[.!?…。！？]+$/, '').trim().toLowerCase();
+    const isShort = (line) => line.split(/\s+/).length <= 4 && !SENTENCE_PUNCTUATION.test(line);
+
+    const counts = new Map();
+    for (const line of lines) {
+      const k = key(line);
+      if (k) counts.set(k, (counts.get(k) || 0) + 1);
+    }
+
+    const seen = new Set();
+    const drop = lines.map((raw) => {
+      const line = raw.trim();
+      if (!line) return false;
+      const k = key(line);
+      if (CHROME_LINES.has(k) || NAVIGATION_PHRASES.has(k)) return true;
+      if (COOKIE_WORD.test(line) && CONSENT_WORD.test(line)) return true;
+      const segments = line.split(/\s*[|•·]\s*/).filter(Boolean);
+      if (segments.length >= 3 && segments.every((s) => s.split(/\s+/).length <= 4)) return true;
+      if (counts.get(k) > 1 && (isShort(line) || seen.has(k))) return true;
+      seen.add(k);
+      return false;
+    });
+
+    // Runs are measured on the original lines, so a menu whose items were
+    // already dropped as repeats still counts towards the run around them.
+    const isNavLine = (line) => line.length <= 30 && isShort(line) && !/\d/.test(line);
+    let run = [];
+    const flush = () => {
+      if (run.length >= 4) for (const i of run) drop[i] = true;
+      run = [];
+    };
+    lines.forEach((raw, i) => {
+      const line = raw.trim();
+      if (!line) return;
+      if (isNavLine(line) || drop[i]) run.push(i);
+      else flush();
+    });
+    flush();
+
+    const cleaned = lines.filter((_, i) => !drop[i]).join('\n').replace(/\n{3,}/g, '\n\n').trim();
+    return cleaned.length >= 10 ? cleaned : text;
+  }
+
+  /**
+   * End each unpunctuated line with a full stop when the next line starts a
+   * new unit (a capital, digit, quote or currency sign). Headlines, titles and
+   * prices carry no terminator, so without this the sentence splitter glued a
+   * whole column of them into one "sentence" — the books listing's summary
+   * was a run of prices. A next line starting in lower case is a wrapped
+   * sentence and is left alone; CJK has no capitals and is never touched.
+   * @param {string} text
+   * @returns {string}
+   */
+  terminateLines(text) {
+    const lines = text.split('\n');
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i].trimEnd();
+      if (!line.trim() || /[.!?…。！？:;,]$/.test(line)) continue;
+      const next = lines.slice(i + 1).find((l) => l.trim());
+      if (next && /^[\p{Lu}\p{N}£$€'"‘“(]/u.test(next.trim())) lines[i] = `${line}.`;
+    }
+    return lines.join('\n');
+  }
+
   /**
    * Extract key points from original text and summary
    * @param {string} originalText - Original text

package/src/tools/llmstxt/generateLLMsTxt.js

@@ -740,9 +740,9 @@ export class GenerateLLMsTxtTool {
     lines.push('If you encounter issues or need clarification on these guidelines:');
     lines.push('1. Check this document for guidance');
     lines.push('2. Review the basic llms.txt file');
-    // Only when section 1 printed something to contact: without either field
-    // this pointed at contact information the file did not contain.
-    if (outputOptions.contactEmail || outputOptions.organizationName) {
+    // Only when the section above lists a contact: an organization name
+    // alone is not one, and without an email it is empty (live sweep 4.4).
+    if (outputOptions.contactEmail) {
       lines.push('3. Contact us using the information above');
     }
     lines.push('');
@@ -833,8 +833,9 @@ export class GenerateLLMsTxtTool {
       });
     }
 
-    // API recommendations
-    if (analysis.apis && analysis.apis.length > 0) {
+    // API recommendations: only endpoints that answered JSON/XML, not links
+    // to API documentation pages.
+    if (analysis.apis && analysis.apis.some((api) => api.type !== 'documentation')) {
       recommendations.push({
         type: 'integration',
         priority: 'high',

package/src/tools/result/readResult.js

@@ -10,7 +10,6 @@
  */
 
 import { z } from 'zod';
-import { selectJsonPath } from 'crawlforge-extractors';
 import { getResultStore, RESULT_HANDLE_PATTERN } from '../../core/ResultStore.js';
 import { MAX_INLINE_CHARS_PARAM, resolveMaxInlineChars, resultTextView } from '../../server/inlineThreshold.js';
 import { dualOutput } from '../../server/registerTool.js';
@@ -27,7 +26,7 @@ export const READ_RESULT_INPUT_SHAPE = {
   length: z.number().int().min(1).max(200000).optional().describe("slice: characters to return (default 10,000); lines: lines to return (default 200, max 5,000)"),
   query: z.string().min(1).max(500).optional().describe("search: the text to find, matched literally, case-insensitive"),
   max_matches: z.number().int().min(1).max(100).optional().default(20).describe("search: matches to return (default 20)"),
-  path: z.string().optional().describe("json_path: dotted keys and array indexes, e.g. \"results[3].content\" — not JSONPath"),
+  path: z.string().optional().describe("json_path: dotted keys and array indexes from the root of the stored result, e.g. \"results[3].content\"; a key holding a \".\" in brackets, e.g. files[\"llms.txt\"]; a JSON string field is read into, e.g. \"body.items[0]\" for a fetch_url JSON body — not JSONPath"),
   ...MAX_INLINE_CHARS_PARAM
 };
 
@@ -57,6 +56,81 @@ export function goneMessage(store, handle) {
       return `Result handle deleted by the server at ${at}.`;
   }
 }
+/**
+ * Split a json_path into keys: dotted keys, [0] indexes, [key] and the
+ * bracket-quoted ["a.b"] / ['a.b'], whose key may hold a ".".
+ */
+export function parseResultPath(path) {
+  const segments = [];
+  let i = 0;
+  while (i < path.length) {
+    if (path[i] === '.') { i++; continue; }
+    if (path[i] === '[') {
+      const quote = path[i + 1];
+      if (quote === '"' || quote === "'") {
+        const close = path.indexOf(`${quote}]`, i + 2);
+        if (close === -1) throw new Error(`Path "${path}" has an unclosed ${quote} key: write it as [${quote}key${quote}]`);
+        segments.push(path.slice(i + 2, close));
+        i = close + 2;
+      } else {
+        const close = path.indexOf(']', i);
+        const end = close === -1 ? path.length : close;
+        segments.push(path.slice(i + 1, end));
+        i = end + 1;
+      }
+      continue;
+    }
+    let end = i;
+    while (end < path.length && path[end] !== '.' && path[end] !== '[') end++;
+    segments.push(path.slice(i, end));
+    i = end;
+  }
+  return segments.filter((segment) => segment !== '');
+}
+
+function describeKeys(value) {
+  if (Array.isArray(value)) return `array of length ${value.length}`;
+  if (value !== null && typeof value === 'object') {
+    const keys = Object.keys(value);
+    return `available keys: ${keys.slice(0, 25).join(', ')}${keys.length > 25 ? `, … (${keys.length} total)` : ''}`;
+  }
+  return `a ${value === null ? 'null' : typeof value} value, which has no keys`;
+}
+
+/**
+ * Resolve a json_path from the root of the stored result. A string holding
+ * JSON (a fetch_url body) is parsed when the path goes on into it.
+ * @throws {Error} "path not found", naming the keys where it stopped and the
+ *   result's top-level keys
+ */
+export function selectResultPath(root, path) {
+  const segments = parseResultPath(path);
+  if (segments.length === 0) throw new Error(`Path "${path}" is empty`);
+  let current = root;
+  const walked = [];
+  for (const segment of segments) {
+    if (typeof current === 'string' && /^\s*[[{]/.test(current)) {
+      try { current = JSON.parse(current); } catch { /* not JSON: a string has no keys */ }
+    }
+    const container = current !== null && typeof current === 'object';
+    const key = Array.isArray(current) ? Number(segment) : segment;
+    if (!container || !Object.hasOwn(current, key)) {
+      const at = walked.length === 0 ? 'the result' : `"${walked.join('.')}"`;
+      const top = walked.length === 0 ? '' : `; the stored result's top-level keys: ${Object.keys(root).join(', ')}`;
+      throw new Error(`Path "${path}" not found: ${at} has no "${segment}" (${describeKeys(current)})${top}`);
+    }
+    current = current[key];
+    walked.push(segment);
+  }
+  return current;
+}
+
+// A bad path on a live handle names its own next step, so withAuth does not
+// append read_result's hint for a handle that is gone.
+const PATH_NEXT_STEP = 'Next step: the handle is still valid - call read_result operation "json_path" again with a path '
+  + 'from the keys above. Paths start at the stored result (a fetch_url body is under "body."), and a key holding '
+  + 'a "." goes in brackets, e.g. files["llms.txt"].';
+
 // read_result declares an outputSchema, so a success carries structuredContent too.
 const ok = (object) => dualOutput(object);
 
@@ -140,17 +214,12 @@ export async function readResultHandler(params) {
     case 'json_path': {
       const { path } = params;
       if (!path) return fail('path is required for operation "json_path"');
-      // A text view whose text is itself JSON (a fetch_url body) is read as
-      // that JSON; otherwise the stored result object is the subject.
-      let subject = entry.payload;
-      if (view === 'text') {
-        try { subject = JSON.parse(text); } catch { /* not JSON: address the result object */ }
-      }
+      // One root for every tool: the whole stored result (live sweep 4.3).
       let value;
       try {
-        value = selectJsonPath(subject, path);
+        value = selectResultPath(entry.payload, path);
       } catch (error) {
-        return fail(error.message);
+        return fail(`${error.message}\n${PATH_NEXT_STEP}`);
       }
       const valueJson = JSON.stringify(value) ?? 'null';
       if (valueJson.length > maxChars) {

package/src/tools/scrape/_mainContent.js

@@ -44,6 +44,7 @@
 import { JSDOM } from 'jsdom';
 import { Readability } from '@mozilla/readability';
 import { load } from 'cheerio';
+import { normalizeCodeBlocks } from '../../utils/htmlToMarkdown.js';
 
 // How much of a table's text is compared against the article to decide whether
 // Readability already kept it. Long enough to be unique to that table.
@@ -177,6 +178,68 @@ export function unwrapMediaWikiHeadings(document) {
   }
 }
 
+// Readability's `byline` regex. The first element matching it (or carrying
+// rel="author" / itemprop="author") under 100 characters is taken as the
+// article's byline and deleted from the content.
+const BYLINE = /byline|author|dateline|writtenby|p-author/i;
+
+// A list page repeats one item markup: 10 `div.quote`, 20 `li.product`. Items
+// shorter than this are menu entries or tag-cloud links, not content.
+const MIN_LIST_ITEMS = 5;
+const MIN_ITEM_CHARS = 40;
+
+/**
+ * The repeated items of a list page in a DOM document: the children of one
+ * parent that share a tag and class, when there are at least MIN_LIST_ITEMS
+ * of them with MIN_ITEM_CHARS of text each.
+ * @param {Document} document
+ * @returns {Element[]}
+ */
+function repeatedItems(document) {
+  const items = [];
+  for (const parent of Array.from(document.querySelectorAll('body, body *'))) {
+    const groups = new Map();
+    for (const child of Array.from(parent.children)) {
+      const className = (child.getAttribute('class') || '').trim();
+      if (!className) continue;
+      const key = `${child.tagName} ${className}`;
+      groups.set(key, [...(groups.get(key) || []), child]);
+    }
+    for (const group of groups.values()) {
+      if (group.length < MIN_LIST_ITEMS) continue;
+      const substantial = group.filter((el) => normalizeWhitespace(el.textContent || '').length >= MIN_ITEM_CHARS);
+      if (substantial.length >= MIN_LIST_ITEMS) items.push(...substantial);
+    }
+  }
+  return items;
+}
+
+/**
+ * Stop Readability taking an author line inside a list item as the page's
+ * byline. It deletes the first byline-like element it meets, so on
+ * quotes.toscrape.com the first quote lost its author ("…our thinking.” by
+ * (about)") from every main-content read (live sweep, 2026-10-20). A byline
+ * belongs to an article, not to one entry of a list, so the attributes that
+ * mark one are stripped from nodes inside repeated items.
+ *
+ * @param {Document} document - mutated in place
+ */
+export function keepListItemBylines(document) {
+  for (const item of repeatedItems(document)) {
+    for (const el of Array.from(item.querySelectorAll('*'))) {
+      if (el.getAttribute('rel') === 'author') el.removeAttribute('rel');
+      if ((el.getAttribute('itemprop') || '').includes('author')) el.removeAttribute('itemprop');
+      if (BYLINE.test(el.id || '')) el.removeAttribute('id');
+      const className = el.getAttribute('class');
+      if (className && BYLINE.test(className)) {
+        const kept = className.split(/\s+/).filter((token) => token && !BYLINE.test(token));
+        if (kept.length > 0) el.setAttribute('class', kept.join(' '));
+        else el.removeAttribute('class');
+      }
+    }
+  }
+}
+
 /**
  * Readability's own data-table test (`_markDataTables`): 10+ rows or more than
  * 4 columns means the table carries data rather than layout, counting
@@ -309,9 +372,11 @@ export function extractMainContent(html, url) {
   if (tooLargeForReadability(html)) return { html: null, title: '', tablesRecovered: 0, tooLarge: true };
   let article;
   try {
-    const dom = new JSDOM(html, { url });
+    // Readability strips classes, so code-block languages move to data attributes first.
+    const dom = new JSDOM(normalizeCodeBlocks(html), { url });
     sanitizeAccidentalClasses(dom.window.document);
     unwrapMediaWikiHeadings(dom.window.document);
+    keepListItemBylines(dom.window.document);
     article = new Readability(dom.window.document).parse();
   } catch {
     return { html: null, title: '', tablesRecovered: 0 };
@@ -356,8 +421,8 @@ export function keepPageHeading(mainHtml, $) {
 }
 
 // Below both of these, Readability's article is a fragment of the page, not
-// its main content.
-export const THIN_MAIN_CONTENT = { maxChars: 1500, maxShare: 0.35 };
+// its main content. On a list page the bar is a share of `listMaxShare`.
+export const THIN_MAIN_CONTENT = { maxChars: 1500, maxShare: 0.35, listMaxShare: 0.6 };
 
 function visibleTextLength(html) {
   const $ = load(html);
@@ -379,10 +444,36 @@ function visibleTextLength(html) {
  */
 export function isThinMainContent(mainHtml, pageHtml) {
   const kept = visibleTextLength(mainHtml || '');
-  if (kept >= THIN_MAIN_CONTENT.maxChars) return null;
   const visible = visibleTextLength(pageHtml || '');
-  if (visible === 0 || kept / visible >= THIN_MAIN_CONTENT.maxShare) return null;
-  return { kept, visible };
+  if (visible === 0) return null;
+  const share = kept / visible;
+  if (kept < THIN_MAIN_CONTENT.maxChars && share < THIN_MAIN_CONTENT.maxShare) return { kept, visible };
+  // A list page's items are its content, and Readability scores them one
+  // block at a time: scrape_with_actions' text lost 3 of quotes.toscrape.com's
+  // 10 quotes (live sweep, 2026-10-20). Keeping much less than the page has
+  // there means items were pruned, not chrome.
+  if (share < THIN_MAIN_CONTENT.listMaxShare && isListPage(pageHtml)) return { kept, visible };
+  return null;
+}
+
+/**
+ * True when `html` repeats one item markup MIN_LIST_ITEMS+ times with
+ * MIN_ITEM_CHARS of text each — repeatedItems' test, on a cheerio parse.
+ * @param {string} html
+ */
+function isListPage(html) {
+  const $ = load(html);
+  return $('body, body *').toArray().some((parent) => {
+    const groups = new Map();
+    for (const child of $(parent).children().toArray()) {
+      const className = ($(child).attr('class') || '').trim();
+      if (!className) continue;
+      const key = `${child.name} ${className}`;
+      groups.set(key, [...(groups.get(key) || []), child]);
+    }
+    return Array.from(groups.values()).some((group) => group.length >= MIN_LIST_ITEMS &&
+      group.filter((el) => normalizeWhitespace($(el).text()).length >= MIN_ITEM_CHARS).length >= MIN_LIST_ITEMS);
+  });
 }
 
 export default extractMainContent;

package/src/tools/search/adapters/dataforseoSearch.js

@@ -155,7 +155,10 @@ export class DataForSEOSearchAdapter {
         device,
         depth,
         organicCount: items.length,
-        seResultsCount: result0?.se_results_count ?? null,
+        // Google's "About N results" total. Mobile SERPs (and many desktop
+        // ones) show no total, and DataForSEO then reports 0; a 0 beside
+        // organic results is unknown, not zero.
+        seResultsCount: result0?.se_results_count === 0 && items.length > 0 ? null : result0?.se_results_count ?? null,
         checkUrl: result0?.check_url ?? null, // Google URL to eyeball the real SERP
         cost: data.cost ?? task0.cost ?? null, // USD charged by DataForSEO for this call
       },

package/src/tools/search/searchWeb.js

@@ -84,6 +84,8 @@ const SearchWebSchema = z.object({
 // never costs an extra backend search.
 const DEDUPE_OVERFETCH = 4;
 const GOOGLE_MAX_RESULTS_PER_REQUEST = 10;
+// Deepest SearXNG page _executeViaSearxng reads to reach an offset.
+const SEARXNG_MAX_PAGES = 5;
 
 /**
  * Index, in the provider's page, of the first item this response does not
@@ -348,7 +350,9 @@ export class SearchWebTool {
       // Where the next page starts. Deduplication backfills this page from
       // the over-fetched margin, so the next page begins after the first
       // provider item NOT shown here, not at offset + limit (which repeated
-      // the backfilled items on the next page).
+      // the backfilled items on the next page). So next_offset is offset +
+      // the provider items this page consumed, duplicates included: 5 results
+      // with next_offset 6 means one of items 0..5 was a duplicate.
       const nextOffset = localizedParams.offset + firstUnshownIndex(
         processedResults,
         localizedParams.limit,
@@ -469,17 +473,38 @@ export class SearchWebTool {
    * @returns {Promise<Object>} Standard search_web response object
    */
   async _executeViaSearxng(validated) {
-    // page is 1-based; offset is 0-based items, so map via limit
-    const page = Math.floor(validated.offset / validated.limit) + 1;
-
-    const adapterResult = await searchViaSearxng({
-      query: validated.query,
-      // SearXNG returns a whole page per request, so the dedup margin is free.
-      limit: validated.limit + DEDUPE_OVERFETCH,
-      page,
-      safeSearch: validated.safe_search,
-      language: validated.lang
-    });
+    // SearXNG is addressed by page, and its page size is set by the instance
+    // and its engines, not by `limit`. So `offset` counts items in SearXNG's
+    // result stream: read pages 1, 2, ... until the window
+    // [offset, offset + limit + margin) is covered, a page adds no new URL
+    // (end of results, or an instance that repeats a page), or the page cap.
+    const wanted = validated.offset + validated.limit + DEDUPE_OVERFETCH;
+    const stream = [];
+    const seen = new Set();
+    let firstPage = null;
+    for (let page = 1; page <= SEARXNG_MAX_PAGES && stream.length < wanted; page++) {
+      let pageResult;
+      try {
+        pageResult = await searchViaSearxng({
+          query: validated.query,
+          limit: Number.MAX_SAFE_INTEGER, // keep the whole page
+          page,
+          safeSearch: validated.safe_search,
+          language: validated.lang
+        });
+      } catch (error) {
+        if (page === 1) throw error;
+        break; // a later page failing ends the stream; keep what was read
+      }
+      firstPage ??= pageResult;
+      const fresh = (pageResult.items || []).filter(item => !seen.has(item.link) && seen.add(item.link));
+      if (fresh.length === 0) break;
+      stream.push(...fresh);
+    }
+    const adapterResult = {
+      ...firstPage,
+      items: stream.slice(validated.offset, wanted)
+    };
 
     // Run through shared post-processing (deduplication, ranking)
     let processedResults = await this.processResults(adapterResult);
@@ -503,6 +528,14 @@ export class SearchWebTool {
       };
     }
 
+    // Same rule as execute(): the next page starts at the first stream item
+    // this page does not show, so duplicates removed here are not re-read.
+    const nextOffset = validated.offset + firstUnshownIndex(
+      processedResults,
+      validated.limit,
+      adapterResult.items.length
+    );
+
     // Drop the over-fetched margin (see execute()).
     if (processedResults.length > validated.limit) {
       processedResults = processedResults.slice(0, validated.limit);
@@ -541,9 +574,7 @@ export class SearchWebTool {
       search_time: adapterResult.searchInformation?.searchTime || 0,
       offset: validated.offset,
       limit: validated.limit,
-      // SearXNG is paged (offset maps to page floor(offset/limit)+1), so the
-      // next page starts at the next multiple of limit.
-      next_offset: page * validated.limit,
+      next_offset: nextOffset,
       cached: false,
       provider: {
         name: 'searxng',