crawlforge-mcp-server 6.22.3 → 6.23.0

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -13,6 +13,7 @@ import { flattenText } from 'crawlforge-extractors';
 import ExtractContentTool from '../extract/extractContent.js';
 import { stealthDocumentVerdict } from '../../utils/stealthVerdict.js';
 import { elementText } from '../../utils/elementText.js';
+import { parseSelectorSpec } from '../basic/scrapeStructured.js';
 import { pageTitle } from '../../utils/pageTitle.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
 import { resolveStealthEngine, redactProxyEntry, assertProxyEngineAllowed } from '../../core/StealthBrowserManager.js';
@@ -779,7 +780,7 @@ export class ScrapeWithActionsTool extends EventEmitter {
         }
 
         if (params.extractionOptions?.selectors) {
-          state.content.extracted = this.extractWithSelectors($, params.extractionOptions.selectors);
+          state.content.extracted = this.extractWithSelectors($, params.extractionOptions.selectors, stateData.formValues);
         }
 
         states.push(state);
@@ -827,16 +828,21 @@ export class ScrapeWithActionsTool extends EventEmitter {
         $('script, style, noscript, template').remove();
         extractResult.content = extractResult.content || {};
         if (params.extractionOptions?.selectors) {
-          extractResult.content.extracted = this.extractWithSelectors($, params.extractionOptions.selectors);
+          extractResult.content.extracted = this.extractWithSelectors($, params.extractionOptions.selectors, chainResult.finalFormValues);
         }
         // Readability picks a "main" block and drops form controls; on a small
         // post-action page (a demo form, a results panel) that leaves the
         // footer alone, or a page minus the very buttons the actions produced.
         // A short readable result means it found nothing substantial, so hand
-        // back the whole (equally short) body text instead.
+        // back the whole (equally short) body text instead. So does a page
+        // whose Readability article was not its main content (a landing or
+        // list page): the whole page, as browser_session read returns it,
+        // rather than the boilerplate-removal guess — that read kept all 10
+        // of quotes.toscrape.com's quotes where this text lost 3 (2026-10-20).
         const readable = extractResult.content.text || '';
         const bodyText = flattenText($);
-        if (readable.length < 300 && bodyText.length > readable.length) {
+        const notMainContent = extractResult.extractionMethod && extractResult.extractionMethod !== 'readability';
+        if ((readable.length < 300 || notMainContent) && bodyText.length > readable.length) {
           extractResult.content.text = bodyText;
           extractResult.content.textSource = 'body';
         }
@@ -940,10 +946,29 @@ export class ScrapeWithActionsTool extends EventEmitter {
     return links;
   }
 
-  extractWithSelectors($, selectors) {
+  /**
+   * Run `extractionOptions.selectors` over a captured page. A selector may end
+   * in `@attr` to read that attribute instead of the text, as in
+   * scrape_structured. `@value` on an input, textarea or select reads what the
+   * control holds now — typed or selected — from `formValues` (captured from
+   * the live page, in document order); every other `@attr` reads the
+   * attribute as the page's HTML has it.
+   */
+  extractWithSelectors($, selectors, formValues = []) {
     const extracted = {};
+    const controls = $('input, textarea, select').toArray();
 
-    for (const [key, selector] of Object.entries(selectors)) {
+    const read = (el, attribute) => {
+      if (!attribute) return elementText($, el);
+      if (attribute === 'value') {
+        const live = formValues[controls.indexOf(el)];
+        if (live?.tag === el.name) return live.value;
+      }
+      return $(el).attr(attribute) ?? null;
+    };
+
+    for (const [key, raw] of Object.entries(selectors)) {
+      const { selector, attribute } = parseSelectorSpec(raw);
       try {
         const elements = $(selector);
         
@@ -953,12 +978,13 @@ export class ScrapeWithActionsTool extends EventEmitter {
         if (elements.length === 0) {
           extracted[key] = null;
         } else if (elements.length === 1) {
-          extracted[key] = elementText($, elements.get(0));
+          extracted[key] = read(elements.get(0), attribute);
         } else {
-          extracted[key] = elements.map((_, el) => elementText($, el)).get();
+          // Not .map().get(), which drops the nulls an absent attribute gives.
+          extracted[key] = elements.toArray().map((el) => read(el, attribute));
         }
       } catch (error) {
-        extracted[key] = { error: `Invalid selector: ${selector}` };
+        extracted[key] = { error: `Invalid selector: ${raw}` };
       }
     }
 

package/src/tools/agent/sourceChecks.js

@@ -0,0 +1,103 @@
+/**
+ * sourceChecks.js — deterministic checks on which sources the `agent` trusts
+ * and which dates it reports (live sweep 2026-10-20, item 2.5).
+ *
+ * "Latest stable Rust version and its release date" was first searched as the
+ * bare word "Rust": the game's site (rust.facepunch.com, a root titled "Rust")
+ * passed as the entity's official site, was promoted to the live page, and the
+ * run came back degraded:true blaming it. The answer gave 1.99.1 "as of 16
+ * October 2026": endoflife.date's "Last updated on" date, not the 15 October
+ * release date in its table.
+ *
+ * Imports nothing, so AgentOrchestrator can read it without a cycle.
+ */
+
+/** The registrable label of a hostname: "python" for www.python.org, "bbc" for bbc.co.uk. */
+export function siteLabel(hostname) {
+  const labels = hostname.toLowerCase().split('.');
+  const ccSecondLevel = labels.length >= 3 && labels.at(-1).length === 2 && labels.at(-2).length <= 3;
+  return labels[Math.max(labels.length - (ccSecondLevel ? 3 : 2), 0)];
+}
+
+function labelOf(url) {
+  try { return siteLabel(new URL(url).hostname); } catch { return null; }
+}
+
+/**
+ * The origins a search for the whole question also returns (same site label,
+ * any subdomain). A bare entity name is ambiguous — "Rust" is a language, a
+ * game and a film — and the site that owns the bare name need not be the one
+ * the question is about. A site the question's own search does not surface is
+ * not promoted as its live page. When that search returned nothing there is
+ * nothing to check against, and every origin passes.
+ *
+ * @param {Iterable<string>} origins candidate origins from the entity query
+ * @param {string[]} questionUrls result URLs of the other queries
+ * @returns {Set<string>}
+ */
+export function corroboratedOrigins(origins, questionUrls) {
+  const all = new Set(origins);
+  const labels = new Set((questionUrls || []).map(labelOf).filter(Boolean));
+  if (labels.size === 0) return all;
+  return new Set([...all].filter(origin => labels.has(labelOf(origin))));
+}
+
+const MONTHS = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'];
+const MONTH = '(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\\.?';
+const DATE_RE = new RegExp(
+  `\\b(\\d{4})-(\\d{2})-(\\d{2})\\b` +
+  `|\\b${MONTH}\\s+(\\d{1,2})(?:st|nd|rd|th)?,?\\s+(\\d{4})\\b` +
+  `|\\b(\\d{1,2})(?:st|nd|rd|th)?\\s+${MONTH}\\s+(\\d{4})\\b`,
+  'gi'
+);
+
+/** Every date in a text, written any of three ways, keyed as YYYY-MM-DD. */
+function datesIn(text) {
+  const found = [];
+  for (const m of String(text || '').matchAll(DATE_RE)) {
+    let y, mo, d;
+    if (m[1]) [y, mo, d] = [m[1], Number(m[2]), Number(m[3])];
+    else if (m[4]) [y, mo, d] = [m[6], MONTHS.indexOf(m[4].toLowerCase()) + 1, Number(m[5])];
+    else [y, mo, d] = [m[9], MONTHS.indexOf(m[8].toLowerCase()) + 1, Number(m[7])];
+    found.push({ raw: m[0], key: `${y}-${String(mo).padStart(2, '0')}-${String(d).padStart(2, '0')}`, index: m.index });
+  }
+  return found;
+}
+
+/** A date written right after one of these is when a page changed, not when the thing happened. */
+const UPDATE_LABEL = /\b(?:updated|modified|edited|revised|retrieved|accessed|crawled)\b[^\d\n]{0,20}$/i;
+
+/** A task that asks when something was updated wants exactly those dates. */
+const ASKS_FOR_UPDATES = /\b(?:updated?|modified|last (?:changed|edited))\b/i;
+
+/**
+ * Dates the answer states that its sources give only as a page's
+ * "last updated" / "modified" date, never as anything else. Each comes with
+ * the first source that labels it so, so a warning names a page the run used.
+ *
+ * @param {string} answer
+ * @param {{url: string, text?: string}[]} evidence
+ * @param {string} prompt
+ * @returns {{date: string, url: string}[]}
+ */
+export function updateLabelledDates(answer, evidence, prompt) {
+  if (!answer || ASKS_FOR_UPDATES.test(prompt || '')) return [];
+  const byKey = new Map();
+  for (const e of evidence || []) {
+    const text = e.text || '';
+    for (const { key, index } of datesIn(text)) {
+      const lineStart = text.lastIndexOf('\n', index - 1) + 1;
+      const updated = UPDATE_LABEL.test(text.slice(Math.max(lineStart, index - 40), index));
+      const seen = byKey.get(key) || { other: false, url: null };
+      if (updated) seen.url = seen.url || e.url;
+      else seen.other = true;
+      byKey.set(key, seen);
+    }
+  }
+  const flagged = [];
+  for (const { raw, key } of datesIn(answer.replace(/https?:\/\/\S+/g, ' '))) {
+    const seen = byKey.get(key);
+    if (seen?.url && !seen.other && !flagged.some(f => f.key === key)) flagged.push({ key, date: raw, url: seen.url });
+  }
+  return flagged.map(({ date, url }) => ({ date, url }));
+}

package/src/tools/basic/extractMetadata.js

@@ -88,9 +88,10 @@ export async function extractMetadataHandler({ url, user_agent, respect_robots,
       $('h1').first().text().trim() ||
       '';
 
-    const description =
+    const description = (
       $('meta[name="description"]').attr('content') ||
-      $('meta[property="og:description"]').attr('content') || '';
+      $('meta[property="og:description"]').attr('content') || ''
+    ).trim();
     const keywords = $('meta[name="keywords"]').attr('content') || '';
     const canonical = $('link[rel="canonical"]').attr('href') || '';
 
@@ -112,9 +113,13 @@ export async function extractMetadataHandler({ url, user_agent, respect_robots,
     const author = $('meta[name="author"]').attr('content') || '';
     const robots = $('meta[name="robots"]').attr('content') || '';
     const viewport = $('meta[name="viewport"]').attr('content') || '';
-    const charset =
+    // Only the charset token: the http-equiv form used to come back whole,
+    // "text/html; charset=utf-8" (live sweep 2026-10-20).
+    const charset = (
       $('meta[charset]').attr('charset') ||
-      $('meta[http-equiv="Content-Type"]').attr('content') || '';
+      ($('meta[http-equiv="Content-Type"]').attr('content') || '').match(/charset\s*=\s*["']?([^\s;"']+)/i)?.[1] ||
+      ''
+    ).trim().toUpperCase();
 
     const jsonLd = parseJsonLd($);
     const microdata = parseMicrodata($);

package/src/tools/basic/extractText.js

@@ -14,7 +14,7 @@ import { flattenText, gridTables, absoluteUrls } from 'crawlforge-extractors';
 import { JSDOM } from 'jsdom';
 import { Readability } from '@mozilla/readability';
 import { fetchLadder, ladderErrorResult, isJsonType } from '../../utils/fetchLadder.js';
-import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
+import { htmlToMarkdown, normalizeCodeBlocks } from '../../utils/htmlToMarkdown.js';
 import { tooLargeForReadability } from '../scrape/_mainContent.js';
 
 // Block-level elements whose boundaries should become paragraph breaks
@@ -67,11 +67,13 @@ export function extractBlockText($) {
  * @returns {string}
  */
 export function readabilityToMarkdown(html, pageUrl) {
-  let articleHtml = html;
+  // Code-block languages live in classes Readability strips (2.2).
+  const codeHtml = normalizeCodeBlocks(html);
+  let articleHtml = codeHtml;
   // A page too large for Readability is converted whole (R25).
-  if (!tooLargeForReadability(html)) {
+  if (!tooLargeForReadability(codeHtml)) {
     try {
-      const dom = new JSDOM(html, { url: pageUrl });
+      const dom = new JSDOM(codeHtml, { url: pageUrl });
       const reader = new Readability(dom.window.document);
       const article = reader.parse();
       if (article) articleHtml = article.content;

package/src/tools/basic/fetchUrl.js

@@ -36,6 +36,9 @@ export async function fetchUrlHandler({ url, headers, timeout, user_agent, respe
           contentType: response.headers.get('content-type') || 'unknown',
           size: body.length,
           url: response.url,
+          // Whether a redirect was followed, and the URL the body came from.
+          redirected: response.redirected === true,
+          final_url: response.url || url,
           // Time the target took to respond and deliver the body, in ms.
           // Excludes the per-host throttle, so it can back a latency check.
           responseTime: response._responseTime

package/src/tools/basic/scrapeStructured.js

@@ -5,6 +5,9 @@
  *     fix elements_found to report real per-field DOM match counts.
  * 3.2: Optional row_selector returns row-aligned records instead of the
  *     default parallel arrays, which are matched independently per field.
+ * 6.4: A field may be an object {selector, multiple}; multiple:true returns
+ *     every match as an array (inside a row it otherwise keeps only the first).
+ *     @href and @src values are resolved to absolute URLs against the page URL.
  */
 
 import { load } from 'cheerio';
@@ -25,7 +28,7 @@ const ATTR_SUFFIX_RE = /@[A-Za-z_:][\w:.-]*$/;
  * @param {string} raw
  * @returns {{ selector: string, attribute: string|null }}
  */
-function parseSelectorSpec(raw) {
+export function parseSelectorSpec(raw) {
   const match = ATTR_SUFFIX_RE.exec(raw);
   // Only treat it as an attribute suffix if it isn't inside brackets/quotes,
   // i.e. the selector portion before it has balanced [ ] and quotes.
@@ -41,6 +44,42 @@ function parseSelectorSpec(raw) {
   return { selector: raw, attribute: null };
 }
 
+/**
+ * A field is either a selector string ("css" or "css@attr") or an object
+ * { selector: "css@attr", multiple?: boolean }.
+ * @param {string|{ selector: string, multiple?: boolean }} field
+ * @returns {{ raw: string, selector: string, attribute: string|null, multiple: boolean }}
+ */
+export function parseFieldSpec(field) {
+  const raw = typeof field === 'string' ? field : field?.selector;
+  if (typeof raw !== 'string' || raw === '') {
+    throw new Error('a field must be a selector string or an object with a selector string');
+  }
+  return { raw, ...parseSelectorSpec(raw), multiple: typeof field === 'object' && field.multiple === true };
+}
+
+// Attributes holding a URL, resolved against the page URL so a relative
+// "/item?id=1" comes back usable.
+const URL_ATTRIBUTES = new Set(['href', 'src']);
+
+/**
+ * The value of one matched element: its text, or the named attribute (null
+ * when absent), with href/src made absolute against `baseUrl`.
+ */
+function extractValue($, el, attribute, baseUrl) {
+  if (!attribute) return elementText($, el);
+  const value = $(el).attr(attribute);
+  if (value == null) return null;
+  if (URL_ATTRIBUTES.has(attribute.toLowerCase()) && baseUrl) {
+    try {
+      return new URL(value.trim(), baseUrl).href;
+    } catch {
+      return value;
+    }
+  }
+  return value;
+}
+
 /**
  * Row-aligned extraction: every field is matched *inside* each row element, so
  * record N of the output is row N of the page. A field a row lacks is null
@@ -48,37 +87,46 @@ function parseSelectorSpec(raw) {
  * parallel-array output does — there each field is matched independently across
  * the whole document and the arrays are not row-aligned.
  *
+ * A field with multiple:true is an array of every match inside the row ([] when
+ * the row has none); otherwise it is the first match.
+ *
  * @param {import('cheerio').CheerioAPI} $
- * @param {{ row_selector: string, selectors: Record<string, string>, max_results?: number }} params
+ * @param {{ row_selector: string, selectors: Record<string, string|object>, max_results?: number, baseUrl?: string }} params
  * @returns {{ records: object[], rowsFound: number, matchCounts: Record<string, number> }}
  */
-function extractRows($, { row_selector, selectors, max_results }) {
+function extractRows($, { row_selector, selectors, max_results, baseUrl }) {
   const allRows = $(row_selector);
   // In row mode max_results caps rows, not per-field matches.
   const rows = (max_results != null && max_results > 0)
     ? allRows.slice(0, max_results)
     : allRows;
 
-  const specs = Object.entries(selectors).map(([field, raw]) => [field, raw, parseSelectorSpec(raw)]);
+  const specs = Object.entries(selectors).map(([field, spec]) => [field, spec]);
   const matchCounts = Object.fromEntries(specs.map(([field]) => [field, 0]));
 
   const records = rows.toArray().map((rowEl) => {
     const $row = $(rowEl);
     const record = {};
-    for (const [field, raw, { selector, attribute }] of specs) {
+    for (const [field, spec] of specs) {
       try {
+        const { selector, attribute, multiple } = parseFieldSpec(spec);
         // A field selector may name the row element itself (row "tr.athing",
         // field "tr@id"); .find() searches descendants only.
-        const el = $row.is(selector) ? $row : $row.find(selector).first();
-        if (el.length === 0) {
+        const matches = $row.is(selector) ? $row : $row.find(selector);
+        if (multiple) {
+          record[field] = matches.toArray().map((el) => extractValue($, el, attribute, baseUrl));
+          if (matches.length > 0) matchCounts[field] += 1;
+          continue;
+        }
+        if (matches.length === 0) {
           record[field] = null;
           continue;
         }
-        record[field] = attribute ? (el.attr(attribute) ?? null) : elementText($, el.get(0));
+        record[field] = extractValue($, matches.get(0), attribute, baseUrl);
         matchCounts[field] += 1;
       } catch (selectorError) {
         record[field] = {
-          error: `Invalid selector: ${raw}`,
+          error: `Invalid selector: ${typeof spec === 'string' ? spec : spec?.selector}`,
           message: selectorError.message
         };
       }
@@ -90,7 +138,7 @@ function extractRows($, { row_selector, selectors, max_results }) {
 }
 
 /**
- * @param {{ url: string, selectors: Record<string, string>, row_selector?: string,
+ * @param {{ url: string, selectors: Record<string, string|{ selector: string, multiple?: boolean }>, row_selector?: string,
  *   max_results?: number, user_agent?: string, respect_robots?: boolean }} params
  */
 export async function scrapeStructuredHandler({ url, selectors, row_selector, max_results, user_agent, respect_robots }) {
@@ -108,7 +156,7 @@ export async function scrapeStructuredHandler({ url, selectors, row_selector, ma
     const $ = load(html);
 
     if (row_selector) {
-      const { records, rowsFound, matchCounts } = extractRows($, { row_selector, selectors, max_results });
+      const { records, rowsFound, matchCounts } = extractRows($, { row_selector, selectors, max_results, baseUrl: response.url || url });
       return {
         content: [{
           type: 'text',
@@ -127,29 +175,26 @@ export async function scrapeStructuredHandler({ url, selectors, row_selector, ma
     const results = {};
     const matchCounts = {};
 
+    const baseUrl = response.url || url;
     for (const [fieldName, rawSelector] of Object.entries(selectors)) {
       try {
-        const { selector, attribute } = parseSelectorSpec(rawSelector);
+        const { selector, attribute, multiple } = parseFieldSpec(rawSelector);
         let elements = $(selector);
         const domCount = elements.length;
         matchCounts[fieldName] = domCount;
 
         if (domCount === 0) {
-          results[fieldName] = null;
+          results[fieldName] = multiple ? [] : null;
         } else {
           // Apply max_results cap if specified
           if (max_results != null && max_results > 0 && domCount > max_results) {
             elements = elements.slice(0, max_results);
           }
 
-          const extract = (el) => {
-            if (attribute) {
-              return $(el).attr(attribute) ?? null;
-            }
-            return elementText($, el);
-          };
+          const extract = (el) => extractValue($, el, attribute, baseUrl);
 
-          if (elements.length === 1) {
+          // multiple:true always answers with an array, even for one match.
+          if (elements.length === 1 && !multiple) {
             results[fieldName] = extract(elements.get(0));
           } else {
             // cheerio's .map().get() drops null/undefined results, which would
@@ -160,7 +205,7 @@ export async function scrapeStructuredHandler({ url, selectors, row_selector, ma
         }
       } catch (selectorError) {
         results[fieldName] = {
-          error: `Invalid selector: ${rawSelector}`,
+          error: `Invalid selector: ${typeof rawSelector === 'string' ? rawSelector : rawSelector?.selector}`,
           message: selectorError.message
         };
         matchCounts[fieldName] = 0;

package/src/tools/crawl/mapSite.js

@@ -48,6 +48,21 @@ export function searchScore(url, search) {
   return terms.filter((t) => haystack.includes(t)).length;
 }
 
+/**
+ * normalizeUrl, plus a directory index folded into its directory: `/` and
+ * `/index.html` were listed as two URLs (live sweep 2026-10-20). Kept here,
+ * not in the shared normalizer, so other tools' URL keys do not change.
+ * @param {string} url
+ * @returns {string}
+ */
+function normalizeMapUrl(url) {
+  const normalized = normalizeUrl(url);
+  const u = new URL(normalized);
+  if (!/\/index\.html?$/i.test(u.pathname)) return normalized;
+  u.pathname = u.pathname.replace(/index\.html?$/i, '');
+  return normalizeUrl(u.href);
+}
+
 // A gate refusal is the answer to the request, not a page we failed to read.
 const REFUSAL_CODES = new Set(['ROBOTS_DISALLOWED', 'HOST_BLOCKED', 'USE_REDDIT_SEARCH', 'SSRF_BLOCKED']);
 
@@ -159,7 +174,7 @@ export class MapSiteTool {
       // Try to fetch sitemap first
       if (validated.include_sitemap) {
         const sitemapUrls = await this.fetchSitemapUrls(baseUrl, domainFilter, poolLimit, scopePath);
-        sitemapUrls.forEach(url => urls.add(normalizeUrl(url)));
+        sitemapUrls.forEach(url => urls.add(normalizeMapUrl(url)));
       }
 
       // Fetch and parse the main page for additional URLs. A start page that
@@ -170,11 +185,11 @@ export class MapSiteTool {
         // The start page is a page of the site too: a one-page site with no
         // sitemap used to map to 0 URLs (R25 1.2).
         if (!domainFilter || domainFilter.isAllowed(validated.url).allowed) {
-          urls.add(normalizeUrl(validated.url));
+          urls.add(normalizeMapUrl(validated.url));
         }
         pageUrls.forEach(url => {
           if (urls.size < poolLimit) {
-            urls.add(normalizeUrl(url));
+            urls.add(normalizeMapUrl(url));
           }
         });
       } catch (error) {

package/src/tools/extract/extractStructured.js

@@ -154,6 +154,88 @@ export function verifyFieldShapes(data, schema) {
   return { data: out, unverified };
 }
 
+/**
+ * The value of the label/value row whose label names the field: a two-cell
+ * <tr> (`<th>UPC</th><td>…</td>`) or a <dt>/<dd> pair. Every word of the
+ * field name must be in the label; among several matches the label with the
+ * fewest extra words wins. The CSS fallback only looked at class, id, itemprop
+ * and meta, so a product-information table never answered anything:
+ * books.toscrape's UPC came back missing (live sweep 2026-10-20, 2.3).
+ *
+ * @param {import('cheerio').CheerioAPI} $
+ * @param {string} key - schema field name
+ * @returns {string|null}
+ */
+function labelledValue($, key) {
+  const want = keyWords(key);
+  if (want.length === 0) return null;
+  let best = null;
+  const consider = (label, value) => {
+    const words = keyWords(label);
+    if (!value || !want.every((w) => words.includes(w))) return;
+    const extra = words.length - want.length;
+    if (!best || extra < best.extra) best = { value, extra };
+  };
+  $('tr').each((_, tr) => {
+    const cells = $(tr).children('th, td');
+    if (cells.length === 2) consider(cells.first().text(), cells.last().text().trim());
+  });
+  $('dt').each((_, dt) => consider($(dt).text(), $(dt).next('dd').text().trim()));
+  return best ? best.value : null;
+}
+
+/** Field-name words a page's own wording leaves out: "in_stock_count". */
+const COUNT_WORDS = new Set(['count', 'number', 'num', 'total', 'qty', 'quantity']);
+
+/**
+ * For a numeric field: the text after the field's phrase on the first page
+ * line that has it with a number after it. `in_stock_count` reads
+ * "In stock (22 available)" as " (22 available)", which the caller coerces to
+ * 22. Only the text after the phrase counts, so a number earlier on the line
+ * is never taken.
+ *
+ * @param {string} text - flattened page text, one line per block
+ * @param {string} key
+ * @returns {string|null}
+ */
+function textAfterPhrase(text, key) {
+  const words = keyWords(key).filter((w) => !COUNT_WORDS.has(w));
+  if (words.length === 0) return null;
+  const phrase = new RegExp(`\\b${words.join('[\\s_-]*')}\\b`, 'i');
+  for (const line of text.split('\n')) {
+    const match = phrase.exec(line);
+    if (!match) continue;
+    const rest = line.slice(match.index + match[0].length);
+    if (/\d/.test(rest)) return rest;
+  }
+  return null;
+}
+
+const RATING_WORDS = { one: 1, two: 2, three: 3, four: 4, five: 5 };
+
+/**
+ * A star rating carried only in a class token: books.toscrape renders three
+ * stars as `<p class="star-rating Three">` with five identical icons inside,
+ * so no text on the page says 3. For a field named for a rating or stars, the
+ * first class token on a star/rating element that spells a number one to five
+ * is that number. Decision (2.3): it is returned as a number for a
+ * number/integer field and as its digit string ("3") otherwise.
+ *
+ * @param {import('cheerio').CheerioAPI} $
+ * @param {string} key
+ * @returns {number|null}
+ */
+function classRating($, key) {
+  if (!keyWords(key).some((w) => w === 'rating' || w === 'stars' || w === 'star')) return null;
+  for (const el of $('[class*="star"], [class*="rating"]').toArray()) {
+    for (const token of ($(el).attr('class') || '').split(/\s+/)) {
+      const n = RATING_WORDS[token.toLowerCase()];
+      if (n) return n;
+    }
+  }
+  return null;
+}
+
 /** Lowercased, whitespace-collapsed text for a containment check. */
 function squash(text) {
   return text.toLowerCase().replace(/\s+/g, ' ').trim();
@@ -502,10 +584,30 @@ export class ExtractStructuredTool {
       const failedRequired = extractionResult.valid !== true
         && (missingRequired.length > 0 || invalidRequired.length > 0);
 
+      // Every requested field is in `data`. A field no method filled used to
+      // be left out with no null and no note, so a caller could not tell
+      // "not on the page" from "not looked for" (live sweep 2026-10-20, 2.3).
+      // An empty array is an answer (no items), not a missing value.
+      const data = { ...(extractionResult.data || {}) };
+      const missingFields = [];
+      if (!Array.isArray(extractionResult.data)) {
+        for (const key of Object.keys(schema.properties || {})) {
+          const value = data[key];
+          if (value === undefined || value === null || (typeof value === 'string' && value.trim() === '')) {
+            data[key] = null;
+            missingFields.push(key);
+          }
+        }
+      }
+      if (missingFields.length > 0) {
+        extractionNotes.push(`Not found on the page, returned as null: ${missingFields.join(', ')}`);
+      }
+
       return {
         success: !failedRequired,
         url,
-        data: extractionResult.data || {},
+        data,
+        missing_fields: missingFields,
         extraction_method: extractionMethod,
         // The provider and model that produced an LLM extraction (R24).
         ...(extractionMethod === 'llm' && extractionResult.model
@@ -556,6 +658,7 @@ export class ExtractStructuredTool {
     const properties = schema.properties || {};
     const extracted = {};
     let fieldsFound = 0;
+    let pageText = null; // flattened lazily, only for a numeric field that needs it
 
     for (const [key, fieldSchema] of Object.entries(properties)) {
       // Schema keys become CSS selector fragments below (class/id/data-attr).
@@ -647,6 +750,15 @@ export class ExtractStructuredTool {
           }
         }
 
+        // A labelled table or definition-list row: <th>UPC</th><td>…</td>.
+        if (!(key in extracted)) {
+          const labelled = labelledValue($, key);
+          if (labelled) {
+            extracted[key] = this._coerceValue(labelled, fieldSchema);
+            fieldsFound++;
+          }
+        }
+
         // Last resort: semantic element selectors for well-known field names
         // (e.g. title -> <h1>/<title>) so common fields resolve without hints.
         if (!(key in extracted)) {
@@ -673,6 +785,27 @@ export class ExtractStructuredTool {
             }
           }
         }
+
+        // A rating encoded only as a class token (`star-rating Three`).
+        if (!(key in extracted)) {
+          const stars = classRating($, key);
+          if (stars !== null) {
+            const numeric = fieldSchema.type === 'number' || fieldSchema.type === 'integer';
+            extracted[key] = numeric ? stars : String(stars);
+            fieldsFound++;
+          }
+        }
+
+        // A numeric field stated in running text: "In stock (22 available)".
+        if (!(key in extracted) && (fieldSchema.type === 'number' || fieldSchema.type === 'integer')) {
+          if (pageText === null) pageText = flattenBodyText($);
+          const after = textAfterPhrase(pageText, key);
+          const value = after === null ? null : this._coerceValue(after, fieldSchema);
+          if (typeof value === 'number') {
+            extracted[key] = value;
+            fieldsFound++;
+          }
+        }
       } catch (_fieldError) {
         // Invalid selector for this key — skip the field, keep going.
         continue;