crawlforge-mcp-server 6.23.2 → 6.25.0

package/src/tools/extract/_fetchAndParse.js

@@ -16,6 +16,7 @@ import { safeFetch } from '../../utils/ssrfGuard.js';
 import { config } from '../../constants/config.js';
 import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
+import { pageFromResponse, pageHeader, throughPageCache } from '../../core/pageCache.js';
 
 const DEFAULT_TIMEOUT_MS = 15000;
 
@@ -108,16 +109,60 @@ export { flattenText as flattenBodyText };
  * @param {number}   [options.timeoutMs]
  * @param {string[]} [options.stripTags]   — additional tags to strip (default: script, style, noscript, iframe, svg)
  * @param {boolean}  [options.errorDocuments] — return a non-2xx HTML/text body with its status instead of throwing
+ * @param {number}   [options.maxAge]   — page cache: reuse a copy at most this many ms old (default 10 min; 0 = fetch fresh)
+ * @param {boolean}  [options.noCache]  — page cache: neither read nor store
  * @returns {Promise<{ html: string, $: import('cheerio').CheerioAPI, textContent: string, finalUrl: string, warnings: string[], status: number }>}
  */
 export async function fetchAndParse(url, options = {}) {
+  const { stripTags = ['script', 'style', 'noscript', 'iframe', 'svg'] } = options;
+
+  // The page itself may come from the process page cache (src/core/pageCache.js);
+  // the parse below is always this call's own, so no two calls share a `$`.
+  const page = await throughPageCache(
+    url,
+    options,
+    'html',
+    async () => {
+      const fetched = await fetchPage(url, options);
+      return { result: fetched, page: fetched };
+    },
+    (entry, warnings) => ({ ...entry, warnings })
+  );
+
+  const html = page.body;
+  const { warnings, status } = page;
+  const finalUrl = page.url;
+
+  // text/plain and application/json aren't markup: running them through the
+  // HTML parser risks misinterpreting substrings (e.g. a "<script>" value
+  // inside a JSON string) as real tags and stripping/mangling content.
+  if (classifyContentType(pageHeader(page, 'content-type')) === 'text') {
+    return { html, $: load(''), textContent: html.trim(), finalUrl, warnings, status };
+  }
+
+  const $ = load(html);
+
+  if (stripTags.length > 0) {
+    $(stripTags.join(', ')).remove();
+  }
+
+  const textContent = flattenText($);
+
+  return { html, $, textContent, finalUrl, warnings, status };
+}
+
+/**
+ * The network fetch behind fetchAndParse: the gate, the request, the status
+ * and content-type checks and the capped body read.
+ * @returns {Promise<object>} the page as pageFromResponse gives it, plus `warnings`
+ */
+async function fetchPage(url, options) {
   const {
     userAgent,
     respectRobots,
     tool,
     apiKey,
     timeoutMs = DEFAULT_TIMEOUT_MS,
-    stripTags = ['script', 'style', 'noscript', 'iframe', 'svg'],
     errorDocuments = false
   } = options;
 
@@ -157,21 +202,5 @@ export async function fetchAndParse(url, options = {}) {
   }
 
   const html = await readTextWithSizeCap(response);
-
-  // text/plain and application/json aren't markup: running them through the
-  // HTML parser risks misinterpreting substrings (e.g. a "<script>" value
-  // inside a JSON string) as real tags and stripping/mangling content.
-  if (classification === 'text') {
-    return { html, $: load(''), textContent: html.trim(), finalUrl: response.url, warnings, status: response.status };
-  }
-
-  const $ = load(html);
-
-  if (stripTags.length > 0) {
-    $(stripTags.join(', ')).remove();
-  }
-
-  const textContent = flattenText($);
-
-  return { html, $, textContent, finalUrl: response.url, warnings, status: response.status };
+  return { ...pageFromResponse(response, html), warnings };
 }

package/src/tools/extract/extractEmbeddedState.js

@@ -21,7 +21,7 @@ import { z } from 'zod';
 import { fetchAndParse } from './_fetchAndParse.js';
 // Both live in crawlforge-extractors so the REST API's extract_embedded_state
 // runs this exact reader — one RSC flight-stream parser, not two.
-import { extractEmbeddedState, selectJsonPath, findJsonPaths } from 'crawlforge-extractors';
+import { extractEmbeddedState, selectJsonPath, findJsonPaths, parseJsonPath } from 'crawlforge-extractors';
 import { stealthDocumentVerdict } from '../../utils/stealthVerdict.js';
 import { pageTitle } from '../../utils/pageTitle.js';
 import { setActualCost, markPreflightRefusal } from '../../server/requestContext.js';
@@ -38,6 +38,12 @@ const EMBEDDED_STATE_BASE_CREDITS = 2;
 // `find` ask for less up front.
 const LARGE_RESULT_BYTES = 256_000;
 
+// A path miss carries the key tree where the path stopped, so the next path is
+// aimed rather than guessed (repeat-call plan 2.1). Capped per object and in
+// characters; over the character cap it drops to one level.
+const MISS_TREE_KEYS = 30;
+const MISS_TREE_CHARS = 2000;
+
 // A wall, as opposed to a missing page or a broken server (G4: a 404 or a 5xx
 // never escalates — see aBrowserMightPass).
 const WALL_STATUSES = new Set([403, 429, 444]);
@@ -75,24 +81,49 @@ function label(value) {
  * Shared shape with the REST route — keep the two in step.
  * @param {unknown} value
  * @param {number} depth
+ * @param {number} [maxKeys] per object; the rest are counted under "…"
  */
-export function describeKeys(value, depth) {
+export function describeKeys(value, depth, maxKeys = Infinity) {
   if (Array.isArray(value)) {
     if (depth <= 1) return label(value);
     return value.length > 0
-      ? { length: value.length, first: describeKeys(value[0], depth - 1) }
+      ? { length: value.length, first: describeKeys(value[0], depth - 1, maxKeys) }
       : { length: 0 };
   }
   if (value !== null && typeof value === 'object') {
     const keys = {};
-    for (const [key, child] of Object.entries(value)) {
-      keys[key] = depth > 1 ? describeKeys(child, depth - 1) : label(child);
+    const entries = Object.entries(value);
+    for (const [key, child] of entries.slice(0, maxKeys)) {
+      keys[key] = depth > 1 ? describeKeys(child, depth - 1, maxKeys) : label(child);
     }
+    if (entries.length > maxKeys) keys['…'] = `${entries.length - maxKeys} more keys`;
     return keys;
   }
   return label(value);
 }
 
+/**
+ * The key tree where `path` stops resolving: `null` when the value there has
+ * no keys (the path ran into a string or number).
+ * @param {unknown} root
+ * @param {string} path
+ * @returns {{ at: string, tree: string } | null}
+ */
+function missKeyTree(root, path) {
+  let current = root;
+  const walked = [];
+  for (const segment of parseJsonPath(path)) {
+    const key = Array.isArray(current) ? Number(segment) : segment;
+    if (current === null || typeof current !== 'object' || !Object.hasOwn(current, key)) break;
+    current = current[key];
+    walked.push(segment);
+  }
+  if (current === null || typeof current !== 'object') return null;
+  let tree = JSON.stringify(describeKeys(current, 2, MISS_TREE_KEYS));
+  if (tree.length > MISS_TREE_CHARS) tree = JSON.stringify(describeKeys(current, 1, MISS_TREE_KEYS));
+  return { at: walked.join('.'), tree };
+}
+
 function httpError(status) {
   return new Error(`HTTP ${status}: ${STATUS_CODES[status] ?? ''}`.trimEnd());
 }
@@ -120,7 +151,7 @@ function failed(verdict, status, gotState) {
  */
 export function createExtractEmbeddedStateHandler({ escalateFetch } = {}) {
   return async function extractEmbeddedStateHandler({
-    url, path, keys_only, find, raw = false, escalate, escalate_engine = 'auto', wait_for, user_agent, respect_robots
+    url, path, keys_only, find, raw = false, escalate, escalate_engine = 'auto', wait_for, user_agent, respect_robots, maxAge
   }) {
     // Each replaces `data` with something else, so asking for both has no
     // answer. Nothing was fetched, so - like a robots refusal - it costs nothing.
@@ -149,6 +180,7 @@ export function createExtractEmbeddedStateHandler({ escalateFetch } = {}) {
       const fetched = await fetchAndParse(url, {
         userAgent: user_agent,
         respectRobots: respect_robots,
+        maxAge,
         tool: 'extract_embedded_state',
         errorDocuments: true
       });
@@ -283,7 +315,26 @@ export function createExtractEmbeddedStateHandler({ escalateFetch } = {}) {
         }
       }
 
-      const data = effectivePath ? selectJsonPath(state.data, effectivePath) : state.data;
+      let data = state.data;
+      if (effectivePath) {
+        try {
+          data = selectJsonPath(state.data, effectivePath);
+        } catch (error) {
+          // A path miss: the key tree where it stopped, and a next step that
+          // stays on this tool (the fallback hint would say to drop `path`).
+          reportCost();
+          const miss = missKeyTree(state.data, effectivePath);
+          return {
+            content: [{
+              type: 'text',
+              text: `Failed to extract embedded state: ${error.message}` +
+                (miss ? `\nKeys at ${miss.at ? `"${miss.at}"` : 'the top level'}: ${miss.tree}` : '') +
+                '\nNext step: Call again with a path built from the keys above, or with find:"<key>" to get the paths where a field of that name lives.'
+            }],
+            isError: true
+          };
+        }
+      }
       const bytes = Buffer.byteLength(JSON.stringify(data) ?? '');
 
       if (!path && !keys_only && find === undefined && bytes > LARGE_RESULT_BYTES) {

package/src/tools/extract/extractStructured.js

@@ -9,6 +9,7 @@ import { ElicitationHelper } from '../../core/ElicitationHelper.js'; // D1.4
 import { load } from 'cheerio';
 import { LLMManager } from '../../core/llm/LLMManager.js';
 import { CRAWLFORGE_USER_AGENT } from '../../utils/fetchIdentity.js';
+import { MAX_AGE_PARAM } from '../../utils/maxAgeParam.js';
 import { fetchAndParse, flattenBodyText } from './_fetchAndParse.js';
 import { fetchErrorMessage } from '../../utils/fetchError.js';
 import { extractMainContent } from '../scrape/_mainContent.js';
@@ -267,6 +268,7 @@ const ExtractStructuredSchema = z.object({
   selectorHints: z.record(z.string()).optional(),
   respect_robots: z.boolean().optional(),
   user_agent: z.string().optional(),
+  ...MAX_AGE_PARAM,
   verify_numbers: z.boolean().optional().default(true)
 });
 
@@ -324,7 +326,7 @@ export class ExtractStructuredTool {
 
     try {
       const validated = ExtractStructuredSchema.parse(params);
-      const { url, schema: requestedSchema, prompt, llmConfig, fallbackToSelectors, selectorHints, respect_robots, user_agent, verify_numbers } = validated;
+      const { url, schema: requestedSchema, prompt, llmConfig, fallbackToSelectors, selectorHints, respect_robots, user_agent, maxAge, verify_numbers } = validated;
       // `type` is optional on input, and the tool's own example omits it. The
       // validator reads a typeless schema as "anything", so the CSS fallback
       // reported valid:true for "2 reviews" in an integer field (R24). A schema
@@ -387,6 +389,7 @@ export class ExtractStructuredTool {
       const { html, $, textContent, warnings } = await fetchAndParse(url, {
         userAgent: user_agent || this.userAgent,
         respectRobots: respect_robots,
+        maxAge,
         tool: 'extract_structured'
       });
 

package/src/tools/extract/extractWithLlm.js

@@ -440,6 +440,7 @@ export class ExtractWithLlm {
    * @param {number}  [params.maxTokens] - Max output tokens (default 4096)
    * @param {boolean} [params.respect_robots] - Per-request robots.txt override
    * @param {string}  [params.user_agent]     - Per-request identity override
+   * @param {number}  [params.maxAge]         - Page-cache age limit in ms, passed to fetchAndParse
    * @param {boolean} [params.verify_numbers] - Numeric provenance guard (default true)
    * @param {AbortSignal} [params.signal] - Aborts the model request(s); an aborted
    *   call skips the retry and the sampling fallback
@@ -456,6 +457,7 @@ export class ExtractWithLlm {
       maxTokens = 4096,
       respect_robots,
       user_agent,
+      maxAge,
       verify_numbers = true,
       signal
     } = params;
@@ -501,6 +503,7 @@ export class ExtractWithLlm {
         const { html, textContent, warnings } = await fetchAndParse(url, {
           respectRobots: respect_robots,
           userAgent: user_agent,
+          maxAge,
           tool: 'extract_with_llm'
         });
         text = textContent;

package/src/tools/scrape/_brandingColor.js

@@ -0,0 +1,185 @@
+/**
+ * _brandingColor.js — CSS colour parsing for the branding format.
+ *
+ * parseColor turns hex, rgb(), hsl(), oklch(), oklab(), lab(), lch() and the
+ * basic named colours into { r, g, b, a } (channels 0-255, alpha 0-1). The
+ * wide-gamut forms matter because Tailwind v4 writes its palette in oklch and
+ * its build (Lightning CSS) ships it as lab(). The helpers below it
+ * compare colours: how saturated one is, how light, how far apart two are.
+ */
+
+const NAMED_COLORS = {
+  black: '#000000', white: '#ffffff', red: '#ff0000', green: '#008000',
+  blue: '#0000ff', yellow: '#ffff00', orange: '#ffa500', purple: '#800080',
+  gray: '#808080', grey: '#808080', silver: '#c0c0c0', navy: '#000080',
+  teal: '#008080', maroon: '#800000', olive: '#808000', lime: '#00ff00',
+  aqua: '#00ffff', fuchsia: '#ff00ff',
+};
+
+function clamp(n, lo, hi) {
+  return Math.max(lo, Math.min(hi, n));
+}
+
+function toHex2(n) {
+  return clamp(Math.round(n), 0, 255).toString(16).padStart(2, '0');
+}
+
+// A channel or alpha argument: `none` is 0, a percentage scales to `full`.
+function num(part, full = 1) {
+  if (part === 'none') return 0;
+  if (part.endsWith('%')) return (parseFloat(part) / 100) * full;
+  return parseFloat(part);
+}
+
+function alphaOf(part) {
+  if (part == null) return 1;
+  const a = num(part);
+  return Number.isNaN(a) ? 1 : clamp(a, 0, 1);
+}
+
+function hslToRgb(h, s, l) {
+  const c = (1 - Math.abs(2 * l - 1)) * s;
+  const x = c * (1 - Math.abs(((h / 60) % 2) - 1));
+  const mm = l - c / 2;
+  let r = 0, g = 0, b = 0;
+  const hh = ((h % 360) + 360) % 360;
+  if (hh < 60) [r, g, b] = [c, x, 0];
+  else if (hh < 120) [r, g, b] = [x, c, 0];
+  else if (hh < 180) [r, g, b] = [0, c, x];
+  else if (hh < 240) [r, g, b] = [0, x, c];
+  else if (hh < 300) [r, g, b] = [x, 0, c];
+  else [r, g, b] = [c, 0, x];
+  return [(r + mm) * 255, (g + mm) * 255, (b + mm) * 255];
+}
+
+// Linear-light channels -> gamma-encoded sRGB 0-255, clipped to the gamut.
+function encode(lin) {
+  return lin.map((x) => {
+    const v = clamp(x, 0, 1);
+    return (v <= 0.0031308 ? 12.92 * v : 1.055 * v ** (1 / 2.4) - 0.055) * 255;
+  });
+}
+
+// CIE Lab (D50) -> XYZ D50 -> Bradford to D65 -> linear sRGB (CSS Color 4 matrices).
+function labToRgb(L, a, b) {
+  const e = 216 / 24389;
+  const k = 24389 / 27;
+  const fy = (L + 16) / 116;
+  const fx = fy + a / 500;
+  const fz = fy - b / 200;
+  const x = (fx ** 3 > e ? fx ** 3 : (116 * fx - 16) / k) * 0.3457 / 0.3585;
+  const y = L > k * e ? fy ** 3 : L / k;
+  const z = (fz ** 3 > e ? fz ** 3 : (116 * fz - 16) / k) * (1 - 0.3457 - 0.3585) / 0.3585;
+  const X = 0.9554734527042182 * x - 0.023098536874261423 * y + 0.0632593086610217 * z;
+  const Y = -0.028369706963208136 * x + 1.0099954580058226 * y + 0.021041398966943008 * z;
+  const Z = 0.012314001688319899 * x - 0.020507696433477912 * y + 1.3303659366080753 * z;
+  return encode([
+    3.2409699419045226 * X - 1.537383177570094 * Y - 0.4986107602930034 * Z,
+    -0.9692436362808796 * X + 1.8759675015077202 * Y + 0.04155505740717559 * Z,
+    0.05563007969699366 * X - 0.20397695888897652 * Y + 1.0569715142428786 * Z,
+  ]);
+}
+
+// OKLab -> sRGB (Björn Ottosson's reference matrices), clipped to the gamut.
+function oklabToRgb(L, a, b) {
+  const l = (L + 0.3963377774 * a + 0.2158037573 * b) ** 3;
+  const m = (L - 0.1055613458 * a - 0.0638541728 * b) ** 3;
+  const s = (L - 0.0894841775 * a - 1.2914855480 * b) ** 3;
+  return encode([
+    4.0767416621 * l - 3.3077115913 * m + 0.2309699292 * s,
+    -1.2684380046 * l + 2.6097574011 * m - 0.3413193965 * s,
+    -0.0041960863 * l - 0.7034186147 * m + 1.7076147010 * s,
+  ]);
+}
+
+/**
+ * Parse a CSS colour into { r, g, b, a }. Returns null for anything that is
+ * not a literal colour (gradients, currentColor, var(), transparent).
+ */
+export function parseColor(raw) {
+  if (!raw) return null;
+  const v = String(raw).trim().toLowerCase();
+
+  const named = NAMED_COLORS[v];
+  if (named) return parseColor(named);
+
+  let m = v.match(/^#([0-9a-f]{3,8})$/);
+  if (m) {
+    let h = m[1];
+    if (h.length === 3 || h.length === 4) h = h.split('').map((c) => c + c).join('');
+    if (h.length !== 6 && h.length !== 8) return null;
+    return {
+      r: parseInt(h.slice(0, 2), 16),
+      g: parseInt(h.slice(2, 4), 16),
+      b: parseInt(h.slice(4, 6), 16),
+      a: h.length === 8 ? parseInt(h.slice(6, 8), 16) / 255 : 1,
+    };
+  }
+
+  m = v.match(/^(rgba?|hsla?|oklch|oklab|lch|lab)\(([^()]+)\)$/);
+  if (!m) return null;
+  const parts = m[2].split(/[,/\s]+/).filter(Boolean);
+  if (parts.length < 3) return null;
+  const a = alphaOf(parts[3]);
+  let rgb;
+  switch (m[1]) {
+    case 'rgb':
+    case 'rgba':
+      rgb = parts.slice(0, 3).map((p) => num(p, 255));
+      break;
+    case 'hsl':
+    case 'hsla':
+      rgb = hslToRgb(parseFloat(parts[0]), num(parts[1].endsWith('%') ? parts[1] : parts[1] + '%'), num(parts[2].endsWith('%') ? parts[2] : parts[2] + '%'));
+      break;
+    case 'oklch': {
+      const L = num(parts[0]);
+      const C = num(parts[1], 0.4);
+      const H = (num(parts[2]) * Math.PI) / 180;
+      rgb = oklabToRgb(L, C * Math.cos(H), C * Math.sin(H));
+      break;
+    }
+    case 'oklab':
+      rgb = oklabToRgb(num(parts[0]), num(parts[1], 0.4), num(parts[2], 0.4));
+      break;
+    case 'lch': {
+      const C = num(parts[1], 150);
+      const H = (num(parts[2]) * Math.PI) / 180;
+      rgb = labToRgb(num(parts[0], 100), C * Math.cos(H), C * Math.sin(H));
+      break;
+    }
+    case 'lab':
+      rgb = labToRgb(num(parts[0], 100), num(parts[1], 125), num(parts[2], 125));
+      break;
+  }
+  if (rgb.some((x) => Number.isNaN(x))) return null;
+  return { r: rgb[0], g: rgb[1], b: rgb[2], a };
+}
+
+/** '#rrggbb', alpha dropped. */
+export function toHex({ r, g, b }) {
+  return '#' + toHex2(r) + toHex2(g) + toHex2(b);
+}
+
+/** '#rrggbb', or '#rrggbbaa' when the colour is translucent. */
+export function toHexAlpha(c) {
+  return c.a < 1 ? toHex(c) + toHex2(c.a * 255) : toHex(c);
+}
+
+/** 0 for a grey, 1 for a fully saturated colour. */
+export function chroma({ r, g, b }) {
+  return (Math.max(r, g, b) - Math.min(r, g, b)) / 255;
+}
+
+/** WCAG relative luminance, 0 (black) to 1 (white). */
+export function luminance({ r, g, b }) {
+  const lin = (x) => {
+    const v = x / 255;
+    return v <= 0.03928 ? v / 12.92 : ((v + 0.055) / 1.055) ** 2.4;
+  };
+  return 0.2126 * lin(r) + 0.7152 * lin(g) + 0.0722 * lin(b);
+}
+
+/** Euclidean distance in RGB, 0 to ~441. */
+export function distance(x, y) {
+  return Math.hypot(x.r - y.r, x.g - y.g, x.b - y.b);
+}