crawlforge-mcp-server 6.23.2 → 6.24.0

package/src/tools/extract/extractEmbeddedState.js

@@ -21,7 +21,7 @@ import { z } from 'zod';
 import { fetchAndParse } from './_fetchAndParse.js';
 // Both live in crawlforge-extractors so the REST API's extract_embedded_state
 // runs this exact reader — one RSC flight-stream parser, not two.
-import { extractEmbeddedState, selectJsonPath, findJsonPaths } from 'crawlforge-extractors';
+import { extractEmbeddedState, selectJsonPath, findJsonPaths, parseJsonPath } from 'crawlforge-extractors';
 import { stealthDocumentVerdict } from '../../utils/stealthVerdict.js';
 import { pageTitle } from '../../utils/pageTitle.js';
 import { setActualCost, markPreflightRefusal } from '../../server/requestContext.js';
@@ -38,6 +38,12 @@ const EMBEDDED_STATE_BASE_CREDITS = 2;
 // `find` ask for less up front.
 const LARGE_RESULT_BYTES = 256_000;
 
+// A path miss carries the key tree where the path stopped, so the next path is
+// aimed rather than guessed (repeat-call plan 2.1). Capped per object and in
+// characters; over the character cap it drops to one level.
+const MISS_TREE_KEYS = 30;
+const MISS_TREE_CHARS = 2000;
+
 // A wall, as opposed to a missing page or a broken server (G4: a 404 or a 5xx
 // never escalates — see aBrowserMightPass).
 const WALL_STATUSES = new Set([403, 429, 444]);
@@ -75,24 +81,49 @@ function label(value) {
  * Shared shape with the REST route — keep the two in step.
  * @param {unknown} value
  * @param {number} depth
+ * @param {number} [maxKeys] per object; the rest are counted under "…"
  */
-export function describeKeys(value, depth) {
+export function describeKeys(value, depth, maxKeys = Infinity) {
   if (Array.isArray(value)) {
     if (depth <= 1) return label(value);
     return value.length > 0
-      ? { length: value.length, first: describeKeys(value[0], depth - 1) }
+      ? { length: value.length, first: describeKeys(value[0], depth - 1, maxKeys) }
       : { length: 0 };
   }
   if (value !== null && typeof value === 'object') {
     const keys = {};
-    for (const [key, child] of Object.entries(value)) {
-      keys[key] = depth > 1 ? describeKeys(child, depth - 1) : label(child);
+    const entries = Object.entries(value);
+    for (const [key, child] of entries.slice(0, maxKeys)) {
+      keys[key] = depth > 1 ? describeKeys(child, depth - 1, maxKeys) : label(child);
     }
+    if (entries.length > maxKeys) keys['…'] = `${entries.length - maxKeys} more keys`;
     return keys;
   }
   return label(value);
 }
 
+/**
+ * The key tree where `path` stops resolving: `null` when the value there has
+ * no keys (the path ran into a string or number).
+ * @param {unknown} root
+ * @param {string} path
+ * @returns {{ at: string, tree: string } | null}
+ */
+function missKeyTree(root, path) {
+  let current = root;
+  const walked = [];
+  for (const segment of parseJsonPath(path)) {
+    const key = Array.isArray(current) ? Number(segment) : segment;
+    if (current === null || typeof current !== 'object' || !Object.hasOwn(current, key)) break;
+    current = current[key];
+    walked.push(segment);
+  }
+  if (current === null || typeof current !== 'object') return null;
+  let tree = JSON.stringify(describeKeys(current, 2, MISS_TREE_KEYS));
+  if (tree.length > MISS_TREE_CHARS) tree = JSON.stringify(describeKeys(current, 1, MISS_TREE_KEYS));
+  return { at: walked.join('.'), tree };
+}
+
 function httpError(status) {
   return new Error(`HTTP ${status}: ${STATUS_CODES[status] ?? ''}`.trimEnd());
 }
@@ -120,7 +151,7 @@ function failed(verdict, status, gotState) {
  */
 export function createExtractEmbeddedStateHandler({ escalateFetch } = {}) {
   return async function extractEmbeddedStateHandler({
-    url, path, keys_only, find, raw = false, escalate, escalate_engine = 'auto', wait_for, user_agent, respect_robots
+    url, path, keys_only, find, raw = false, escalate, escalate_engine = 'auto', wait_for, user_agent, respect_robots, maxAge
   }) {
     // Each replaces `data` with something else, so asking for both has no
     // answer. Nothing was fetched, so - like a robots refusal - it costs nothing.
@@ -149,6 +180,7 @@ export function createExtractEmbeddedStateHandler({ escalateFetch } = {}) {
       const fetched = await fetchAndParse(url, {
         userAgent: user_agent,
         respectRobots: respect_robots,
+        maxAge,
         tool: 'extract_embedded_state',
         errorDocuments: true
       });
@@ -283,7 +315,26 @@ export function createExtractEmbeddedStateHandler({ escalateFetch } = {}) {
         }
       }
 
-      const data = effectivePath ? selectJsonPath(state.data, effectivePath) : state.data;
+      let data = state.data;
+      if (effectivePath) {
+        try {
+          data = selectJsonPath(state.data, effectivePath);
+        } catch (error) {
+          // A path miss: the key tree where it stopped, and a next step that
+          // stays on this tool (the fallback hint would say to drop `path`).
+          reportCost();
+          const miss = missKeyTree(state.data, effectivePath);
+          return {
+            content: [{
+              type: 'text',
+              text: `Failed to extract embedded state: ${error.message}` +
+                (miss ? `\nKeys at ${miss.at ? `"${miss.at}"` : 'the top level'}: ${miss.tree}` : '') +
+                '\nNext step: Call again with a path built from the keys above, or with find:"<key>" to get the paths where a field of that name lives.'
+            }],
+            isError: true
+          };
+        }
+      }
       const bytes = Buffer.byteLength(JSON.stringify(data) ?? '');
 
       if (!path && !keys_only && find === undefined && bytes > LARGE_RESULT_BYTES) {

package/src/tools/extract/extractStructured.js

@@ -9,6 +9,7 @@ import { ElicitationHelper } from '../../core/ElicitationHelper.js'; // D1.4
 import { load } from 'cheerio';
 import { LLMManager } from '../../core/llm/LLMManager.js';
 import { CRAWLFORGE_USER_AGENT } from '../../utils/fetchIdentity.js';
+import { MAX_AGE_PARAM } from '../../utils/maxAgeParam.js';
 import { fetchAndParse, flattenBodyText } from './_fetchAndParse.js';
 import { fetchErrorMessage } from '../../utils/fetchError.js';
 import { extractMainContent } from '../scrape/_mainContent.js';
@@ -267,6 +268,7 @@ const ExtractStructuredSchema = z.object({
   selectorHints: z.record(z.string()).optional(),
   respect_robots: z.boolean().optional(),
   user_agent: z.string().optional(),
+  ...MAX_AGE_PARAM,
   verify_numbers: z.boolean().optional().default(true)
 });
 
@@ -324,7 +326,7 @@ export class ExtractStructuredTool {
 
     try {
       const validated = ExtractStructuredSchema.parse(params);
-      const { url, schema: requestedSchema, prompt, llmConfig, fallbackToSelectors, selectorHints, respect_robots, user_agent, verify_numbers } = validated;
+      const { url, schema: requestedSchema, prompt, llmConfig, fallbackToSelectors, selectorHints, respect_robots, user_agent, maxAge, verify_numbers } = validated;
       // `type` is optional on input, and the tool's own example omits it. The
       // validator reads a typeless schema as "anything", so the CSS fallback
       // reported valid:true for "2 reviews" in an integer field (R24). A schema
@@ -387,6 +389,7 @@ export class ExtractStructuredTool {
       const { html, $, textContent, warnings } = await fetchAndParse(url, {
         userAgent: user_agent || this.userAgent,
         respectRobots: respect_robots,
+        maxAge,
         tool: 'extract_structured'
       });
 

package/src/tools/extract/extractWithLlm.js

@@ -440,6 +440,7 @@ export class ExtractWithLlm {
    * @param {number}  [params.maxTokens] - Max output tokens (default 4096)
    * @param {boolean} [params.respect_robots] - Per-request robots.txt override
    * @param {string}  [params.user_agent]     - Per-request identity override
+   * @param {number}  [params.maxAge]         - Page-cache age limit in ms, passed to fetchAndParse
    * @param {boolean} [params.verify_numbers] - Numeric provenance guard (default true)
    * @param {AbortSignal} [params.signal] - Aborts the model request(s); an aborted
    *   call skips the retry and the sampling fallback
@@ -456,6 +457,7 @@ export class ExtractWithLlm {
       maxTokens = 4096,
       respect_robots,
       user_agent,
+      maxAge,
       verify_numbers = true,
       signal
     } = params;
@@ -501,6 +503,7 @@ export class ExtractWithLlm {
         const { html, textContent, warnings } = await fetchAndParse(url, {
           respectRobots: respect_robots,
           userAgent: user_agent,
+          maxAge,
           tool: 'extract_with_llm'
         });
         text = textContent;

package/src/tools/scrape/unifiedScrape.js

@@ -33,6 +33,7 @@ import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
 import { stripHiddenFromDom } from '../../utils/hiddenContent.js';
 import { extractBlockText } from '../basic/extractText.js';
 import { pageTitle } from '../../utils/pageTitle.js';
+import { MAX_AGE_PARAM } from '../../utils/maxAgeParam.js';
 import { noteHostBlocked, clearHostBlocked, getHostBlock } from '../../utils/hostRateLimiter.js';
 
 // ── Schema ────────────────────────────────────────────────────────────────────
@@ -168,7 +169,8 @@ export const UnifiedScrapeSchema = z.object({
   // Compliance overrides, per request: identify as yourself for a target you
   // have your own agreement with, and take responsibility for ignoring robots.
   user_agent: z.string().optional(),
-  respect_robots: z.boolean().optional()
+  respect_robots: z.boolean().optional(),
+  ...MAX_AGE_PARAM
 });
 
 // ── Helpers ───────────────────────────────────────────────────────────────────
@@ -432,6 +434,7 @@ export class UnifiedScrapeTool {
           timeoutMs,
           userAgent: validated.user_agent,
           respectRobots: validated.respect_robots,
+          maxAge: validated.maxAge,
           tool: 'scrape',
           stripTags: [], // we handle boilerplate ourselves
           // A real wall reaches a plain fetch as a 403 with the challenge in

package/src/utils/canonicalUrl.js

@@ -0,0 +1,29 @@
+/**
+ * canonicalUrl — one spelling per page, for keys that must match when two
+ * calls ask for the same page (the invocation log's urlKey now, the page cache
+ * of REPEAT_CALL_REDUCTION_PLAN Phase 1 next).
+ *
+ * Rules (plan 1.2): lowercase scheme and host, drop the default port (80 for
+ * http, 443 for https) and the #fragment, keep path and query string exactly
+ * as written. An empty path is the root path ("/"), which is what is requested.
+ * Anything that does not parse as a URL returns null.
+ */
+
+export function canonicalUrl(input) {
+  if (typeof input !== 'string') return null;
+  let u;
+  try { u = new URL(input.trim()); } catch { return null; }
+
+  if (u.protocol !== 'http:' && u.protocol !== 'https:') {
+    u.hash = '';
+    return u.href;
+  }
+
+  // WHATWG URL already lowercases scheme and host and drops a default port,
+  // but it also rewrites the path (dot segments, percent-encoding), so the
+  // path and query are taken from the original string instead.
+  const rest = input.trim().match(/^[a-z][a-z0-9+.-]*:\/\/[^/?#]*([^#]*)/i)?.[1] ?? '';
+  const userinfo = u.username ? `${u.username}${u.password ? `:${u.password}` : ''}@` : '';
+  const pathAndQuery = rest === '' || rest.startsWith('?') ? `/${rest}` : rest;
+  return `${u.protocol}//${userinfo}${u.host}${pathAndQuery}`;
+}

package/src/utils/fetchLadder.js

@@ -122,7 +122,7 @@ async function plainFetch(url, options) {
 
 /**
  * @param {string} url
- * @param {{ tool: 'extract_text'|'extract_links', userAgent?: string, respectRobots?: boolean,
+ * @param {{ tool: 'extract_text'|'extract_links', userAgent?: string, respectRobots?: boolean, maxAge?: number,
  *   escalate?: boolean, escalateEngine?: string,
  *   escalateFetch?: (args: { url: string, engine: string, respectRobots?: boolean }) => Promise<object> }} options
  * @returns {Promise<
@@ -132,10 +132,10 @@ async function plainFetch(url, options) {
  *   `fields` carries `escalated` (only when escalate was asked for), `stealth`
  *   (when the stage ran) and `rendered: false` (a client-rendered shell)
  */
-export async function fetchLadder(url, { tool, userAgent, respectRobots, escalate = false, escalateEngine = 'auto', escalateFetch } = {}) {
+export async function fetchLadder(url, { tool, userAgent, respectRobots, maxAge, escalate = false, escalateEngine = 'auto', escalateFetch } = {}) {
   let escalationRan = false;
   try {
-    const response = await plainFetch(url, { timeout: LADDER_TIMEOUT_MS, userAgent, respectRobots, tool });
+    const response = await plainFetch(url, { timeout: LADDER_TIMEOUT_MS, userAgent, respectRobots, maxAge, tool });
     const type = mediaType(response);
     if (response.ok && type && !TEXTUAL_TYPE.test(type)) return { unsupported: type };
 

package/src/utils/maxAgeParam.js

@@ -0,0 +1,18 @@
+/**
+ * maxAgeParam — the one declaration of `maxAge` (repeat-call reduction plan,
+ * Phase 1, 1.5 / D2). Passed through as `options.maxAge` to the two page-cache
+ * seams, fetchWithTimeout and fetchAndParse; undefined means the server's
+ * default TTL there.
+ */
+
+import { z } from 'zod';
+import { pageCacheActive } from '../core/pageCache.js';
+
+export const MAX_AGE_PARAM = {
+  maxAge: z.number().int().min(0).optional().describe("Reuse this session's copy of the page if it was fetched within this many ms (default 600000 = 10 min); 0 always fetches fresh.")
+};
+
+// Phase 3 (3.1): the one sentence the nine cached tools' descriptions carry.
+// Empty where the cache is off (hosted HTTP, CRAWLFORGE_PAGE_CACHE_MB=0).
+export const PAGE_CACHE_TEXT = ' Asking again for the same page with different formats, paths or selectors reuses this session\'s copy for 10 minutes at 1 credit; pass maxAge: 0 to force a fresh fetch.';
+export const PAGE_CACHE_SENTENCE = pageCacheActive() ? PAGE_CACHE_TEXT : '';

package/src/utils/robotsGate.js

@@ -273,6 +273,29 @@ export function outboundHeaders(userAgent, signature = {}) {
   };
 }
 
+/**
+ * The gate's decision, refusing as {@link preflightFetch} does, without the
+ * per-host throttle. A page-cache hit (src/core/pageCache.js) sends no
+ * request, so there is nothing to space, but the blocklist and robots.txt are
+ * asked again: either may have changed since the copy was fetched.
+ *
+ * @param {string} url
+ * @param {object} [options] see {@link robotsPreflight}
+ * @param {{ redirectedFrom?: string }} [reached] set when `url` is where a
+ *   redirect from the requested URL led
+ * @returns {Promise<{ allowed: true, userAgent: string, crawlDelayMs: number,
+ *   warnings: string[], overridden: boolean }>}
+ * @throws {BlockedHostError|RobotsDisallowedError}
+ */
+export async function preflightDecision(url, options = {}, reached = {}) {
+  const decision = await robotsPreflight(url, options);
+  if (!decision.allowed) {
+    markPreflightRefusal('ROBOTS_DISALLOWED');
+    throw new RobotsDisallowedError(url, reached);
+  }
+  return decision;
+}
+
 /**
  * The call-site helper: run the gate, honour Crawl-delay and any recorded
  * `Retry-After`, and hand back the identity headers to send and the gate for
@@ -286,11 +309,7 @@ export function outboundHeaders(userAgent, signature = {}) {
  * @throws {BlockedHostError|RobotsDisallowedError}
  */
 export async function preflightFetch(url, options = {}) {
-  const decision = await robotsPreflight(url, options);
-  if (!decision.allowed) {
-    markPreflightRefusal('ROBOTS_DISALLOWED');
-    throw new RobotsDisallowedError(url);
-  }
+  const decision = await preflightDecision(url, options);
 
   await throttleHost(url, { crawlDelayMs: decision.crawlDelayMs });