crawlforge-mcp-server 6.23.1 → 6.24.0

package/src/tools/extract/extractEmbeddedState.js

@@ -21,7 +21,7 @@ import { z } from 'zod';
 import { fetchAndParse } from './_fetchAndParse.js';
 // Both live in crawlforge-extractors so the REST API's extract_embedded_state
 // runs this exact reader — one RSC flight-stream parser, not two.
-import { extractEmbeddedState, selectJsonPath, findJsonPaths } from 'crawlforge-extractors';
+import { extractEmbeddedState, selectJsonPath, findJsonPaths, parseJsonPath } from 'crawlforge-extractors';
 import { stealthDocumentVerdict } from '../../utils/stealthVerdict.js';
 import { pageTitle } from '../../utils/pageTitle.js';
 import { setActualCost, markPreflightRefusal } from '../../server/requestContext.js';
@@ -38,6 +38,12 @@ const EMBEDDED_STATE_BASE_CREDITS = 2;
 // `find` ask for less up front.
 const LARGE_RESULT_BYTES = 256_000;
 
+// A path miss carries the key tree where the path stopped, so the next path is
+// aimed rather than guessed (repeat-call plan 2.1). Capped per object and in
+// characters; over the character cap it drops to one level.
+const MISS_TREE_KEYS = 30;
+const MISS_TREE_CHARS = 2000;
+
 // A wall, as opposed to a missing page or a broken server (G4: a 404 or a 5xx
 // never escalates — see aBrowserMightPass).
 const WALL_STATUSES = new Set([403, 429, 444]);
@@ -75,24 +81,49 @@ function label(value) {
  * Shared shape with the REST route — keep the two in step.
  * @param {unknown} value
  * @param {number} depth
+ * @param {number} [maxKeys] per object; the rest are counted under "…"
  */
-export function describeKeys(value, depth) {
+export function describeKeys(value, depth, maxKeys = Infinity) {
   if (Array.isArray(value)) {
     if (depth <= 1) return label(value);
     return value.length > 0
-      ? { length: value.length, first: describeKeys(value[0], depth - 1) }
+      ? { length: value.length, first: describeKeys(value[0], depth - 1, maxKeys) }
       : { length: 0 };
   }
   if (value !== null && typeof value === 'object') {
     const keys = {};
-    for (const [key, child] of Object.entries(value)) {
-      keys[key] = depth > 1 ? describeKeys(child, depth - 1) : label(child);
+    const entries = Object.entries(value);
+    for (const [key, child] of entries.slice(0, maxKeys)) {
+      keys[key] = depth > 1 ? describeKeys(child, depth - 1, maxKeys) : label(child);
     }
+    if (entries.length > maxKeys) keys['…'] = `${entries.length - maxKeys} more keys`;
     return keys;
   }
   return label(value);
 }
 
+/**
+ * The key tree where `path` stops resolving: `null` when the value there has
+ * no keys (the path ran into a string or number).
+ * @param {unknown} root
+ * @param {string} path
+ * @returns {{ at: string, tree: string } | null}
+ */
+function missKeyTree(root, path) {
+  let current = root;
+  const walked = [];
+  for (const segment of parseJsonPath(path)) {
+    const key = Array.isArray(current) ? Number(segment) : segment;
+    if (current === null || typeof current !== 'object' || !Object.hasOwn(current, key)) break;
+    current = current[key];
+    walked.push(segment);
+  }
+  if (current === null || typeof current !== 'object') return null;
+  let tree = JSON.stringify(describeKeys(current, 2, MISS_TREE_KEYS));
+  if (tree.length > MISS_TREE_CHARS) tree = JSON.stringify(describeKeys(current, 1, MISS_TREE_KEYS));
+  return { at: walked.join('.'), tree };
+}
+
 function httpError(status) {
   return new Error(`HTTP ${status}: ${STATUS_CODES[status] ?? ''}`.trimEnd());
 }
@@ -120,7 +151,7 @@ function failed(verdict, status, gotState) {
  */
 export function createExtractEmbeddedStateHandler({ escalateFetch } = {}) {
   return async function extractEmbeddedStateHandler({
-    url, path, keys_only, find, raw = false, escalate, escalate_engine = 'auto', wait_for, user_agent, respect_robots
+    url, path, keys_only, find, raw = false, escalate, escalate_engine = 'auto', wait_for, user_agent, respect_robots, maxAge
   }) {
     // Each replaces `data` with something else, so asking for both has no
     // answer. Nothing was fetched, so - like a robots refusal - it costs nothing.
@@ -149,6 +180,7 @@ export function createExtractEmbeddedStateHandler({ escalateFetch } = {}) {
       const fetched = await fetchAndParse(url, {
         userAgent: user_agent,
         respectRobots: respect_robots,
+        maxAge,
         tool: 'extract_embedded_state',
         errorDocuments: true
       });
@@ -283,7 +315,26 @@ export function createExtractEmbeddedStateHandler({ escalateFetch } = {}) {
         }
       }
 
-      const data = effectivePath ? selectJsonPath(state.data, effectivePath) : state.data;
+      let data = state.data;
+      if (effectivePath) {
+        try {
+          data = selectJsonPath(state.data, effectivePath);
+        } catch (error) {
+          // A path miss: the key tree where it stopped, and a next step that
+          // stays on this tool (the fallback hint would say to drop `path`).
+          reportCost();
+          const miss = missKeyTree(state.data, effectivePath);
+          return {
+            content: [{
+              type: 'text',
+              text: `Failed to extract embedded state: ${error.message}` +
+                (miss ? `\nKeys at ${miss.at ? `"${miss.at}"` : 'the top level'}: ${miss.tree}` : '') +
+                '\nNext step: Call again with a path built from the keys above, or with find:"<key>" to get the paths where a field of that name lives.'
+            }],
+            isError: true
+          };
+        }
+      }
       const bytes = Buffer.byteLength(JSON.stringify(data) ?? '');
 
       if (!path && !keys_only && find === undefined && bytes > LARGE_RESULT_BYTES) {

package/src/tools/extract/extractStructured.js

@@ -9,6 +9,7 @@ import { ElicitationHelper } from '../../core/ElicitationHelper.js'; // D1.4
 import { load } from 'cheerio';
 import { LLMManager } from '../../core/llm/LLMManager.js';
 import { CRAWLFORGE_USER_AGENT } from '../../utils/fetchIdentity.js';
+import { MAX_AGE_PARAM } from '../../utils/maxAgeParam.js';
 import { fetchAndParse, flattenBodyText } from './_fetchAndParse.js';
 import { fetchErrorMessage } from '../../utils/fetchError.js';
 import { extractMainContent } from '../scrape/_mainContent.js';
@@ -267,6 +268,7 @@ const ExtractStructuredSchema = z.object({
   selectorHints: z.record(z.string()).optional(),
   respect_robots: z.boolean().optional(),
   user_agent: z.string().optional(),
+  ...MAX_AGE_PARAM,
   verify_numbers: z.boolean().optional().default(true)
 });
 
@@ -324,7 +326,7 @@ export class ExtractStructuredTool {
 
     try {
       const validated = ExtractStructuredSchema.parse(params);
-      const { url, schema: requestedSchema, prompt, llmConfig, fallbackToSelectors, selectorHints, respect_robots, user_agent, verify_numbers } = validated;
+      const { url, schema: requestedSchema, prompt, llmConfig, fallbackToSelectors, selectorHints, respect_robots, user_agent, maxAge, verify_numbers } = validated;
       // `type` is optional on input, and the tool's own example omits it. The
       // validator reads a typeless schema as "anything", so the CSS fallback
       // reported valid:true for "2 reviews" in an integer field (R24). A schema
@@ -387,6 +389,7 @@ export class ExtractStructuredTool {
       const { html, $, textContent, warnings } = await fetchAndParse(url, {
         userAgent: user_agent || this.userAgent,
         respectRobots: respect_robots,
+        maxAge,
         tool: 'extract_structured'
       });
 

package/src/tools/extract/extractWithLlm.js

@@ -440,6 +440,7 @@ export class ExtractWithLlm {
    * @param {number}  [params.maxTokens] - Max output tokens (default 4096)
    * @param {boolean} [params.respect_robots] - Per-request robots.txt override
    * @param {string}  [params.user_agent]     - Per-request identity override
+   * @param {number}  [params.maxAge]         - Page-cache age limit in ms, passed to fetchAndParse
    * @param {boolean} [params.verify_numbers] - Numeric provenance guard (default true)
    * @param {AbortSignal} [params.signal] - Aborts the model request(s); an aborted
    *   call skips the retry and the sampling fallback
@@ -456,6 +457,7 @@ export class ExtractWithLlm {
       maxTokens = 4096,
       respect_robots,
       user_agent,
+      maxAge,
       verify_numbers = true,
       signal
     } = params;
@@ -501,6 +503,7 @@ export class ExtractWithLlm {
         const { html, textContent, warnings } = await fetchAndParse(url, {
           respectRobots: respect_robots,
           userAgent: user_agent,
+          maxAge,
           tool: 'extract_with_llm'
         });
         text = textContent;

package/src/tools/extract/processDocument.js

@@ -187,15 +187,22 @@ const MAX_TITLE_CHARS = 200;
 /**
  * A title for a document with none in its metadata: the first non-empty line
  * of its text when that line is short enough to be one, else null. arXiv
- * stamps and page numbers before it are skipped.
+ * stamps, page numbers and lines of a prose paragraph before it are skipped:
+ * a line ending a sentence, or one the next line continues in lower case
+ * (the arXiv "Attention Is All You Need" PDF opens with a permission notice,
+ * live sweep 7.3).
  * @param {string} text
  * @returns {string|null}
  */
 export function titleFromText(text) {
-  for (const line of String(text || '').split(/\r?\n/)) {
-    const candidate = line.replace(/\s+/g, ' ').trim();
-    if (candidate === '' || NON_TITLE_LINE_RE.test(candidate)) continue;
-    return candidate.length <= MAX_TITLE_CHARS ? candidate : null;
+  const lines = String(text || '')
+    .split(/\r?\n/)
+    .map((line) => line.replace(/\s+/g, ' ').trim())
+    .filter((line) => line !== '' && !NON_TITLE_LINE_RE.test(line));
+  for (const [i, candidate] of lines.entries()) {
+    if (candidate.length > MAX_TITLE_CHARS) return null;
+    const isProse = /[.!?]$/.test(candidate) || /^\p{Ll}/u.test(lines[i + 1] || '');
+    if (!isProse) return candidate;
   }
   return null;
 }

package/src/tools/llmstxt/generateLLMsTxt.js

@@ -218,18 +218,21 @@ export class GenerateLLMsTxtTool {
       listed.add(key);
       return true;
     };
-    const emitSection = (heading, urls) => {
-      // Coerce to an array: sitemap/sections may arrive as a flat array, a
-      // grouped object ({path: [...]}), or a single value.
+    // Coerce to a URL array: sitemap/sections may arrive as a flat array, a
+    // grouped object ({path: [...]}), or a single value.
+    const toUrlList = (urls) => {
       let arr = [];
       if (Array.isArray(urls)) {
         arr = urls;
       } else if (urls && typeof urls === 'object') {
         arr = Object.values(urls).flat();
       }
-      const list = arr
+      return arr
         .map((u) => (typeof u === 'string' ? u : (u?.url || u?.loc)))
-        .filter(Boolean)
+        .filter(Boolean);
+    };
+    const emitSection = (heading, urls) => {
+      const list = toUrlList(urls)
         .filter(firstListing)
         .slice(0, 25);
       if (list.length === 0) return;
@@ -262,19 +265,22 @@ export class GenerateLLMsTxtTool {
     // listed only when no named section matched at all, which silently dropped
     // the homepage and whole sections (e.g. /specification) from sites that
     // happen to also have a /docs.
-    emitSection('Pages', flatten('other'));
+    // Only endpoints observed answering JSON/XML are APIs. Links to API
+    // documentation are HTML pages ("Python's C API" on docs.python.org was
+    // listed under APIs, live sweep 7.4), so they are listed as pages.
+    const allApis = (Array.isArray(analysis.apis) ? analysis.apis : []).filter((api) => api?.url);
+    const isMachineReadable = (api) => /json|xml/i.test(api.contentType || '');
+    const docLinks = allApis.filter((api) => !isMachineReadable(api)).map((api) => api.url);
 
     // Fallback: if no categorized section produced output, list the raw
-    // sitemap so llms.txt always carries a URL inventory. Must run BEFORE the
-    // APIs section — an APIs entry alone used to set hasBody and suppress it.
-    const hasBody = lines.some((l) => l.startsWith('## '));
-    if (!hasBody) {
-      emitSection('Pages', analysis.structure?.sitemap || []);
-    }
+    // sitemap so llms.txt always carries a URL inventory. Decided BEFORE the
+    // linked pages and APIs are added — those alone used to suppress it.
+    const pageUrls = flatten('other');
+    const hasBody = lines.some((l) => l.startsWith('## ')) || pageUrls.length > 0;
+    emitSection('Pages', [...(hasBody ? pageUrls : toUrlList(analysis.structure?.sitemap)), ...docLinks]);
 
     // APIs as their own section.
-    const apis = (Array.isArray(analysis.apis) ? analysis.apis : [])
-      .filter((api) => api?.url && firstListing(api.url));
+    const apis = allApis.filter((api) => isMachineReadable(api) && firstListing(api.url));
     if (apis.length > 0) {
       lines.push('## APIs');
       lines.push('');
@@ -310,6 +316,11 @@ export class GenerateLLMsTxtTool {
       counts.set(clean, (counts.get(clean) || 0) + 1);
     };
 
+    // Link text of API documentation links first: a title captured from the
+    // page itself replaces it below.
+    for (const api of Array.isArray(analysis.apis) ? analysis.apis : []) {
+      if (api?.type === 'documentation') add(api.url, api.title);
+    }
     for (const entries of Object.values(analysis.contentTypes || {})) {
       if (!Array.isArray(entries)) continue;
       for (const entry of entries) {

package/src/tools/scrape/unifiedScrape.js

@@ -33,6 +33,7 @@ import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
 import { stripHiddenFromDom } from '../../utils/hiddenContent.js';
 import { extractBlockText } from '../basic/extractText.js';
 import { pageTitle } from '../../utils/pageTitle.js';
+import { MAX_AGE_PARAM } from '../../utils/maxAgeParam.js';
 import { noteHostBlocked, clearHostBlocked, getHostBlock } from '../../utils/hostRateLimiter.js';
 
 // ── Schema ────────────────────────────────────────────────────────────────────
@@ -168,7 +169,8 @@ export const UnifiedScrapeSchema = z.object({
   // Compliance overrides, per request: identify as yourself for a target you
   // have your own agreement with, and take responsibility for ignoring robots.
   user_agent: z.string().optional(),
-  respect_robots: z.boolean().optional()
+  respect_robots: z.boolean().optional(),
+  ...MAX_AGE_PARAM
 });
 
 // ── Helpers ───────────────────────────────────────────────────────────────────
@@ -432,6 +434,7 @@ export class UnifiedScrapeTool {
           timeoutMs,
           userAgent: validated.user_agent,
           respectRobots: validated.respect_robots,
+          maxAge: validated.maxAge,
           tool: 'scrape',
           stripTags: [], // we handle boilerplate ourselves
           // A real wall reaches a plain fetch as a 403 with the challenge in

package/src/utils/canonicalUrl.js

@@ -0,0 +1,29 @@
+/**
+ * canonicalUrl — one spelling per page, for keys that must match when two
+ * calls ask for the same page (the invocation log's urlKey now, the page cache
+ * of REPEAT_CALL_REDUCTION_PLAN Phase 1 next).
+ *
+ * Rules (plan 1.2): lowercase scheme and host, drop the default port (80 for
+ * http, 443 for https) and the #fragment, keep path and query string exactly
+ * as written. An empty path is the root path ("/"), which is what is requested.
+ * Anything that does not parse as a URL returns null.
+ */
+
+export function canonicalUrl(input) {
+  if (typeof input !== 'string') return null;
+  let u;
+  try { u = new URL(input.trim()); } catch { return null; }
+
+  if (u.protocol !== 'http:' && u.protocol !== 'https:') {
+    u.hash = '';
+    return u.href;
+  }
+
+  // WHATWG URL already lowercases scheme and host and drops a default port,
+  // but it also rewrites the path (dot segments, percent-encoding), so the
+  // path and query are taken from the original string instead.
+  const rest = input.trim().match(/^[a-z][a-z0-9+.-]*:\/\/[^/?#]*([^#]*)/i)?.[1] ?? '';
+  const userinfo = u.username ? `${u.username}${u.password ? `:${u.password}` : ''}@` : '';
+  const pathAndQuery = rest === '' || rest.startsWith('?') ? `/${rest}` : rest;
+  return `${u.protocol}//${userinfo}${u.host}${pathAndQuery}`;
+}

package/src/utils/fetchLadder.js

@@ -122,7 +122,7 @@ async function plainFetch(url, options) {
 
 /**
  * @param {string} url
- * @param {{ tool: 'extract_text'|'extract_links', userAgent?: string, respectRobots?: boolean,
+ * @param {{ tool: 'extract_text'|'extract_links', userAgent?: string, respectRobots?: boolean, maxAge?: number,
  *   escalate?: boolean, escalateEngine?: string,
  *   escalateFetch?: (args: { url: string, engine: string, respectRobots?: boolean }) => Promise<object> }} options
  * @returns {Promise<
@@ -132,10 +132,10 @@ async function plainFetch(url, options) {
  *   `fields` carries `escalated` (only when escalate was asked for), `stealth`
  *   (when the stage ran) and `rendered: false` (a client-rendered shell)
  */
-export async function fetchLadder(url, { tool, userAgent, respectRobots, escalate = false, escalateEngine = 'auto', escalateFetch } = {}) {
+export async function fetchLadder(url, { tool, userAgent, respectRobots, maxAge, escalate = false, escalateEngine = 'auto', escalateFetch } = {}) {
   let escalationRan = false;
   try {
-    const response = await plainFetch(url, { timeout: LADDER_TIMEOUT_MS, userAgent, respectRobots, tool });
+    const response = await plainFetch(url, { timeout: LADDER_TIMEOUT_MS, userAgent, respectRobots, maxAge, tool });
     const type = mediaType(response);
     if (response.ok && type && !TEXTUAL_TYPE.test(type)) return { unsupported: type };
 

package/src/utils/maxAgeParam.js

@@ -0,0 +1,18 @@
+/**
+ * maxAgeParam — the one declaration of `maxAge` (repeat-call reduction plan,
+ * Phase 1, 1.5 / D2). Passed through as `options.maxAge` to the two page-cache
+ * seams, fetchWithTimeout and fetchAndParse; undefined means the server's
+ * default TTL there.
+ */
+
+import { z } from 'zod';
+import { pageCacheActive } from '../core/pageCache.js';
+
+export const MAX_AGE_PARAM = {
+  maxAge: z.number().int().min(0).optional().describe("Reuse this session's copy of the page if it was fetched within this many ms (default 600000 = 10 min); 0 always fetches fresh.")
+};
+
+// Phase 3 (3.1): the one sentence the nine cached tools' descriptions carry.
+// Empty where the cache is off (hosted HTTP, CRAWLFORGE_PAGE_CACHE_MB=0).
+export const PAGE_CACHE_TEXT = ' Asking again for the same page with different formats, paths or selectors reuses this session\'s copy for 10 minutes at 1 credit; pass maxAge: 0 to force a fresh fetch.';
+export const PAGE_CACHE_SENTENCE = pageCacheActive() ? PAGE_CACHE_TEXT : '';

package/src/utils/robotsGate.js

@@ -273,6 +273,29 @@ export function outboundHeaders(userAgent, signature = {}) {
   };
 }
 
+/**
+ * The gate's decision, refusing as {@link preflightFetch} does, without the
+ * per-host throttle. A page-cache hit (src/core/pageCache.js) sends no
+ * request, so there is nothing to space, but the blocklist and robots.txt are
+ * asked again: either may have changed since the copy was fetched.
+ *
+ * @param {string} url
+ * @param {object} [options] see {@link robotsPreflight}
+ * @param {{ redirectedFrom?: string }} [reached] set when `url` is where a
+ *   redirect from the requested URL led
+ * @returns {Promise<{ allowed: true, userAgent: string, crawlDelayMs: number,
+ *   warnings: string[], overridden: boolean }>}
+ * @throws {BlockedHostError|RobotsDisallowedError}
+ */
+export async function preflightDecision(url, options = {}, reached = {}) {
+  const decision = await robotsPreflight(url, options);
+  if (!decision.allowed) {
+    markPreflightRefusal('ROBOTS_DISALLOWED');
+    throw new RobotsDisallowedError(url, reached);
+  }
+  return decision;
+}
+
 /**
  * The call-site helper: run the gate, honour Crawl-delay and any recorded
  * `Retry-After`, and hand back the identity headers to send and the gate for
@@ -286,11 +309,7 @@ export function outboundHeaders(userAgent, signature = {}) {
  * @throws {BlockedHostError|RobotsDisallowedError}
  */
 export async function preflightFetch(url, options = {}) {
-  const decision = await robotsPreflight(url, options);
-  if (!decision.allowed) {
-    markPreflightRefusal('ROBOTS_DISALLOWED');
-    throw new RobotsDisallowedError(url);
-  }
+  const decision = await preflightDecision(url, options);
 
   await throttleHost(url, { crawlDelayMs: decision.crawlDelayMs });
 

package/src/utils/ssrfGuard.js

@@ -342,7 +342,8 @@ export async function safeFetch(url, options = {}) {
     return await fetchResigned(url, { ...options, ...guard });
   } catch (err) {
     if (isSsrfError(err)) {
-      throw new Error(err.cause?.message || err.message);
+      // Keep the code: callers tell a refusal from a fetch failure by it.
+      throw Object.assign(new Error(err.cause?.message || err.message), { code: 'SSRF_BLOCKED' });
     }
     throw withFetchCause(err, url);
   }