crawlforge-mcp-server 6.4.0 → 6.6.0

package/src/core/llm/LLMManager.js

@@ -27,6 +27,70 @@ function endsMidJson(text) {
   return !trimmed.endsWith('}') && !trimmed.endsWith(']');
 }
 
+/**
+ * Recover the complete rows of a JSON response the model stopped writing.
+ *
+ * Walks the text tracking strings and the bracket stack, remembers the end of
+ * every complete element written directly inside an array, cuts there and
+ * closes what is still open. Returns null when no element completed, so a
+ * caller can still fall back. A schema asking for a table of rows routinely
+ * overruns a small model's output budget — the ECB key-rates table came back
+ * cut off at 1,800 and again at 3,600 tokens and the whole extraction failed,
+ * although dozens of rows were complete (R21, 2026-09-09).
+ *
+ * @param {string} text
+ * @returns {{ data: unknown, rows: number }|null}
+ */
+export function salvageTruncatedJson(text) {
+  const src = text.trim();
+  if (!src.startsWith('{') && !src.startsWith('[')) return null;
+  const stack = [];
+  let inString = false;
+  let cut = -1;
+  for (let i = 0; i < src.length; i++) {
+    const ch = src[i];
+    if (inString) {
+      if (ch === '\\') i++;
+      else if (ch === '"') {
+        inString = false;
+        // A string written directly into an array is a complete element.
+        if (stack[stack.length - 1] === '[') cut = i + 1;
+      }
+      continue;
+    }
+    if (ch === '"') inString = true;
+    else if (ch === '{' || ch === '[') stack.push(ch);
+    else if (ch === '}' || ch === ']') {
+      stack.pop();
+      if (stack[stack.length - 1] === '[') cut = i + 1;
+    }
+  }
+  if (cut < 0) return null;
+  // Close everything still open at the cut, innermost first.
+  const open = [];
+  inString = false;
+  for (let i = 0; i < cut; i++) {
+    const ch = src[i];
+    if (inString) { if (ch === '\\') i++; else if (ch === '"') inString = false; continue; }
+    if (ch === '"') inString = true;
+    else if (ch === '{' || ch === '[') open.push(ch);
+    else if (ch === '}' || ch === ']') open.pop();
+  }
+  const closers = open.reverse().map((c) => (c === '{' ? '}' : ']')).join('');
+  let data;
+  try {
+    data = JSON.parse(src.slice(0, cut) + closers);
+  } catch {
+    return null;
+  }
+  let rows = 0;
+  (function count(v) {
+    if (Array.isArray(v)) { rows += v.length; v.forEach(count); }
+    else if (v && typeof v === 'object') Object.values(v).forEach(count);
+  })(data);
+  return rows > 0 ? { data, rows } : null;
+}
+
 /**
  * LLM Manager
  * Manages multiple LLM providers and provides unified interface
@@ -987,12 +1051,25 @@ Extract the data and return valid JSON:`;
         try {
           parsed = JSON.parse(cleaned);
         } catch (parseError) {
+          if (!endsMidJson(cleaned)) throw parseError;
           // Name the cause when the JSON simply stops: "unexpected end of
           // input at position 2608" tells a caller nothing they can act on,
           // whereas "cut off at the 1000-token limit" points at the schema.
-          throw endsMidJson(cleaned)
-            ? new Error(`model response was cut off at the ${budget}-token output limit (${cleaned.length} chars) — the schema asks for more rows than fit`)
-            : parseError;
+          const cutOff = `model response was cut off at the ${budget}-token output limit (${cleaned.length} chars) — the schema asks for more rows than fit`;
+          // The doubled budget was the retry; when that is cut off too, the
+          // rows it did complete are worth more than a fallback that has none.
+          const salvaged = attempt > 0 ? salvageTruncatedJson(cleaned) : null;
+          if (!salvaged) throw new Error(cutOff);
+          const validation = this.validateAgainstSchema(salvaged.data, schema);
+          this.logger.warn('LLM structured extraction salvaged a cut-off response', { budget, rows: salvaged.rows });
+          return {
+            data: salvaged.data,
+            method: 'llm',
+            valid: validation.valid,
+            validationErrors: validation.errors,
+            partial: true,
+            warning: `${cutOff}; kept the ${salvaged.rows} complete row(s) it had written. Ask for fewer rows (a prompt naming the rows you need, or a narrower schema) to get the rest.`
+          };
         }
 
         const validation = this.validateAgainstSchema(parsed, schema);
@@ -1016,7 +1093,7 @@ Extract the data and return valid JSON:`;
     // Report which path produced the data. Callers previously labelled this
     // result "llm", so a failed LLM call was returned as a high-confidence
     // LLM extraction.
-    return { ...this.fallbackStructuredExtraction(content, schema), error: lastError.message };
+    return { ...this.fallbackStructuredExtraction(content, schema, lastError.message), error: lastError.message };
   }
 
   /**
@@ -1036,7 +1113,7 @@ Extract the data and return valid JSON:`;
   /**
    * Fallback structured extraction without LLM — keyword/regex matching for primitives
    */
-  fallbackStructuredExtraction(content, schema) {
+  fallbackStructuredExtraction(content, schema, reason = 'no LLM provider available') {
     const extracted = {};
     const properties = schema.properties || {};
 
@@ -1065,7 +1142,10 @@ Extract the data and return valid JSON:`;
       data: extracted,
       method: 'keyword_fallback',
       valid: false,
-      validationErrors: ['Used fallback extraction — no LLM provider available']
+      // Name the real reason: this fallback also runs after an LLM attempt
+      // that failed, and "no LLM provider available" beside a working Ollama
+      // sent a caller looking at the wrong thing (R21, 2026-09-09).
+      validationErrors: [`Used fallback extraction — ${reason}`]
     };
   }
 

package/src/core/processing/PDFProcessor.js

@@ -12,7 +12,9 @@ import { config } from '../../constants/config.js';
 import { identityHeaders } from '../../utils/fetchIdentity.js';
 
 const PDFProcessorSchema = z.object({
-  source: z.string().min(1),
+  // A Buffer is the 'buffer' source: process_document hands over a body it
+  // has already fetched and sniffed as a PDF (R21, 2026-09-09).
+  source: z.union([z.string().min(1), z.instanceof(Buffer)]),
   sourceType: z.enum(['url', 'file', 'buffer']).default('url'),
   options: z.object({
     extractMetadata: z.boolean().default(true),

package/src/server/fallbackHints.js

@@ -32,6 +32,7 @@ export const FALLBACK_HINTS = Object.freeze({
   extract_with_llm: 'If Ollama is unreachable pass provider:"openai" or "anthropic" with a key, or use extract_structured (CSS fallback needs no LLM).',
   list_ollama_models: 'Ollama is not reachable - use extract_with_llm with provider:"openai"/"anthropic", or extract_structured.',
   scrape_with_actions: 'Check the selector against scrape formats:["html"] output; for a one-shot render of a blocked page use stealth_mode operation:"scrape".',
+  browser_session: 'A session that expired or was closed cannot be reused - start again with operation:"open". If a ref missed, take another snapshot first: navigation invalidates refs. For a chain that needs no session, use scrape_with_actions.',
   deep_research: 'Use agent for a shorter answer, or search_web followed by scrape on the sources that matter.',
   scrape: 'After a 403/429/CAPTCHA/challenge page or an empty shell use stealth_mode operation:"scrape"; if the content needs a click or login use scrape_with_actions.',
   agent: 'Use deep_research for exhaustive sourcing, or search_web followed by scrape on the sources that matter.',
@@ -80,6 +81,9 @@ export function appendFallbackHint(toolName, result) {
     // plain text
   }
 
+  // An error that already names its next step — a reddit.com refusal points
+  // at reddit_search — keeps it: the tool's generic hint would contradict it.
+  if (parsed?.next_step || /\bNext step:/.test(first.text)) return result;
   const hint = hintFor(toolName, parsed);
   if (!hint || first.text.includes(hint)) return result;
   if (parsed) {

package/src/server/inlineThreshold.js

@@ -72,6 +72,27 @@ export function resultTextView(resultObject, textPaths = []) {
   return { view: 'json', view_path: null, text: JSON.stringify(resultObject, null, 2) };
 }
 
+/**
+ * The non-text fields of `content` small enough to stay inline: everything
+ * except the text views (markdown, text, html, …) up to a quarter of the
+ * inline budget each. Null when there is nothing to keep.
+ */
+export function keepSmallContentFields(content, textPaths = [], maxInline = DEFAULT_MAX_INLINE_CHARS) {
+  if (!content || typeof content !== 'object' || Array.isArray(content)) return null;
+  const textLeaves = new Set(
+    textPaths.filter((p) => p.startsWith('content.')).map((p) => p.slice('content.'.length))
+  );
+  const cap = Math.max(1000, Math.floor(maxInline / 4));
+  const kept = {};
+  for (const [key, value] of Object.entries(content)) {
+    if (textLeaves.has(key) || value === undefined) continue;
+    if (typeof value === 'string' && value.length > cap) continue;
+    if (typeof value === 'object' && value !== null && JSON.stringify(value).length > cap) continue;
+    kept[key] = value;
+  }
+  return Object.keys(kept).length > 0 ? kept : null;
+}
+
 function warningsOf(resultObject) {
   return Array.isArray(resultObject.warnings) ? resultObject.warnings.filter((w) => typeof w === 'string') : [];
 }
@@ -129,7 +150,9 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
   }
 
   const preview = text.slice(0, maxInline);
-  const hint = `Result is ${json.length} chars as JSON, over the inline limit of ${maxInline}; preview holds the first ${preview.length} chars of ${viewDesc} (${text.length} chars in total) and the full result is kept for 1 hour under result_handle ${handle}: ${readWith}.`;
+  const keptFields = keepSmallContentFields(resultObject.content, config.textPaths, maxInline);
+  const keptDesc = keptFields ? `; content.${Object.keys(keptFields).join(', content.')} kept inline` : '';
+  const hint = `Result is ${json.length} chars as JSON, over the inline limit of ${maxInline}; preview holds the first ${preview.length} chars of ${viewDesc} (${text.length} chars in total)${keptDesc} and the full result is kept for 1 hour under result_handle ${handle}: ${readWith}.`;
 
   const shaped = {};
   for (const [key, value] of Object.entries(resultObject)) {
@@ -143,6 +166,13 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
   if (resultObject.redaction && typeof resultObject.redaction === 'object') {
     shaped.redaction = resultObject.redaction;
   }
+  // The query-scoped formats live beside the page text under `content`
+  // (highlights, answer, json, metadata, links). They are the small, exact
+  // answer the caller paid for, and truncating the markdown must not drop
+  // them: an nhs.uk scrape with highlights and a question came back as a
+  // markdown preview and nothing else (R21, 2026-09-09). Keep every
+  // non-text `content` field that fits a quarter of the inline budget.
+  if (keptFields) shaped.content = keptFields;
   Object.assign(shaped, {
     preview,
     result_handle: handle,

package/src/server/requestContext.js

@@ -6,14 +6,18 @@
  * plumbing. AsyncLocalStorage bridges that gap: the transport runs each
  * request inside a context, and withAuth reads it at invocation time.
  *
- * Today the only flag is `internal`: a request authenticated with the
+ * The first flag is `internal`: a request authenticated with the
  * INTERNAL_PROXY_SECRET (the crawlforge-website REST proxy). Internal requests
  * run tools normally but are billing-exempt — the website has already checked
  * and charged the end user's credits, so metering here would double-bill.
  *
- * The flag lives on the request context, never on the MCP session: a session
- * id created by an internal request grants nothing to a later request that
- * authenticates by other means.
+ * The second is `ownerToken`, which says WHICH of the website's customers an
+ * internal request is being made for (see internalOwnerToken below).
+ *
+ * Both live on the request context, never on the MCP session: a session id
+ * created by an internal request grants nothing to a later request that
+ * authenticates by other means, and an owner established on one request is not
+ * inherited by the next one down the same MCP session.
  */
 
 import { AsyncLocalStorage } from 'node:async_hooks';
@@ -25,6 +29,23 @@ export function isInternalRequest() {
   return requestContext.getStore()?.internal === true;
 }
 
+/**
+ * The end user this internal-proxy request is being made for, or null.
+ *
+ * An opaque per-user token the website derives with an HMAC keyed on the shared
+ * internal secret (mcpOwnerToken in crawlforge-website
+ * src/lib/tools/mcp-proxy.ts), carried on the X-CrawlForge-Owner header. It is
+ * not reversible to a user id here and is not meant to be: all a stateful tool
+ * needs is a value that is stable for one customer and distinct between them.
+ *
+ * Only ever set on a request that already proved the internal secret, and only
+ * after the transport has validated its shape — authenticateRequest in
+ * transports/streamableHttp.js is the single place that decides both.
+ */
+export function internalOwnerToken() {
+  return requestContext.getStore()?.ownerToken ?? null;
+}
+
 /**
  * Record that the compliance gate refused this invocation before anything was
  * fetched — robots.txt disallowed the path, or the host is on the permanent
@@ -36,7 +57,7 @@ export function isInternalRequest() {
  * reaches withAuth. It also survives both routes a refusal can take — thrown,
  * or swallowed into an isError result.
  *
- * @param {string} code 'ROBOTS_DISALLOWED' | 'HOST_BLOCKED'
+ * @param {string} code 'ROBOTS_DISALLOWED' | 'HOST_BLOCKED' | 'USE_REDDIT_SEARCH'
  */
 export function markPreflightRefusal(code) {
   const store = requestContext.getStore();

package/src/server/toolFilter.js

@@ -1,7 +1,7 @@
 /**
  * toolFilter — client-side tool selection (Phase 6).
  *
- * Lets an MCP client load a subset of the 30 registered tools via env vars,
+ * Lets an MCP client load a subset of the 31 registered tools via env vars,
  * cutting context bloat (mirrors Bright Data / Exa's TOOLS / GROUPS pattern).
  *
  * Pure module: no I/O, no logging; process.env is only read via
@@ -17,7 +17,7 @@ export const TOOL_GROUPS = {
   search: ['search_web', 'serp_rank', 'reddit_search'],
   crawl: ['crawl_deep', 'map_site'],
   extract: ['extract_content', 'process_document', 'summarize_content', 'analyze_content', 'extract_structured', 'extract_with_llm', 'list_ollama_models', 'extract_embedded_state'],
-  batch: ['batch_scrape', 'get_batch_results', 'scrape_with_actions'],
+  batch: ['batch_scrape', 'get_batch_results', 'scrape_with_actions', 'browser_session'],
   research: ['deep_research'],
   tracking: ['track_changes'],
   llmstxt: ['generate_llms_txt'],

package/src/server/transports/streamableHttp.js

@@ -314,9 +314,11 @@ export async function connectStreamableHttp(server, authManager, logger, options
       // — never expose an unauthenticated MCP endpoint on a public interface.
       // `internal` marks a request from the website's REST proxy
       // (INTERNAL_PROXY_SECRET): it is billing-exempt in withAuth because the
-      // website already charged the end user. Request-scoped only — never
-      // persisted on the session.
+      // website already charged the end user. `ownerToken` says which of the
+      // website's customers it is being made for. Both are request-scoped only
+      // — never persisted on the session.
       let internal = false;
+      let ownerToken;
       if (!(authManager.isCreatorMode() && hostIsLoopback)) {
         const authResult = await authenticateRequest(req, authManager, oauthProvider);
         if (!authResult.ok) {
@@ -332,6 +334,7 @@ export async function connectStreamableHttp(server, authManager, logger, options
           return;
         }
         internal = authResult.internal === true;
+        ownerToken = authResult.ownerToken;
       }
 
       // Era routing. Only a POST can carry the 2026-07-28 per-request envelope;
@@ -355,7 +358,7 @@ export async function connectStreamableHttp(server, authManager, logger, options
         if (parsedBody !== undefined) {
           const probe = await toWebRequest(req, parsedBody);
           if (!(await isLegacyRequest(probe, parsedBody))) {
-            await requestContext.run({ internal }, () => serveModern(req, res, parsedBody));
+            await requestContext.run({ internal, ownerToken }, () => serveModern(req, res, parsedBody));
             return;
           }
         }
@@ -370,7 +373,7 @@ export async function connectStreamableHttp(server, authManager, logger, options
 
       if (existing) {
         await requestContext.run(
-          { internal, servingServer: existing.server, servingEra: 'legacy' },
+          { internal, ownerToken, servingServer: existing.server, servingEra: 'legacy' },
           () => existing.transport.handleRequest(req, res, parsedBody)
         );
         return;
@@ -406,7 +409,7 @@ export async function connectStreamableHttp(server, authManager, logger, options
       try {
         await sessionServer.connect(transport);
         await requestContext.run(
-          { internal, servingServer: sessionServer, servingEra: 'legacy' },
+          { internal, ownerToken, servingServer: sessionServer, servingEra: 'legacy' },
           () => transport.handleRequest(req, res, parsedBody)
         );
       } catch (err) {
@@ -453,6 +456,28 @@ export async function connectStreamableHttp(server, authManager, logger, options
   };
 }
 
+/**
+ * Hex, and bounded. The website emits 32 characters (mcpOwnerToken in
+ * crawlforge-website src/lib/tools/mcp-proxy.ts); the range is wider so the two
+ * repos can pick a different HMAC slice without a lockstep deploy, and narrow
+ * enough that nothing unbounded, non-printable or structured can ever become
+ * part of an owner id.
+ */
+const OWNER_TOKEN_RE = /^[0-9a-f]{16,64}$/;
+
+/**
+ * The owner token an internal request claims, or undefined.
+ *
+ * Undefined covers absent AND malformed alike, and the difference must not
+ * matter: everything downstream treats "no owner" as "no session", so a value
+ * that fails this check is simply not an owner rather than a strange one. Never
+ * relax this into a coercion — the token becomes part of a tenant key.
+ */
+function readOwnerToken(req) {
+  const value = (req.headers['x-crawlforge-owner'] || '').toString();
+  return OWNER_TOKEN_RE.test(value) ? value : undefined;
+}
+
 /**
  * Validate a request's credentials.
  *
@@ -460,13 +485,14 @@ export async function connectStreamableHttp(server, authManager, logger, options
  *   - `X-Internal-Secret: <INTERNAL_PROXY_SECRET>` — server-to-server requests
  *     from the crawlforge-website REST proxy. Returns { ok, internal: true };
  *     internal requests are billing-exempt in withAuth (the website already
- *     charged the end user). Only active when the env var is set.
+ *     charged the end user). Only active when the env var is set. Such a
+ *     request may also carry `X-CrawlForge-Owner` — see readOwnerToken.
  *   - `Authorization: Bearer <crawlforge-api-key>` (legacy static key)
  *   - `X-API-Key: <crawlforge-api-key>` (legacy static key)
  *   - `Authorization: Bearer <oauth-access-token>` if OAuth is enabled —
  *     the OAuth provider validates the token and maps it to the API key.
  *
- * @returns {Promise<{ok: true, internal?: boolean} | {ok: false, status: number, error: string, message: string, reason: string}>}
+ * @returns {Promise<{ok: true, internal?: boolean, ownerToken?: string} | {ok: false, status: number, error: string, message: string, reason: string}>}
  */
 async function authenticateRequest(req, authManager, oauthProvider) {
   // Internal proxy path first: presenting the header at all means the caller
@@ -480,7 +506,11 @@ async function authenticateRequest(req, authManager, oauthProvider) {
       const provided = createHash('sha256').update(providedSecret).digest();
       const expected = createHash('sha256').update(internalSecret).digest();
       if (timingSafeEqual(provided, expected)) {
-        return { ok: true, internal: true };
+        // Read ONLY here, on the branch that has just proved the secret. A
+        // request that authenticated any other way — or none — never has its
+        // owner header looked at, so claiming an owner requires already being
+        // the proxy.
+        return { ok: true, internal: true, ownerToken: readOwnerToken(req) };
       }
     }
     return {

package/src/skills/agent-skills/crawlforge-batch-automation/SKILL.md

@@ -97,13 +97,20 @@ browser actions before extraction.
 }
 ```
 
-Allowed action types: `wait`, `click`, `type`, `press`, `scroll`, `screenshot`,
-`executeJavaScript`. `executeJavaScript` is disabled unless the deploy sets
+Allowed action types: `snapshot`, `wait`, `click`, `type`, `press`, `scroll`,
+`screenshot`, `executeJavaScript`, `select`, `hover`, `navigate`. Start a chain
+with `{"type": "snapshot"}` to list the page's interactive elements with stable
+refs (`@e1`, `@e2` …) and target those in later actions instead of guessing CSS
+selectors. `executeJavaScript` is disabled unless the deploy sets
 `ALLOW_JAVASCRIPT_EXECUTION=true`. 1–20 actions per call. Screenshots are stored
 as `crawlforge://screenshot/{actionId}` resources. Full action schemas:
 [actions](references/actions.md). CLI:
 `crawlforge actions https://example.com --script login.json --screenshot`.
 
+One-shot: the browser closes when the call returns. When the flow spans more
+than one call, or you need to see the page before choosing what to click, use
+`browser_session` instead (crawlforge-browser-sessions).
+
 ## generate_llms_txt — AI policy file (cost: 5)
 
 ```json

package/src/skills/agent-skills/crawlforge-batch-automation/references/actions.md

@@ -1,10 +1,16 @@
 # scrape_with_actions — Action Types
 
 `scrape_with_actions` runs an ordered `actions[]` array (1–20 items) before
-scraping. Only these 7 action types are allowed (allow-listed in ActionExecutor).
-Each action object has a `type` plus type-specific fields. Common optional
-fields on every action: `timeout`, `description`, `continueOnError`, `retries`
-(0–5), `captureAfter`.
+scraping. Only these action types are allowed (allow-listed in ActionExecutor):
+`snapshot`, `wait`, `click`, `type`, `press`, `scroll`, `screenshot`,
+`executeJavaScript`, `select`, `hover`, `navigate`. Each action object has a
+`type` plus type-specific fields. Common optional fields on every action:
+`timeout`, `description`, `continueOnError`, `retries` (0–5), `captureAfter`.
+
+**Start with a snapshot.** Section 8 returns the page's interactive elements
+with stable refs, and any action's `selector` may name one (`@e1`) instead of a
+CSS selector you have not seen. That is the difference between landing a
+multi-step flow first try and burning the call on a guess.
 
 ## 1. wait
 
@@ -101,6 +107,46 @@ Disabled unless the deployment sets `ALLOW_JAVASCRIPT_EXECUTION=true`.
 { "type": "executeJavaScript", "script": "return document.title", "returnResult": true }
 ```
 
+## 8. snapshot
+
+List the page's interactive elements, each with a stable ref later actions can
+target. Refs are assigned `@e1…@eN` in document order and are **invalidated by
+navigation** — snapshot again after one, or acting on an old ref fails with a
+named error telling you to.
+
+| Field | Type | Notes |
+|-------|------|-------|
+| `interactiveOnly` | boolean | Default true. False also lists headings and landmarks, which carry no ref. |
+| `maxNodes` | number | Cap on nodes listed (default 200, max 1000). The result sets `truncated` when the cap stopped the walk. |
+
+```json
+{ "type": "snapshot" }
+```
+
+The tree comes back in `actionResults[i].result.tree`, alongside `snapshotId`,
+`url`, `title`, `refCount`, `nodeCount`, `truncated` and `interactiveOnly`:
+
+```
+[document] "Sign in"
+  @e1 [textbox] "Email"
+  @e2 [textbox] "Password"
+  @e3 [button] "Sign in"
+```
+
+```json
+[
+  { "type": "snapshot" },
+  { "type": "type", "selector": "@e1", "text": "user@example.com" },
+  { "type": "click", "selector": "@e3" }
+]
+```
+
+The tool is stateless, so a chain cannot adapt to its own snapshot mid-flight:
+read the refs in one call, act on them in the next. A fresh load of the same
+page numbers them the same way, and the second chain snapshots again first so
+the refs are stamped on the document it is acting against. The walk covers the
+main frame only — elements inside iframes and shadow DOM get no refs.
+
 ## Top-level options
 
 | Option | Default | Notes |