crawlforge-mcp-server 6.21.1 → 6.22.1

package/src/core/crawlers/BFSCrawler.js

@@ -44,6 +44,10 @@ export class BFSCrawler {
     this.sessionContext = sessionContext;
 
     this.visited = new Map(); // normalized key -> the URL as fetched
+    // Normalized keys of the pages fetched, by where each fetch landed after
+    // redirects. Kept apart from visited so the page a redirect lands on is
+    // not fetched again, without spending a second max_pages slot on it.
+    this.landed = new Set();
     this.results = [];
     this.errors = [];
     this.filterDecisions = []; // Track filtering decisions for analysis
@@ -153,7 +157,7 @@ export class BFSCrawler {
 
     // Check if already visited
     const normalizedUrl = normalizeUrl(url);
-    if (this.visited.has(normalizedUrl)) {
+    if (this.visited.has(normalizedUrl) || this.landed.has(normalizedUrl)) {
       return;
     }
     // The form that goes on the wire: a bare origin gains its root "/",
@@ -215,7 +219,7 @@ export class BFSCrawler {
     // top of this task ran before the awaited robots lookup, so concurrent
     // queue tasks may have filled the budget (or claimed this URL) since.
     // This block is synchronous, so the cap is exact.
-    if (this.visited.size >= this.maxPages || this.visited.has(normalizedUrl)) {
+    if (this.visited.size >= this.maxPages || this.visited.has(normalizedUrl) || this.landed.has(normalizedUrl)) {
       return;
     }
     this.visited.set(normalizedUrl, url);
@@ -263,17 +267,41 @@ export class BFSCrawler {
         if (cacheKey) await this.cache.set(cacheKey, pageData);
       }
 
+      // Links resolve against where the fetch landed, not the URL asked for:
+      // docs.python.org answers /3/tutorial with a redirect to /3/tutorial/,
+      // and appetite.html resolved against the first is /3/appetite.html, a
+      // 404 (R25). A seed that lands on another origin (http -> https, apex
+      // -> www) moves the crawl's origin with it, or every link it holds
+      // would read as external.
+      const pageUrl = pageData.finalUrl || url;
+      const landedKey = normalizeUrl(pageUrl);
+      if (isSeed && landedKey !== normalizedUrl) this.baseUrl = new URL(landedKey);
+      // Another link (one that redirects here, or this URL itself) reached the
+      // same page first: this copy is a duplicate.
+      if (this.landed.has(landedKey)) return;
+      this.landed.add(landedKey);
+
       // Process links for analysis
       if (this.enableLinkAnalysis && this.linkAnalyzer && pageData.links) {
         // Parse the page once and reuse for every link; re-parsing per link is
         // O(links × page size) and starves the event loop on link-dense pages.
         const $page = pageData.originalHtml ? load(pageData.originalHtml) : null;
         const pageBodyText = $page ? $page('body').text() : '';
+        // Index the anchors once by raw href, first one winning: a per-link
+        // `a[href="..."]` query scanned the whole page for every link (16,766
+        // scans of the 7.3 MB python changelog, R25) and blocked other calls.
+        const anchorsByHref = new Map();
+        if ($page) {
+          $page('a[href]').each((_, element) => {
+            const href = $page(element).attr('href');
+            if (!anchorsByHref.has(href)) anchorsByHref.set(href, element);
+          });
+        }
         for (const link of pageData.links) {
-          const absoluteUrl = this.resolveUrl(link, url);
+          const absoluteUrl = this.resolveUrl(link, pageUrl);
           if (absoluteUrl) {
             // Extract anchor text and context from link
-            const linkMetadata = this.extractLinkMetadata(link, $page, pageBodyText);
+            const linkMetadata = this.extractLinkMetadata(link, $page, pageBodyText, anchorsByHref);
             this.linkAnalyzer.addLink(normalizedUrl, normalizeUrl(absoluteUrl), linkMetadata);
           }
         }
@@ -301,8 +329,9 @@ export class BFSCrawler {
         for (const link of pageData.links) {
           if (this.visited.size >= this.maxPages) break;
           
-          const absoluteUrl = this.resolveUrl(link, url);
-          if (absoluteUrl && !this.visited.has(normalizeUrl(absoluteUrl))) {
+          const absoluteUrl = this.resolveUrl(link, pageUrl);
+          const key = absoluteUrl && normalizeUrl(absoluteUrl);
+          if (key && !this.visited.has(key) && !this.landed.has(key)) {
             // Not awaited: this task already holds a queue slot, so awaiting a
             // child would keep that slot pinned for the rest of the recursive
             // crawl (starving other tasks when concurrency <= depth, and making
@@ -395,7 +424,9 @@ export class BFSCrawler {
       // readBody decodes with the body's real charset — response.text() is
       // UTF-8-only and mangled ISO-8859-1 pages ("portugu�s" on lua.org).
       const html = await readBody(response);
-      return this.parsePage(html, url);
+      // response.url is where the redirects ended; links resolve against it.
+      const finalUrl = response.url || url;
+      return { ...this.parsePage(html, finalUrl), finalUrl };
     } catch (error) {
       clearTimeout(timeoutId);
       throw error;
@@ -656,18 +687,21 @@ export class BFSCrawler {
    * @param {string} href - The href attribute value
    * @param {Object} $ - Cheerio document for the page (parsed once per page)
    * @param {string} bodyText - Pre-computed body text of the page
+   * @param {Map} anchorsByHref - Raw href -> first <a> element with it
    * @returns {Object} Link metadata
    */
-  extractLinkMetadata(href, $, bodyText = '') {
+  extractLinkMetadata(href, $, bodyText = '', anchorsByHref) {
     if (!$) return {};
 
     try {
-      const linkElement = $(`a[href="${href}"]`).first();
+      const element = anchorsByHref.get(href);
 
-      if (linkElement.length === 0) {
+      if (!element) {
         return { href };
       }
 
+      const linkElement = $(element);
+
       const anchorText = linkElement.text().trim();
       const title = linkElement.attr('title');
       const rel = linkElement.attr('rel');
@@ -968,6 +1002,10 @@ export class BFSCrawler {
     const segments = pathname.split('/').filter(s => s);
     
     if (segments.length === 0) return '/';
+    // A file (bugs.html) is a page, not a directory: no trailing "/" or "/...".
+    if (segments.length <= 2 && /.\.[a-z0-9]+$/i.test(segments[segments.length - 1])) {
+      return `/${segments.join('/')}`;
+    }
     if (segments.length === 1) return `/${segments[0]}/`;
     
     // Return first two segments as pattern

package/src/core/processing/ContentProcessor.js

@@ -8,7 +8,7 @@ import { JSDOM } from 'jsdom';
 import * as cheerio from 'cheerio';
 import { z } from 'zod';
 import { ContentQualityAssessor } from '../../utils/contentUtils.js';
-import { recoverDroppedTables } from '../../tools/scrape/_mainContent.js';
+import { recoverDroppedTables, unwrapMediaWikiHeadings, tooLargeForReadability } from '../../tools/scrape/_mainContent.js';
 import { pageTitle } from '../../utils/pageTitle.js';
 import { flattenText } from 'crawlforge-extractors';
 
@@ -73,7 +73,9 @@ export class ContentProcessor {
       const document = dom.window.document;
 
       // Extract main content using Mozilla Readability
-      if (processingOptions.removeBoilerplate) {
+      // A page too large for Readability keeps no `readability` block (R25).
+      if (processingOptions.removeBoilerplate && !tooLargeForReadability(html)) {
+        unwrapMediaWikiHeadings(document);
         const reader = new Readability(document, {
           debug: false,
           maxElemsToDivide: 300,

package/src/core/queue/QueueManager.js

@@ -1,4 +1,4 @@
-import PQueue from 'p-queue';
+import PQueue, { TimeoutError } from 'p-queue';
 
 export class QueueManager {
   constructor(options = {}) {
@@ -46,7 +46,21 @@ export class QueueManager {
 
   async add(fn, options = {}) {
     const { priority = 0 } = options;
-    return this.queue.add(fn, { priority });
+    try {
+      return await this.queue.add(fn, { priority });
+    } catch (error) {
+      // p-queue's message gives the queue state when the task STARTED, which
+      // reads like a queue problem. The task itself ran too long, often
+      // because synchronous work elsewhere held the event loop.
+      if (error instanceof TimeoutError) {
+        throw new Error(
+          `Task ran over its ${this.queue.timeout / 1000} s time budget; ` +
+          'it may have been starved by other work running in this process.',
+          { cause: error }
+        );
+      }
+      throw error;
+    }
   }
 
   async addAll(tasks, options = {}) {

package/src/resources/ResourceRegistry.js

@@ -34,6 +34,19 @@ const RESOURCE_MIME = {
   screenshot: 'image/png',
 };
 
+/**
+ * A screenshot's MIME type, read from its magic bytes: callers ask for png or
+ * jpeg (and could get webp), and the bytes cannot disagree with themselves.
+ * Anything unrecognised keeps the old image/png label.
+ * @param {Buffer} buf
+ * @returns {string}
+ */
+export function sniffImageMime(buf) {
+  if (buf.length >= 3 && buf[0] === 0xff && buf[1] === 0xd8 && buf[2] === 0xff) return 'image/jpeg';
+  if (buf.length >= 12 && buf.toString('ascii', 0, 4) === 'RIFF' && buf.toString('ascii', 8, 12) === 'WEBP') return 'image/webp';
+  return RESOURCE_MIME.screenshot;
+}
+
 /**
  * Parse a crawlforge:// URI into its components.
  * @param {string} uri
@@ -67,8 +80,8 @@ export class ResourceRegistry {
     // In-memory stores for lightweight resource tracking
     /** @type {Map<string, { data: any, createdAt: number, ttl: number }>} */
     this._crawlSitemaps = new Map(); // sessionId -> sitemap
-    /** @type {Map<string, { data: Buffer, createdAt: number, ttl: number }>} */
-    this._screenshots = new Map(); // actionId -> PNG buffer
+    /** @type {Map<string, { data: Buffer, mimeType: string, createdAt: number, ttl: number }>} */
+    this._screenshots = new Map(); // actionId -> image buffer
 
     // Default TTL: 1 hour
     this.defaultTtl = 60 * 60 * 1000;
@@ -90,7 +103,7 @@ export class ResourceRegistry {
   /**
    * Store a screenshot for later retrieval.
    * @param {string} actionId
-   * @param {Buffer|string} screenshotData - PNG buffer or base64 string
+   * @param {Buffer|string} screenshotData - image buffer or base64 string
    */
   storeScreenshot(actionId, screenshotData) {
     const buf = Buffer.isBuffer(screenshotData)
@@ -98,6 +111,7 @@ export class ResourceRegistry {
       : Buffer.from(screenshotData, 'base64');
     this._screenshots.set(actionId, {
       data: buf,
+      mimeType: sniffImageMime(buf),
       createdAt: Date.now(),
       ttl: this.defaultTtl,
     });
@@ -176,7 +190,7 @@ export class ResourceRegistry {
           uri: `crawlforge://screenshot/${actionId}`,
           name: `Screenshot ${actionId}`,
           description: 'Screenshot from a CrawlForge browser tool',
-          mimeType: RESOURCE_MIME.screenshot,
+          mimeType: entry.mimeType,
         });
       }
     }
@@ -302,7 +316,7 @@ export class ResourceRegistry {
     return {
       contents: [{
         uri,
-        mimeType: RESOURCE_MIME.screenshot,
+        mimeType: entry.mimeType,
         blob: entry.data.toString('base64'),
       }],
     };

package/src/schemas/toolOutputSchemas.js

@@ -17,16 +17,16 @@
 import { z } from 'zod';
 import { SCRAPE_STRING_FORMATS, SCRAPE_OBJECT_FORMATS } from '../tools/scrape/formats.js';
 
-// `_cost` is injected by withAuth into the legacy JSON-text copy of the
-// result (never into structuredContent directly) — included here anyway so
-// clients that copy `content[0].text` into structuredContent-shaped code
-// don't fail validation.
+// `_cost` and `next_step` are injected by withAuth into both the JSON-text copy
+// of the result and structuredContent (R25 0.2), so every shape declares them.
 const costShape = z.object({
   projected: z.number().optional().describe('Credits projected for this call before execution'),
-  actual: z.number().optional().describe('Credits actually charged (0 in creator mode, half-rate on error)'),
+  actual: z.number().optional().describe('Credits actually charged (0 in creator mode, and 0 for a failed call)'),
   remaining_credits: z.number().nullable().optional().describe('Credits remaining on the account after this call, if known'),
   projection_note: z.string().optional().describe('Human-readable note about how the cost was projected')
-}).passthrough().optional().describe('Cost-transparency metadata (D3.5), present when injected into the text copy of the result');
+}).passthrough().optional().describe('Cost-transparency metadata (D3.5): what this call was projected to cost and what it was charged');
+
+const nextStepShape = z.string().optional().describe('Failed calls only: the tool or fix to try next');
 
 // ── result handles (Phase 2) ────────────────────────────────────────────────
 
@@ -143,7 +143,8 @@ const scrapeShape = {
   warnings: z.array(z.string()).optional().describe('Per-format warnings; partial success never fails the whole call'),
   redaction: redactionShape,
   ...resultHandleShape,
-  _cost: costShape
+  _cost: costShape,
+  next_step: nextStepShape
 };
 
 // ── map_site ────────────────────────────────────────────────────────────────
@@ -186,7 +187,8 @@ const mapSiteShape = {
   }).passthrough()).optional().describe('Present only when the `search` param was set'),
   warnings: z.array(z.string()).optional().describe('Present when the map is partial, e.g. the start page could not be read and the URLs come from the sitemap only, or when the result is over max_inline_chars'),
   ...resultHandleShape,
-  _cost: costShape
+  _cost: costShape,
+  next_step: nextStepShape
 };
 
 // ── serp_rank ───────────────────────────────────────────────────────────────
@@ -220,7 +222,8 @@ const serpRankShape = {
   checkUrl: z.string().optional().describe('Link to view the real SERP on DataForSEO'),
   cost: z.number().optional().describe('USD charged by DataForSEO for this lookup (separate from CrawlForge credits)'),
   checkedAt: z.string().optional(),
-  _cost: costShape
+  _cost: costShape,
+  next_step: nextStepShape
 };
 
 // ── reddit_search ───────────────────────────────────────────────────────────
@@ -273,7 +276,8 @@ const redditSearchShape = {
   fallback_used: z.string().optional().describe('Present when the primary archive failed and the fallback served the result'),
   notes: z.array(z.string()).optional().describe('Data-provenance caveats (archive freshness, coverage gaps)'),
   checkedAt: z.string().optional(),
-  _cost: costShape
+  _cost: costShape,
+  next_step: nextStepShape
 };
 
 // ── search_web ──────────────────────────────────────────────────────────────
@@ -334,7 +338,8 @@ const searchWebShape = {
   redaction: redactionShape,
   warnings: z.array(z.string()).optional().describe('Notes on this result; over max_inline_chars, where the full result is kept and how to read it'),
   ...resultHandleShape,
-  _cost: costShape
+  _cost: costShape,
+  next_step: nextStepShape
 };
 
 // ── extract_structured ───────────────────────────────────────────────────────
@@ -366,7 +371,8 @@ const extractStructuredShape = {
     }).passthrough()).optional(),
     skipped: z.string().optional().describe('"empty_source" when there was nothing to check against')
   }).passthrough().optional(),
-  _cost: costShape
+  _cost: costShape,
+  next_step: nextStepShape
 };
 
 // ── crawl_deep ────────────────────────────────────────────────────────────────
@@ -417,7 +423,8 @@ const crawlDeepShape = {
   cached: z.boolean().optional().describe('True when this response was replayed from an earlier crawl rather than crawled now; crawled_at gives its age'),
   redaction: redactionShape,
   ...resultHandleShape,
-  _cost: costShape
+  _cost: costShape,
+  next_step: nextStepShape
 };
 
 // ── read_result ───────────────────────────────────────────────────────────────
@@ -455,7 +462,8 @@ const readResultShape = {
   preview: z.string().optional(),
   truncated: z.boolean().optional().describe('search: more matches than returned; json_path: value replaced by a preview'),
   warnings: z.array(z.string()).optional(),
-  _cost: costShape
+  _cost: costShape,
+  next_step: nextStepShape
 };
 
 // ── Export ────────────────────────────────────────────────────────────────────

package/src/server/fallbackHints.js

@@ -92,10 +92,22 @@ export const CLASS_HINTS = Object.freeze({
     'The URL points at a private or internal address (localhost, a private network, cloud metadata), which every CrawlForge tool refuses. Do not retry it with this or any other tool; use a public URL.',
   not_found: () =>
     'The URL does not exist (HTTP 404, or the host does not resolve). Check it for typos, or find the right one with map_site on the site or search_web. Retrying it, or switching to stealth_mode, will not help.',
+  model_not_pulled: () =>
+    'The named Ollama model is not installed: pull it (the error gives the command), or call again without `model` so an installed one is picked; list_ollama_models lists them.',
+  refused: () =>
+    'The host refused the connection: nothing is listening on that port, or the server is down. Check the URL and port; retrying right away, or switching to stealth_mode or another tool, will not help.',
   timeout: () =>
     'The request timed out. Retry once with a longer timeout or less work (fewer pages, URLs or actions); if it times out again, the site is too slow from here - do not keep retrying.',
   validation: (tool, params) =>
-    `The input was rejected: fix ${params.length ? params.join(', ') : 'the parameter the error names'} as the error describes and call ${tool} again. Do not switch tools for this.`
+    `The input was rejected: fix ${params.length ? params.join(', ') : 'the parameter the error names'} as the error describes and call ${tool} again. Do not switch tools for this.`,
+  // R25 3.2/3.3: these were told the site was slow (a click) or to escalate (extract_text).
+  selector: (tool) =>
+    'Nothing on the page matched the selector (absent, hidden, or not actionable in time) - a selector problem, not a slow site, so do not retry with a longer timeout. ' +
+    (tool === 'browser_session'
+      ? 'Take another snapshot and act on its refs: navigation invalidates them.'
+      : 'Check the selectors against scrape formats:["html"] output, or use extract_structured when the markup varies.'),
+  unknown_context: () =>
+    'That contextId is not open in this server process (never created, already cleaned up, or lost on a restart). Run stealth_mode operation:"create_context" first and pass the contextId it returns.'
 });
 
 /** The strings that describe what went wrong: the error fields, or the whole text. */
@@ -133,14 +145,28 @@ export function classifyError(parsed, text) {
   if (codes.includes('ROBOTS_DISALLOWED') || refusal === 'ROBOTS_DISALLOWED' || /robots\.txt on \S+ disallows/.test(msg)) return { cls: 'robots' };
   if (codes.includes('HOST_BLOCKED') || refusal === 'HOST_BLOCKED' || /permanent blocklist/.test(msg)) return { cls: 'host_blocked' };
   if (status === 404 || /\bHTTP 404\b|\b404 Not Found\b|\bENOTFOUND\b/i.test(msg)) return { cls: 'not_found' };
+  // "Could not reach <url> (connect ECONNREFUSED 1.2.3.4:9)": after ssrf, so a refused private address stays ssrf.
+  if (/\bECONNREFUSED\b/.test(msg)) return { cls: 'refused' };
+  // "url is required for scrape operation" (stealth_mode, localization): the name is the parameter.
+  const missing = msg.match(/\b([a-z][A-Za-z_]*) is required for (?:\w+ )?operation\b/);
   // Before timeout: a schema message may mention a `timeout` parameter.
-  if (/ZodError|"code":\s*"(?:invalid_[a-z_]+|too_small|too_big|unrecognized_keys|custom)"|\bInvalid (?:input|option|enum value|arguments?)\b|\bvalidation (?:failed|error)\b|\bparameter is required\b/i.test(msg)) {
-    return { cls: 'validation', params: zodPaths(msg) };
+  if (missing || /ZodError|"code":\s*"(?:invalid_[a-z_]+|too_small|too_big|unrecognized_keys|custom)"|\bInvalid (?:input|option|enum value|arguments?)\b|\bvalidation (?:failed|error)\b|\bparameter is required\b/i.test(msg)) {
+    return { cls: 'validation', params: missing ? [missing[1]] : zodPaths(msg) };
   }
-  // A wait for a selector that never appeared is a selector problem, which the
-  // tool's own hint covers; a longer timeout would not help.
-  if (/\btime[ds]? ?out\b|\bTimeout \d+ ?ms exceeded|\bETIMEDOUT\b|aborted due to timeout/i.test(msg) &&
-      !/waiting for (?:locator|selector)|waitFor/i.test(msg)) return { cls: 'timeout' };
+  if (/\bContext not found\b/.test(msg)) return { cls: 'unknown_context' };
+  // extract_with_llm: 'Ollama model "x" is not pulled. Run: "ollama pull x"' - Ollama is up, the model is missing.
+  if (/\bOllama model "[^"]*" is not pulled\b/.test(msg)) return { cls: 'model_not_pulled' };
+  const timedOut = /\btime[ds]? ?out\b|\bTimeout \d+ ?ms exceeded|\bETIMEDOUT\b|aborted due to timeout/i.test(msg);
+  // A selector that matched nothing is a selector problem, not a slow site.
+  // Playwright's "Call log: waiting for locator(...)" is stripped before this
+  // runs (errorText.js), but the API prefix of an element wait stays:
+  // "locator.click: Timeout 4000ms exceeded."
+  if (/\bNo elements? (?:found|matched) (?:for )?selector\b/i.test(msg) ||
+      (timedOut && /\b(?:locator|elementHandle)\.\w+:|\bwaitForSelector\b|waiting for (?:locator|selector)/i.test(msg))) {
+    return { cls: 'selector' };
+  }
+  // Any other wait (page.waitForFunction for a text wait) is the tool hint's.
+  if (timedOut && !/waiting for (?:locator|selector)|waitFor/i.test(msg)) return { cls: 'timeout' };
   return null;
 }
 
@@ -174,6 +200,10 @@ export function appendFallbackHint(toolName, result) {
   if (parsed) {
     parsed.next_step = hint;
     first.text = JSON.stringify(parsed, null, 2);
+    // Clients that show structuredContent never see the text copy (R25 0.2).
+    if (result.structuredContent && typeof result.structuredContent === 'object') {
+      result.structuredContent.next_step = hint;
+    }
     return result;
   }
   first.text = `${first.text}\nNext step: ${hint}`;

package/src/server/withAuth.js

@@ -13,6 +13,8 @@
  *     handler did no work, so it is billed NOTHING and reports no usage. The
  *     SDK re-enters the handler with the reply and the terminal entry bills
  *     once, so a confirmation costs exactly what the call always cost (G4).
+ *   - a failed call (an isError result, a JSON body with success:false, or a
+ *     throw) is charged NOTHING and reports no usage: failures are free
  *   - error results get a "Next step:" hint naming the tool to try next
  *     (src/server/fallbackHints.js) so a failure is not followed by a blind retry
  *   - emits an OTel span via src/observability/tracing.js (no-op if disabled)
@@ -22,9 +24,9 @@
 import { createHash } from 'node:crypto';
 import { isInputRequiredResult } from '@modelcontextprotocol/server';
 import { recordToolInvocation } from '../observability/tracing.js';
-import { isInternalRequest, preflightRefusal, reportedActualCost, requestContext } from './requestContext.js';
+import { isInternalRequest, reportedActualCost, requestContext } from './requestContext.js';
 import { appendFallbackHint } from './fallbackHints.js';
-import { normalizeErrorResult } from './errorText.js';
+import { failure, normalizeErrorResult } from './errorText.js';
 import { INLINE_THRESHOLD_TOOLS, applyInlineThreshold } from './inlineThreshold.js';
 import { REDACTION_TOOLS, redactionSurcharge, runRedactionStage } from './redaction.js';
 import { getResultStore } from '../core/ResultStore.js';
@@ -96,10 +98,6 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
       let redactionModelRan = false;
       let outcome = 'pending';
       let thrown = null;
-      // Only bill the error-path half-charge once the handler has actually run.
-      // A throw from the credit check itself (backend down, key rejected, etc.)
-      // means the tool never executed and must cost nothing.
-      let handlerStarted = false;
       // A handler may report a lower spend (setActualCost). The projection is
       // the ceiling the caller saw before the call, so a report can only ever
       // lower the charge, never raise it.
@@ -139,7 +137,6 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
           }
         }
 
-        handlerStarted = true;
         const result = await handler(params, ctx);
 
         // Phase 4.4: a multi-round-trip handler answers `input_required` when it
@@ -157,18 +154,14 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
         }
 
         // Tools catch their own failures and return { isError:true } rather than
-        // throwing (the shared pattern in server.js). That is still an ERROR
-        // outcome: bill it the half-credit error rate — not full — and log/metric
-        // it as an error, honoring CLAUDE.md's "half credits on error" contract.
-        const isErrorResult = result?.isError === true;
+        // throwing (the shared pattern in server.js), and several return a JSON
+        // body with success:false and no flag at all (R25 0.1). Both are an
+        // ERROR outcome: charged nothing, logged and metered as an error. The
+        // flag is set on the second kind too, so the client sees an error and
+        // the SDK skips outputSchema validation of a failure body.
+        const isErrorResult = result?.isError === true || failure(result) !== null;
+        if (isErrorResult) result.isError = true;
         outcome = isErrorResult ? 'error' : 'success';
-        // A pre-flight refusal (robots.txt disallowed the path, or the host is
-        // blocklisted) means we fetched nothing at all, so it costs nothing —
-        // not even the half-credit error rate, which exists for work that ran
-        // and then failed. Only when the refusal actually sank the call: a
-        // multi-URL tool that skipped one disallowed URL and still returned a
-        // result did real work and bills for it.
-        const refused = isErrorResult && preflightRefusal() !== null;
 
         // Phase 5 (5.3): redact_pii scrubs the page text this call returns.
         // It runs BEFORE the inline-threshold stage below, so what gets
@@ -191,21 +184,20 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
           try { shapeLargeResult(toolName, result, params); } catch { /* shaping must never break the request path */ }
         }
 
-        const base = billable();
-        const charge = creditCost === 0 || refused
-          ? 0
-          : (isErrorResult ? Math.max(1, Math.floor(base * 0.5)) : base);
+        // Failures are free, as the website promises. A pre-flight refusal that
+        // sank the call is one of them; a multi-URL tool that skipped one
+        // disallowed URL and still returned a result is a success and bills.
+        const charge = creditCost === 0 || isErrorResult ? 0 : billable();
 
         // D3.5: Surface cost transparency in all tool responses. For internal
         // proxy requests the meaningful balance is the end user's, which only
         // the website knows — report null rather than the static key's cache.
+        // Creator mode has no balance: null, since outputSchema rejects Infinity.
         try {
           const projection = authManager.projectCost(toolName, params);
-          const remainingCredits = creatorMode
-            ? Infinity
-            : internal
-              ? null
-              : (authManager.creditCache ? [...authManager.creditCache.values()][0] ?? null : null);
+          const remainingCredits = creatorMode || internal
+            ? null
+            : (authManager.creditCache ? [...authManager.creditCache.values()][0] ?? null : null);
           const costMeta = {
             projected: creditCost,
             actual: creatorMode ? 0 : charge,
@@ -223,15 +215,17 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
               // Not JSON — skip injection silently
             }
           }
+          // ...and into structuredContent, which is all some clients show (R25 0.2).
+          if (result?.structuredContent && typeof result.structuredContent === 'object') {
+            result.structuredContent._cost = costMeta;
+          }
         } catch {
           // Cost injection must never break the request path
         }
 
         // Selection hint: tell the model what to try instead of the same call,
         // after stripping ANSI codes, call logs and doubled prefixes from the
-        // error text (R24 4.5). Both act on isError results and on a JSON body
-        // with success:false, which several tools return without the flag
-        // (R24 4.1). Neither touches the charge, settled above.
+        // error text (R24 4.5). Neither touches the charge, settled above.
         try {
           normalizeErrorResult(result);
           appendFallbackHint(toolName, result);
@@ -242,26 +236,15 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
         // the backend has nothing to (re-)price; reporting 0 would still create
         // a serp_rank record the backend could recompute to full cost. The
         // same goes for a handler that reported a 0 spend.
-        if (!creatorMode && creditCost > 0 && !refused && charge > 0) {
-          await authManager.reportUsage(toolName, charge, params, isErrorResult ? 500 : 200, Date.now() - startTime);
+        if (!creatorMode && charge > 0) {
+          await authManager.reportUsage(toolName, charge, params, 200, Date.now() - startTime);
         }
 
         return result;
       } catch (error) {
+        // A throw is a failure, and failures are free: nothing is reported.
         outcome = 'error';
         thrown = error;
-        // Half-charge on error — but never charge a free (0-cost) call, never
-        // let Math.max(1, …) floor a 0 up to 1 credit, and never bill at all
-        // if the handler never ran (e.g. the credit check itself threw).
-        if (!creatorMode && creditCost > 0 && handlerStarted && preflightRefusal() === null) {
-          await authManager.reportUsage(
-            toolName,
-            Math.max(1, Math.floor(billable() * 0.5)),
-            params,
-            500,
-            Date.now() - startTime
-          );
-        }
         throw error;
       } finally {
         const durationMs = Date.now() - startTime;

package/src/skills/agent-skills/crawlforge-batch-automation/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-batch-automation
 description: "Automates large scraping jobs and browser interactions with CrawlForge's batch_scrape, get_batch_results, scrape_with_actions, and generate_llms_txt tools. Use when the user wants to scrape many URLs at once, batch-scrape a list of pages, collect dozens of product, news, or competitor pages, run browser actions (click, type, scroll, wait) before scraping, log in or fill a form before extracting, or generate an llms.txt for a site. Use sync mode for up to about 25 URLs and async with a webhook for large batches; retrieve paginated output with get_batch_results."
 metadata:
-  version: 6.21.1
+  version: 6.22.1
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-browser-sessions/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-browser-sessions
 description: "Keeps one browser page alive across several tool calls with CrawlForge's browser_session tool. Use when the user needs to log in and then read pages behind that login, work through a multi-step flow (search, filter, paginate, fill a wizard), or look at a page before deciding what to click. The loop is: open a session on a URL, snapshot it to list the interactive elements with stable refs (@e1, @e2), act on those refs, read the content, close. Unlike scrape_with_actions, which is one-shot and closes its browser when the call returns, a session survives between calls, so one login is paid for once and a wrong selector costs one call instead of a whole chain."
 metadata:
-  version: 6.21.1
+  version: 6.22.1
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-change-tracking/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-change-tracking
 description: "Monitors web pages for changes over time with CrawlForge's track_changes tool. Use when the user wants to track changes to a page, watch a URL, monitor competitor pricing, detect when content updates, get notified of regulation or product-availability changes, or diff a page against a saved baseline. Workflow: create a baseline with operation create_baseline, then periodically compare with operation compare to get a change percentage and a diff; supports CSS-selector scoping, webhooks, and scheduled monitoring."
 metadata:
-  version: 6.21.1
+  version: 6.22.1
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-deep-research/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-deep-research
 description: "Runs multi-source web research and autonomous question-answering with CrawlForge's deep_research, agent, search_web, and reddit_search tools. Use when the user wants to research a topic, do a deep dive, compare competitors, gather facts with citations, answer a question from the web, search the web, or get a synthesized report from many sources, or search Reddit posts, comments, and threads. deep_research expands queries, fetches and dedupes sources, then synthesizes; agent autonomously plans searches and navigation from a plain-English prompt with no URLs needed; search_web returns ranked results; reddit_search searches Reddit via community archives (reddit.com blocks scrapers). Caps costs via max_urls and confirms before expensive runs."
 metadata:
-  version: 6.21.1
+  version: 6.22.1
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-getting-started/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-getting-started
 description: "Orientation and tool-selection guide for the CrawlForge MCP server's 31 web tools. Use when the user is getting started with CrawlForge, asks which CrawlForge tool to use, how to set up the API key, how skills or the CLI work, what a tool costs in credits, or when one tool fails and a fallback is needed. Routes requests to the right specialized skill (web scraping, deep research, stealth, structured extraction, change tracking, batch automation, browser sessions), and explains MCP-tools-vs-CLI, the Ollama-first LLM fallback chain, and per-tool credit costs."
 metadata:
-  version: 6.21.1
+  version: 6.22.1
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-getting-started/references/credits.md

@@ -85,4 +85,4 @@ metered; there is no free tier. Tools marked "scales" cost more as work grows.
   only 2 when the plain fetch works; `scrape` then `stealth_mode` costs 7 either
   way. Do not set it on a site that reads fine — it raises the projection your
   balance is checked against.
-- Errors are charged at half the tool cost; creator mode is unlimited.
+- Failed calls are not charged; creator mode is unlimited.