crawlforge-mcp-server 6.21.0 → 6.22.0

package/src/resources/ResourceRegistry.js

@@ -34,6 +34,19 @@ const RESOURCE_MIME = {
   screenshot: 'image/png',
 };
 
+/**
+ * A screenshot's MIME type, read from its magic bytes: callers ask for png or
+ * jpeg (and could get webp), and the bytes cannot disagree with themselves.
+ * Anything unrecognised keeps the old image/png label.
+ * @param {Buffer} buf
+ * @returns {string}
+ */
+export function sniffImageMime(buf) {
+  if (buf.length >= 3 && buf[0] === 0xff && buf[1] === 0xd8 && buf[2] === 0xff) return 'image/jpeg';
+  if (buf.length >= 12 && buf.toString('ascii', 0, 4) === 'RIFF' && buf.toString('ascii', 8, 12) === 'WEBP') return 'image/webp';
+  return RESOURCE_MIME.screenshot;
+}
+
 /**
  * Parse a crawlforge:// URI into its components.
  * @param {string} uri
@@ -67,8 +80,8 @@ export class ResourceRegistry {
     // In-memory stores for lightweight resource tracking
     /** @type {Map<string, { data: any, createdAt: number, ttl: number }>} */
     this._crawlSitemaps = new Map(); // sessionId -> sitemap
-    /** @type {Map<string, { data: Buffer, createdAt: number, ttl: number }>} */
-    this._screenshots = new Map(); // actionId -> PNG buffer
+    /** @type {Map<string, { data: Buffer, mimeType: string, createdAt: number, ttl: number }>} */
+    this._screenshots = new Map(); // actionId -> image buffer
 
     // Default TTL: 1 hour
     this.defaultTtl = 60 * 60 * 1000;
@@ -90,7 +103,7 @@ export class ResourceRegistry {
   /**
    * Store a screenshot for later retrieval.
    * @param {string} actionId
-   * @param {Buffer|string} screenshotData - PNG buffer or base64 string
+   * @param {Buffer|string} screenshotData - image buffer or base64 string
    */
   storeScreenshot(actionId, screenshotData) {
     const buf = Buffer.isBuffer(screenshotData)
@@ -98,6 +111,7 @@ export class ResourceRegistry {
       : Buffer.from(screenshotData, 'base64');
     this._screenshots.set(actionId, {
       data: buf,
+      mimeType: sniffImageMime(buf),
       createdAt: Date.now(),
       ttl: this.defaultTtl,
     });
@@ -176,7 +190,7 @@ export class ResourceRegistry {
           uri: `crawlforge://screenshot/${actionId}`,
           name: `Screenshot ${actionId}`,
           description: 'Screenshot from a CrawlForge browser tool',
-          mimeType: RESOURCE_MIME.screenshot,
+          mimeType: entry.mimeType,
         });
       }
     }
@@ -302,7 +316,7 @@ export class ResourceRegistry {
     return {
       contents: [{
         uri,
-        mimeType: RESOURCE_MIME.screenshot,
+        mimeType: entry.mimeType,
         blob: entry.data.toString('base64'),
       }],
     };

package/src/schemas/toolOutputSchemas.js

@@ -17,16 +17,16 @@
 import { z } from 'zod';
 import { SCRAPE_STRING_FORMATS, SCRAPE_OBJECT_FORMATS } from '../tools/scrape/formats.js';
 
-// `_cost` is injected by withAuth into the legacy JSON-text copy of the
-// result (never into structuredContent directly) — included here anyway so
-// clients that copy `content[0].text` into structuredContent-shaped code
-// don't fail validation.
+// `_cost` and `next_step` are injected by withAuth into both the JSON-text copy
+// of the result and structuredContent (R25 0.2), so every shape declares them.
 const costShape = z.object({
   projected: z.number().optional().describe('Credits projected for this call before execution'),
-  actual: z.number().optional().describe('Credits actually charged (0 in creator mode, half-rate on error)'),
+  actual: z.number().optional().describe('Credits actually charged (0 in creator mode, and 0 for a failed call)'),
   remaining_credits: z.number().nullable().optional().describe('Credits remaining on the account after this call, if known'),
   projection_note: z.string().optional().describe('Human-readable note about how the cost was projected')
-}).passthrough().optional().describe('Cost-transparency metadata (D3.5), present when injected into the text copy of the result');
+}).passthrough().optional().describe('Cost-transparency metadata (D3.5): what this call was projected to cost and what it was charged');
+
+const nextStepShape = z.string().optional().describe('Failed calls only: the tool or fix to try next');
 
 // ── result handles (Phase 2) ────────────────────────────────────────────────
 
@@ -143,7 +143,8 @@ const scrapeShape = {
   warnings: z.array(z.string()).optional().describe('Per-format warnings; partial success never fails the whole call'),
   redaction: redactionShape,
   ...resultHandleShape,
-  _cost: costShape
+  _cost: costShape,
+  next_step: nextStepShape
 };
 
 // ── map_site ────────────────────────────────────────────────────────────────
@@ -186,7 +187,8 @@ const mapSiteShape = {
   }).passthrough()).optional().describe('Present only when the `search` param was set'),
   warnings: z.array(z.string()).optional().describe('Present when the map is partial, e.g. the start page could not be read and the URLs come from the sitemap only, or when the result is over max_inline_chars'),
   ...resultHandleShape,
-  _cost: costShape
+  _cost: costShape,
+  next_step: nextStepShape
 };
 
 // ── serp_rank ───────────────────────────────────────────────────────────────
@@ -220,7 +222,8 @@ const serpRankShape = {
   checkUrl: z.string().optional().describe('Link to view the real SERP on DataForSEO'),
   cost: z.number().optional().describe('USD charged by DataForSEO for this lookup (separate from CrawlForge credits)'),
   checkedAt: z.string().optional(),
-  _cost: costShape
+  _cost: costShape,
+  next_step: nextStepShape
 };
 
 // ── reddit_search ───────────────────────────────────────────────────────────
@@ -273,7 +276,8 @@ const redditSearchShape = {
   fallback_used: z.string().optional().describe('Present when the primary archive failed and the fallback served the result'),
   notes: z.array(z.string()).optional().describe('Data-provenance caveats (archive freshness, coverage gaps)'),
   checkedAt: z.string().optional(),
-  _cost: costShape
+  _cost: costShape,
+  next_step: nextStepShape
 };
 
 // ── search_web ──────────────────────────────────────────────────────────────
@@ -306,7 +310,7 @@ const searchWebShape = {
   effective_query: z.string().optional().describe('Present when query expansion changed the query actually used'),
   expanded_queries: z.array(z.string()).optional().describe('Present only when the original query returned nothing: the queries searched, in order - the original, then its expanded form'),
   results: z.array(searchWebResultShape).optional(),
-  total_results: z.union([z.string(), z.number()]).optional(),
+  total_results: z.union([z.string(), z.number()]).nullable().optional().describe('The backend\'s total match count; null when it reports none (Brave Search API)'),
   search_time: z.number().optional(),
   offset: z.number().optional(),
   limit: z.number().optional(),
@@ -334,7 +338,8 @@ const searchWebShape = {
   redaction: redactionShape,
   warnings: z.array(z.string()).optional().describe('Notes on this result; over max_inline_chars, where the full result is kept and how to read it'),
   ...resultHandleShape,
-  _cost: costShape
+  _cost: costShape,
+  next_step: nextStepShape
 };
 
 // ── extract_structured ───────────────────────────────────────────────────────
@@ -366,7 +371,8 @@ const extractStructuredShape = {
     }).passthrough()).optional(),
     skipped: z.string().optional().describe('"empty_source" when there was nothing to check against')
   }).passthrough().optional(),
-  _cost: costShape
+  _cost: costShape,
+  next_step: nextStepShape
 };
 
 // ── crawl_deep ────────────────────────────────────────────────────────────────
@@ -417,7 +423,8 @@ const crawlDeepShape = {
   cached: z.boolean().optional().describe('True when this response was replayed from an earlier crawl rather than crawled now; crawled_at gives its age'),
   redaction: redactionShape,
   ...resultHandleShape,
-  _cost: costShape
+  _cost: costShape,
+  next_step: nextStepShape
 };
 
 // ── read_result ───────────────────────────────────────────────────────────────
@@ -455,7 +462,8 @@ const readResultShape = {
   preview: z.string().optional(),
   truncated: z.boolean().optional().describe('search: more matches than returned; json_path: value replaced by a preview'),
   warnings: z.array(z.string()).optional(),
-  _cost: costShape
+  _cost: costShape,
+  next_step: nextStepShape
 };
 
 // ── Export ────────────────────────────────────────────────────────────────────

package/src/server/fallbackHints.js

@@ -92,10 +92,22 @@ export const CLASS_HINTS = Object.freeze({
     'The URL points at a private or internal address (localhost, a private network, cloud metadata), which every CrawlForge tool refuses. Do not retry it with this or any other tool; use a public URL.',
   not_found: () =>
     'The URL does not exist (HTTP 404, or the host does not resolve). Check it for typos, or find the right one with map_site on the site or search_web. Retrying it, or switching to stealth_mode, will not help.',
+  model_not_pulled: () =>
+    'The named Ollama model is not installed: pull it (the error gives the command), or call again without `model` so an installed one is picked; list_ollama_models lists them.',
+  refused: () =>
+    'The host refused the connection: nothing is listening on that port, or the server is down. Check the URL and port; retrying right away, or switching to stealth_mode or another tool, will not help.',
   timeout: () =>
     'The request timed out. Retry once with a longer timeout or less work (fewer pages, URLs or actions); if it times out again, the site is too slow from here - do not keep retrying.',
   validation: (tool, params) =>
-    `The input was rejected: fix ${params.length ? params.join(', ') : 'the parameter the error names'} as the error describes and call ${tool} again. Do not switch tools for this.`
+    `The input was rejected: fix ${params.length ? params.join(', ') : 'the parameter the error names'} as the error describes and call ${tool} again. Do not switch tools for this.`,
+  // R25 3.2/3.3: these were told the site was slow (a click) or to escalate (extract_text).
+  selector: (tool) =>
+    'Nothing on the page matched the selector (absent, hidden, or not actionable in time) - a selector problem, not a slow site, so do not retry with a longer timeout. ' +
+    (tool === 'browser_session'
+      ? 'Take another snapshot and act on its refs: navigation invalidates them.'
+      : 'Check the selectors against scrape formats:["html"] output, or use extract_structured when the markup varies.'),
+  unknown_context: () =>
+    'That contextId is not open in this server process (never created, already cleaned up, or lost on a restart). Run stealth_mode operation:"create_context" first and pass the contextId it returns.'
 });
 
 /** The strings that describe what went wrong: the error fields, or the whole text. */
@@ -133,14 +145,28 @@ export function classifyError(parsed, text) {
   if (codes.includes('ROBOTS_DISALLOWED') || refusal === 'ROBOTS_DISALLOWED' || /robots\.txt on \S+ disallows/.test(msg)) return { cls: 'robots' };
   if (codes.includes('HOST_BLOCKED') || refusal === 'HOST_BLOCKED' || /permanent blocklist/.test(msg)) return { cls: 'host_blocked' };
   if (status === 404 || /\bHTTP 404\b|\b404 Not Found\b|\bENOTFOUND\b/i.test(msg)) return { cls: 'not_found' };
+  // "Could not reach <url> (connect ECONNREFUSED 1.2.3.4:9)": after ssrf, so a refused private address stays ssrf.
+  if (/\bECONNREFUSED\b/.test(msg)) return { cls: 'refused' };
+  // "url is required for scrape operation" (stealth_mode, localization): the name is the parameter.
+  const missing = msg.match(/\b([a-z][A-Za-z_]*) is required for (?:\w+ )?operation\b/);
   // Before timeout: a schema message may mention a `timeout` parameter.
-  if (/ZodError|"code":\s*"(?:invalid_[a-z_]+|too_small|too_big|unrecognized_keys|custom)"|\bInvalid (?:input|option|enum value|arguments?)\b|\bvalidation (?:failed|error)\b|\bparameter is required\b/i.test(msg)) {
-    return { cls: 'validation', params: zodPaths(msg) };
+  if (missing || /ZodError|"code":\s*"(?:invalid_[a-z_]+|too_small|too_big|unrecognized_keys|custom)"|\bInvalid (?:input|option|enum value|arguments?)\b|\bvalidation (?:failed|error)\b|\bparameter is required\b/i.test(msg)) {
+    return { cls: 'validation', params: missing ? [missing[1]] : zodPaths(msg) };
   }
-  // A wait for a selector that never appeared is a selector problem, which the
-  // tool's own hint covers; a longer timeout would not help.
-  if (/\btime[ds]? ?out\b|\bTimeout \d+ ?ms exceeded|\bETIMEDOUT\b|aborted due to timeout/i.test(msg) &&
-      !/waiting for (?:locator|selector)|waitFor/i.test(msg)) return { cls: 'timeout' };
+  if (/\bContext not found\b/.test(msg)) return { cls: 'unknown_context' };
+  // extract_with_llm: 'Ollama model "x" is not pulled. Run: "ollama pull x"' - Ollama is up, the model is missing.
+  if (/\bOllama model "[^"]*" is not pulled\b/.test(msg)) return { cls: 'model_not_pulled' };
+  const timedOut = /\btime[ds]? ?out\b|\bTimeout \d+ ?ms exceeded|\bETIMEDOUT\b|aborted due to timeout/i.test(msg);
+  // A selector that matched nothing is a selector problem, not a slow site.
+  // Playwright's "Call log: waiting for locator(...)" is stripped before this
+  // runs (errorText.js), but the API prefix of an element wait stays:
+  // "locator.click: Timeout 4000ms exceeded."
+  if (/\bNo elements? (?:found|matched) (?:for )?selector\b/i.test(msg) ||
+      (timedOut && /\b(?:locator|elementHandle)\.\w+:|\bwaitForSelector\b|waiting for (?:locator|selector)/i.test(msg))) {
+    return { cls: 'selector' };
+  }
+  // Any other wait (page.waitForFunction for a text wait) is the tool hint's.
+  if (timedOut && !/waiting for (?:locator|selector)|waitFor/i.test(msg)) return { cls: 'timeout' };
   return null;
 }
 
@@ -174,6 +200,10 @@ export function appendFallbackHint(toolName, result) {
   if (parsed) {
     parsed.next_step = hint;
     first.text = JSON.stringify(parsed, null, 2);
+    // Clients that show structuredContent never see the text copy (R25 0.2).
+    if (result.structuredContent && typeof result.structuredContent === 'object') {
+      result.structuredContent.next_step = hint;
+    }
     return result;
   }
   first.text = `${first.text}\nNext step: ${hint}`;

package/src/server/withAuth.js

@@ -13,6 +13,8 @@
  *     handler did no work, so it is billed NOTHING and reports no usage. The
  *     SDK re-enters the handler with the reply and the terminal entry bills
  *     once, so a confirmation costs exactly what the call always cost (G4).
+ *   - a failed call (an isError result, a JSON body with success:false, or a
+ *     throw) is charged NOTHING and reports no usage: failures are free
  *   - error results get a "Next step:" hint naming the tool to try next
  *     (src/server/fallbackHints.js) so a failure is not followed by a blind retry
  *   - emits an OTel span via src/observability/tracing.js (no-op if disabled)
@@ -22,9 +24,9 @@
 import { createHash } from 'node:crypto';
 import { isInputRequiredResult } from '@modelcontextprotocol/server';
 import { recordToolInvocation } from '../observability/tracing.js';
-import { isInternalRequest, preflightRefusal, reportedActualCost, requestContext } from './requestContext.js';
+import { isInternalRequest, reportedActualCost, requestContext } from './requestContext.js';
 import { appendFallbackHint } from './fallbackHints.js';
-import { normalizeErrorResult } from './errorText.js';
+import { failure, normalizeErrorResult } from './errorText.js';
 import { INLINE_THRESHOLD_TOOLS, applyInlineThreshold } from './inlineThreshold.js';
 import { REDACTION_TOOLS, redactionSurcharge, runRedactionStage } from './redaction.js';
 import { getResultStore } from '../core/ResultStore.js';
@@ -96,10 +98,6 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
       let redactionModelRan = false;
       let outcome = 'pending';
       let thrown = null;
-      // Only bill the error-path half-charge once the handler has actually run.
-      // A throw from the credit check itself (backend down, key rejected, etc.)
-      // means the tool never executed and must cost nothing.
-      let handlerStarted = false;
       // A handler may report a lower spend (setActualCost). The projection is
       // the ceiling the caller saw before the call, so a report can only ever
       // lower the charge, never raise it.
@@ -139,7 +137,6 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
           }
         }
 
-        handlerStarted = true;
         const result = await handler(params, ctx);
 
         // Phase 4.4: a multi-round-trip handler answers `input_required` when it
@@ -157,18 +154,14 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
         }
 
         // Tools catch their own failures and return { isError:true } rather than
-        // throwing (the shared pattern in server.js). That is still an ERROR
-        // outcome: bill it the half-credit error rate — not full — and log/metric
-        // it as an error, honoring CLAUDE.md's "half credits on error" contract.
-        const isErrorResult = result?.isError === true;
+        // throwing (the shared pattern in server.js), and several return a JSON
+        // body with success:false and no flag at all (R25 0.1). Both are an
+        // ERROR outcome: charged nothing, logged and metered as an error. The
+        // flag is set on the second kind too, so the client sees an error and
+        // the SDK skips outputSchema validation of a failure body.
+        const isErrorResult = result?.isError === true || failure(result) !== null;
+        if (isErrorResult) result.isError = true;
         outcome = isErrorResult ? 'error' : 'success';
-        // A pre-flight refusal (robots.txt disallowed the path, or the host is
-        // blocklisted) means we fetched nothing at all, so it costs nothing —
-        // not even the half-credit error rate, which exists for work that ran
-        // and then failed. Only when the refusal actually sank the call: a
-        // multi-URL tool that skipped one disallowed URL and still returned a
-        // result did real work and bills for it.
-        const refused = isErrorResult && preflightRefusal() !== null;
 
         // Phase 5 (5.3): redact_pii scrubs the page text this call returns.
         // It runs BEFORE the inline-threshold stage below, so what gets
@@ -191,21 +184,20 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
           try { shapeLargeResult(toolName, result, params); } catch { /* shaping must never break the request path */ }
         }
 
-        const base = billable();
-        const charge = creditCost === 0 || refused
-          ? 0
-          : (isErrorResult ? Math.max(1, Math.floor(base * 0.5)) : base);
+        // Failures are free, as the website promises. A pre-flight refusal that
+        // sank the call is one of them; a multi-URL tool that skipped one
+        // disallowed URL and still returned a result is a success and bills.
+        const charge = creditCost === 0 || isErrorResult ? 0 : billable();
 
         // D3.5: Surface cost transparency in all tool responses. For internal
         // proxy requests the meaningful balance is the end user's, which only
         // the website knows — report null rather than the static key's cache.
+        // Creator mode has no balance: null, since outputSchema rejects Infinity.
         try {
           const projection = authManager.projectCost(toolName, params);
-          const remainingCredits = creatorMode
-            ? Infinity
-            : internal
-              ? null
-              : (authManager.creditCache ? [...authManager.creditCache.values()][0] ?? null : null);
+          const remainingCredits = creatorMode || internal
+            ? null
+            : (authManager.creditCache ? [...authManager.creditCache.values()][0] ?? null : null);
           const costMeta = {
             projected: creditCost,
             actual: creatorMode ? 0 : charge,
@@ -223,15 +215,17 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
               // Not JSON — skip injection silently
             }
           }
+          // ...and into structuredContent, which is all some clients show (R25 0.2).
+          if (result?.structuredContent && typeof result.structuredContent === 'object') {
+            result.structuredContent._cost = costMeta;
+          }
         } catch {
           // Cost injection must never break the request path
         }
 
         // Selection hint: tell the model what to try instead of the same call,
         // after stripping ANSI codes, call logs and doubled prefixes from the
-        // error text (R24 4.5). Both act on isError results and on a JSON body
-        // with success:false, which several tools return without the flag
-        // (R24 4.1). Neither touches the charge, settled above.
+        // error text (R24 4.5). Neither touches the charge, settled above.
         try {
           normalizeErrorResult(result);
           appendFallbackHint(toolName, result);
@@ -242,26 +236,15 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
         // the backend has nothing to (re-)price; reporting 0 would still create
         // a serp_rank record the backend could recompute to full cost. The
         // same goes for a handler that reported a 0 spend.
-        if (!creatorMode && creditCost > 0 && !refused && charge > 0) {
-          await authManager.reportUsage(toolName, charge, params, isErrorResult ? 500 : 200, Date.now() - startTime);
+        if (!creatorMode && charge > 0) {
+          await authManager.reportUsage(toolName, charge, params, 200, Date.now() - startTime);
         }
 
         return result;
       } catch (error) {
+        // A throw is a failure, and failures are free: nothing is reported.
         outcome = 'error';
         thrown = error;
-        // Half-charge on error — but never charge a free (0-cost) call, never
-        // let Math.max(1, …) floor a 0 up to 1 credit, and never bill at all
-        // if the handler never ran (e.g. the credit check itself threw).
-        if (!creatorMode && creditCost > 0 && handlerStarted && preflightRefusal() === null) {
-          await authManager.reportUsage(
-            toolName,
-            Math.max(1, Math.floor(billable() * 0.5)),
-            params,
-            500,
-            Date.now() - startTime
-          );
-        }
         throw error;
       } finally {
         const durationMs = Date.now() - startTime;

package/src/skills/agent-skills/crawlforge-batch-automation/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-batch-automation
 description: "Automates large scraping jobs and browser interactions with CrawlForge's batch_scrape, get_batch_results, scrape_with_actions, and generate_llms_txt tools. Use when the user wants to scrape many URLs at once, batch-scrape a list of pages, collect dozens of product, news, or competitor pages, run browser actions (click, type, scroll, wait) before scraping, log in or fill a form before extracting, or generate an llms.txt for a site. Use sync mode for up to about 25 URLs and async with a webhook for large batches; retrieve paginated output with get_batch_results."
 metadata:
-  version: 6.21.0
+  version: 6.22.0
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-browser-sessions/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-browser-sessions
 description: "Keeps one browser page alive across several tool calls with CrawlForge's browser_session tool. Use when the user needs to log in and then read pages behind that login, work through a multi-step flow (search, filter, paginate, fill a wizard), or look at a page before deciding what to click. The loop is: open a session on a URL, snapshot it to list the interactive elements with stable refs (@e1, @e2), act on those refs, read the content, close. Unlike scrape_with_actions, which is one-shot and closes its browser when the call returns, a session survives between calls, so one login is paid for once and a wrong selector costs one call instead of a whole chain."
 metadata:
-  version: 6.21.0
+  version: 6.22.0
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-change-tracking/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-change-tracking
 description: "Monitors web pages for changes over time with CrawlForge's track_changes tool. Use when the user wants to track changes to a page, watch a URL, monitor competitor pricing, detect when content updates, get notified of regulation or product-availability changes, or diff a page against a saved baseline. Workflow: create a baseline with operation create_baseline, then periodically compare with operation compare to get a change percentage and a diff; supports CSS-selector scoping, webhooks, and scheduled monitoring."
 metadata:
-  version: 6.21.0
+  version: 6.22.0
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-deep-research/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-deep-research
 description: "Runs multi-source web research and autonomous question-answering with CrawlForge's deep_research, agent, search_web, and reddit_search tools. Use when the user wants to research a topic, do a deep dive, compare competitors, gather facts with citations, answer a question from the web, search the web, or get a synthesized report from many sources, or search Reddit posts, comments, and threads. deep_research expands queries, fetches and dedupes sources, then synthesizes; agent autonomously plans searches and navigation from a plain-English prompt with no URLs needed; search_web returns ranked results; reddit_search searches Reddit via community archives (reddit.com blocks scrapers). Caps costs via max_urls and confirms before expensive runs."
 metadata:
-  version: 6.21.0
+  version: 6.22.0
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-getting-started/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-getting-started
 description: "Orientation and tool-selection guide for the CrawlForge MCP server's 31 web tools. Use when the user is getting started with CrawlForge, asks which CrawlForge tool to use, how to set up the API key, how skills or the CLI work, what a tool costs in credits, or when one tool fails and a fallback is needed. Routes requests to the right specialized skill (web scraping, deep research, stealth, structured extraction, change tracking, batch automation, browser sessions), and explains MCP-tools-vs-CLI, the Ollama-first LLM fallback chain, and per-tool credit costs."
 metadata:
-  version: 6.21.0
+  version: 6.22.0
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-getting-started/references/credits.md

@@ -85,4 +85,4 @@ metered; there is no free tier. Tools marked "scales" cost more as work grows.
   only 2 when the plain fetch works; `scrape` then `stealth_mode` costs 7 either
   way. Do not set it on a site that reads fine — it raises the projection your
   balance is checked against.
-- Errors are charged at half the tool cost; creator mode is unlimited.
+- Failed calls are not charged; creator mode is unlimited.

package/src/skills/agent-skills/crawlforge-stealth-browsing/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-stealth-browsing
 description: "Bypasses bot detection with CrawlForge's stealth_mode tool and looks up a country's locale settings with localization. Use when a site returns 403 or 429, CAPTCHAs, 'please enable JavaScript', or empty content, or is protected by Cloudflare, DataDome, or PerimeterX, or when the user needs a page in a specific locale, language, or timezone. stealth_mode runs a stealth browser (engine auto by default: camoufox/Firefox for advanced fingerprinting, playwright/Chromium when pinned for speed) and can screenshot; localization returns a country's Accept-Language, timezone and currency for you to pass to fetch_url or stealth_mode - it applies nothing itself and uses no proxy. Explains when to escalate from a normal scrape to stealth, and why a hard block needs a residential proxy."
 metadata:
-  version: 6.21.0
+  version: 6.22.0
   source: crawlforge-mcp-server
 ---
 
@@ -32,20 +32,13 @@ Still never reach for `stealth_mode` first.
 
 ## stealth_mode (cost: 5)
 
-`stealth_mode` is operation-based. Typical flow: create a context, then create a
-page that navigates to the target URL.
+`stealth_mode` is operation-based. `operation:"scrape"` is the one-shot path:
+it renders one URL and returns the requested formats (examples below).
 
-```json
-{
-  "tool": "stealth_mode",
-  "params": {
-    "operation": "create_context",
-    "stealthConfig": { "level": "advanced", "simulateHumanBehavior": true }
-  }
-}
-```
-
-Then use the returned `contextId`:
+`create_context` then `create_page` is a navigation and challenge check, not a
+multi-step path: `create_page` loads `urlToTest`, reports its status, title and
+any challenge page it detected, and closes the page before the call returns.
+Nothing on that page carries over to a later call.
 
 ```json
 {
@@ -54,6 +47,10 @@ Then use the returned `contextId`:
 }
 ```
 
+For work over several calls on one page (log in, then read; click through a
+flow), use `browser_session` with `stealth:true` (see the
+crawlforge-browser-sessions skill).
+
 Operations: `scrape`, `configure`, `create_context`, `create_page`,
 `get_stats`, `cleanup`. `stealthConfig.level` is `basic` / `medium` (default) /
 `advanced`. Always run `cleanup` when done to release the browser.

package/src/skills/agent-skills/crawlforge-structured-extraction/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-structured-extraction
 description: "Extracts structured JSON and analyzes content with CrawlForge's extract_structured, extract_with_llm, extract_embedded_state, scrape_structured, scrape_template, process_document, analyze_content, summarize_content, and list_ollama_models tools. Use when the user wants to extract specific fields, pull data into a JSON schema, extract by natural-language prompt, scrape with CSS selectors, read a page's embedded JavaScript state (__NEXT_DATA__, React Server Components, Nuxt, Apollo, Redux), get product, profile, or repo data from known sites (Amazon, LinkedIn, GitHub, YouTube, Reddit, and more), parse a PDF or DOCX, summarize a page, or analyze sentiment, entities, or keywords. Defaults to local Ollama for LLM extraction; OpenAI and Anthropic optional."
 metadata:
-  version: 6.21.0
+  version: 6.22.0
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-web-scraping/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-web-scraping
 description: "Scrapes web pages and returns clean Markdown, HTML, plain text, links, or metadata using CrawlForge's scrape, fetch_url, extract_content, extract_text, extract_links, extract_metadata, map_site, and crawl_deep tools. Use when the user wants to scrape a URL, fetch a page, get the markdown or text of a website, extract links or metadata, read an article, map a site's URLs, generate a sitemap, or crawl a whole website or documentation site. Prefer the unified scrape tool to get multiple formats in one call; use map_site to discover URLs and crawl_deep to walk an entire site."
 metadata:
-  version: 6.21.0
+  version: 6.22.0
   source: crawlforge-mcp-server
 ---
 

package/src/tools/advanced/BrowserSessionTool.js

@@ -25,6 +25,7 @@
 
 import { z } from 'zod';
 import { load } from 'cheerio';
+import { absoluteUrls } from 'crawlforge-extractors';
 import { createHash } from 'node:crypto';
 
 import ActionExecutor from '../../core/ActionExecutor.js';
@@ -577,8 +578,16 @@ export class BrowserSessionTool {
     if (params.formats.includes('markdown')) {
       // Readability finds no article on most app pages, and then no markdown is
       // produced at all (R20, 2026-09-07). Convert the DOM we hold instead of
-      // handing back a placeholder.
-      content.markdown = (params.onlyMainContent && extracted.content?.markdown) || htmlToMarkdown(html);
+      // handing back a placeholder. The raw DOM carries no base URL, so its
+      // links are resolved against the page first: HN's "item?id=…" hrefs came
+      // back relative and unusable (R25, 2026-10-19).
+      let markdown = params.onlyMainContent && extracted.content?.markdown;
+      if (!markdown) {
+        const $page = load(html);
+        absoluteUrls($page, url);
+        markdown = htmlToMarkdown($page.html());
+      }
+      content.markdown = markdown;
     }
     if (params.formats.includes('json')) {
       content.json = {

package/src/tools/advanced/batchScrape/queue.js

@@ -3,7 +3,7 @@
  * Semaphore-based concurrency runner that dispatches work to worker.js.
  */
 
-import { scrapeUrl } from './worker.js';
+import { scrapeUrl, failedEntryHint } from './worker.js';
 
 /** Semaphore for concurrency limiting. */
 class Semaphore {
@@ -57,10 +57,12 @@ export async function scrapeUrlsBatch(urlConfigs, options, defaultTimeout) {
 
   return settled.map((result, index) => {
     if (result.status === 'fulfilled') return result.value;
+    const error = result.reason?.message || 'Unknown error';
     return {
       success: false,
       url: urlConfigs[index].url,
-      error: result.reason?.message || 'Unknown error',
+      error,
+      next_step: failedEntryHint(error),
       timestamp: Date.now()
     };
   });