crawlforge-mcp-server 5.1.0 → 5.2.0

package/src/core/llm/LLMManager.js

@@ -1,5 +1,6 @@
 import { OpenAIProvider } from './OpenAIProvider.js';
 import { AnthropicProvider } from './AnthropicProvider.js';
+import { OllamaProvider } from './OllamaProvider.js';
 import { Logger } from '../../utils/Logger.js';
 
 /**
@@ -12,7 +13,10 @@ export class LLMManager {
     this.providers = new Map();
     this.defaultProvider = null;
     this.fallbackProvider = null;
-    
+    // Ollama needs no credential, so its presence can only be settled by an
+    // HTTP probe. Cached here for the lifetime of the manager; see ready().
+    this._ollamaProbe = null;
+
     this.initializeProviders(options);
   }
 
@@ -23,6 +27,7 @@ export class LLMManager {
     const {
       openai = {},
       anthropic = {},
+      ollama = {},
       defaultProvider = 'auto'
     } = options;
 
@@ -40,6 +45,17 @@ export class LLMManager {
       this.logger.info('Anthropic provider initialized');
     }
 
+    // Initialize Ollama provider. Unlike the cloud providers there is no API
+    // key to gate on — a local Ollama is the zero-config default — so it is
+    // registered optimistically and dropped by ready() if the host is not
+    // reachable. Without this, a machine running Ollama but holding no cloud
+    // keys reported "no LLM providers available" and every caller silently
+    // downgraded to keyword/CSS extraction.
+    if (ollama.enabled !== false && process.env.DISABLE_OLLAMA !== 'true') {
+      this.providers.set('ollama', new OllamaProvider(ollama));
+      this.logger.info('Ollama provider initialized');
+    }
+
     // Set default provider
     this.setDefaultProvider(defaultProvider);
   }
@@ -49,14 +65,12 @@ export class LLMManager {
    */
   setDefaultProvider(providerName) {
     if (providerName === 'auto') {
-      // Auto-select: prefer OpenAI for embeddings, fallback to Anthropic
-      if (this.providers.has('openai')) {
-        this.defaultProvider = 'openai';
-        this.fallbackProvider = this.providers.has('anthropic') ? 'anthropic' : null;
-      } else if (this.providers.has('anthropic')) {
-        this.defaultProvider = 'anthropic';
-        this.fallbackProvider = null;
-      }
+      // Auto-select in order of preference: a cloud provider is only present
+      // when its key was deliberately configured, so it outranks the
+      // zero-config local Ollama.
+      const preference = ['openai', 'anthropic', 'ollama'].filter(name => this.providers.has(name));
+      this.defaultProvider = preference[0] || null;
+      this.fallbackProvider = preference[1] || null;
     } else if (this.providers.has(providerName)) {
       this.defaultProvider = providerName;
       // Set fallback to other available provider
@@ -351,7 +365,10 @@ Extract the data and return valid JSON:`;
       const response = await this.generateCompletion(extractionPrompt, {
         systemPrompt,
         maxTokens: scaledTokens,
-        temperature: 0.1
+        temperature: 0.1,
+        // Constrain the output to a parseable object. Small local models
+        // otherwise wrap the JSON in prose and the parse below throws.
+        format: 'json'
       });
 
       // Strip markdown code fences if present
@@ -362,12 +379,16 @@ Extract the data and return valid JSON:`;
       const validation = this.validateAgainstSchema(parsed, schema);
       return {
         data: parsed,
+        method: 'llm',
         valid: validation.valid,
         validationErrors: validation.errors
       };
     } catch (error) {
       this.logger.warn('LLM structured extraction failed, using fallback', { error: error.message });
-      return this.fallbackStructuredExtraction(content, schema);
+      // Report which path produced the data. Callers previously labelled this
+      // result "llm", so a failed LLM call was returned as a high-confidence
+      // LLM extraction.
+      return { ...this.fallbackStructuredExtraction(content, schema), error: error.message };
     }
   }
 
@@ -434,6 +455,7 @@ Extract the data and return valid JSON:`;
 
     return {
       data: extracted,
+      method: 'keyword_fallback',
       valid: false,
       validationErrors: ['Used fallback extraction — no LLM provider available']
     };
@@ -498,6 +520,34 @@ Extract the data and return valid JSON:`;
     return this.providers.size > 0;
   }
 
+  /**
+   * Resolve whether an LLM can actually be reached, probing Ollama once and
+   * caching the answer. Callers that branch on LLM-vs-fallback should await
+   * this rather than read isAvailable(), which reports Ollama optimistically
+   * because its availability cannot be determined synchronously.
+   *
+   * An unreachable Ollama is de-registered, so isAvailable() becomes accurate
+   * from that point on.
+   * @returns {Promise<boolean>}
+   */
+  async ready() {
+    const ollama = this.providers.get('ollama');
+    if (ollama) {
+      if (this._ollamaProbe === null) {
+        this._ollamaProbe = ollama.isAvailable();
+      }
+      const reachable = await this._ollamaProbe;
+      if (!reachable) {
+        this.providers.delete('ollama');
+        if (this.defaultProvider === 'ollama' || this.fallbackProvider === 'ollama') {
+          this.setDefaultProvider('auto');
+        }
+        this.logger.warn('Ollama is not reachable; provider de-registered');
+      }
+    }
+    return this.isAvailable();
+  }
+
   /**
    * Get available providers metadata
    */

package/src/core/llm/OllamaProvider.js

@@ -0,0 +1,139 @@
+import { LLMProvider } from './LLMProvider.js';
+import { ollamaBaseUrl, ollamaHeaders, selectOllamaModel } from '../../utils/ollamaConfig.js';
+
+/**
+ * Ollama Provider
+ * Implements LLM operations against a local (or hosted) Ollama server.
+ *
+ * Ollama needs no API key, so it is the provider that is available by default.
+ * Reachability is established by probing /api/tags rather than by the presence
+ * of a credential — see LLMManager.ready().
+ */
+export class OllamaProvider extends LLMProvider {
+  constructor(options = {}) {
+    super(options);
+
+    // Resolved lazily: choosing the best installed model needs an HTTP call.
+    this.model = options.model || null;
+    this.embeddingModel = options.embeddingModel || process.env.OLLAMA_EMBEDDING_MODEL || null;
+    this.timeout = options.timeout || 120000;
+  }
+
+  /** The model to use, selecting the best installed one on first use. */
+  async resolveModel() {
+    if (!this.model) this.model = await selectOllamaModel();
+    return this.model;
+  }
+
+  async generateCompletion(prompt, options = {}) {
+    const model = await this.resolveModel();
+    const {
+      maxTokens = 1000,
+      temperature = 0.7,
+      systemPrompt = null,
+      // 'json' constrains the model to emit a parseable object, or pass a JSON
+      // Schema to constrain the shape as well. Small local models otherwise
+      // wrap JSON in prose and the caller's JSON.parse fails.
+      format = null
+    } = options;
+
+    const messages = [];
+    if (systemPrompt) {
+      messages.push({ role: 'system', content: systemPrompt });
+    }
+    messages.push({ role: 'user', content: prompt });
+
+    const body = {
+      model,
+      messages,
+      stream: false,
+      options: { num_predict: maxTokens, temperature }
+    };
+    if (format) body.format = format;
+
+    let response;
+    try {
+      response = await fetch(`${ollamaBaseUrl()}/api/chat`, {
+        method: 'POST',
+        headers: ollamaHeaders({ 'Content-Type': 'application/json' }),
+        body: JSON.stringify(body),
+        signal: AbortSignal.timeout(this.timeout)
+      });
+    } catch (error) {
+      const message = /fetch failed|ECONNREFUSED/i.test(error.message)
+        ? `Ollama is not running at ${ollamaBaseUrl()}. Start it with "ollama serve".`
+        : `Ollama request failed: ${error.message}`;
+      this.logger.error('Ollama completion failed', { error: message });
+      throw new Error(message);
+    }
+
+    if (!response.ok) {
+      const errText = await response.text().catch(() => '');
+      const message = response.status === 404 && /model.*not found|pull/i.test(errText)
+        ? `Ollama model "${model}" is not pulled. Run: "ollama pull ${model}"`
+        : `Ollama API error ${response.status}: ${errText.slice(0, 200)}`;
+      this.logger.error('Ollama completion failed', { error: message });
+      throw new Error(message);
+    }
+
+    const json = await response.json();
+    const content = json?.message?.content;
+    if (!content) {
+      throw new Error('No completion generated');
+    }
+    return content.trim();
+  }
+
+  async generateEmbedding(text) {
+    const model = this.embeddingModel || await this.resolveModel();
+    const response = await fetch(`${ollamaBaseUrl()}/api/embeddings`, {
+      method: 'POST',
+      headers: ollamaHeaders({ 'Content-Type': 'application/json' }),
+      body: JSON.stringify({ model, prompt: text }),
+      signal: AbortSignal.timeout(this.timeout)
+    });
+
+    if (!response.ok) {
+      const errText = await response.text().catch(() => '');
+      throw new Error(`Ollama embedding error ${response.status}: ${errText.slice(0, 200)}`);
+    }
+
+    const json = await response.json();
+    if (!Array.isArray(json.embedding) || json.embedding.length === 0) {
+      throw new Error('No embedding generated');
+    }
+    return json.embedding;
+  }
+
+  /**
+   * Reachability check. Lists installed models rather than running a completion,
+   * so it costs nothing and stays fast even on a cold model.
+   * @returns {Promise<boolean>}
+   */
+  async isAvailable() {
+    try {
+      const response = await fetch(`${ollamaBaseUrl()}/api/tags`, {
+        headers: ollamaHeaders(),
+        signal: AbortSignal.timeout(3000)
+      });
+      return response.ok;
+    } catch {
+      return false;
+    }
+  }
+
+  getMetadata() {
+    return {
+      ...super.getMetadata(),
+      name: 'Ollama',
+      baseUrl: ollamaBaseUrl(),
+      model: this.model || '(selected on first use)',
+      embeddingModel: this.embeddingModel || '(same as model)',
+      capabilities: {
+        completion: true,
+        embedding: true,
+        similarity: true
+      }
+    };
+  }
+}

package/src/core/processing/BrowserProcessor.js

@@ -246,9 +246,23 @@ export class BrowserProcessor {
    * @returns {Promise<void>}
    */
   async initBrowser() {
-    if (!this.browser) {
-      this.browser = await chromium.launch({
+    // A Chromium that was OOM-killed or crashed doesn't error on reuse — its
+    // protocol calls hang. Detect the corpse and relaunch instead.
+    if (this.browser && !this.browser.isConnected()) {
+      this.browser = null;
+    }
+    if (this.browser) {
+      return;
+    }
+    // Only one launch in flight: concurrent cold-start callers would each
+    // launch a Chromium and orphan every instance but the last assignment.
+    if (!this._launchPromise) {
+      this._launchPromise = chromium.launch({
         headless: true,
+        // Playwright never reads this env var itself; hosted images (see
+        // Dockerfile) set it to their system Chromium instead of downloading
+        // Playwright's browsers.
+        executablePath: process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH || undefined,
         args: [
           '--no-sandbox',
           '--disable-dev-shm-usage',
@@ -258,8 +272,20 @@ export class BrowserProcessor {
           '--disable-backgrounding-occluded-windows',
           '--disable-renderer-backgrounding'
         ]
+      }).then((browser) => {
+        // Drop the handle when Chromium dies so the next call relaunches
+        // instead of reusing a corpse.
+        browser.on('disconnected', () => {
+          if (this.browser === browser) {
+            this.browser = null;
+          }
+        });
+        this.browser = browser;
+      }).finally(() => {
+        this._launchPromise = null;
       });
     }
+    await this._launchPromise;
   }
 
   /**

package/src/schemas/toolOutputSchemas.js

@@ -248,7 +248,7 @@ const searchWebShape = {
 const extractStructuredShape = {
   url: z.string().optional(),
   data: z.record(z.unknown()).optional().describe('Extracted fields matching the requested schema'),
-  extraction_method: z.string().optional().describe('"llm" | "css_fallback" | "none"'),
+  extraction_method: z.string().optional().describe('"llm" | "css_fallback" | "keyword_fallback" | "none"'),
   confidence: z.number().optional(),
   schema_used: z.record(z.unknown()).optional(),
   processingTime: z.number().optional(),
@@ -301,6 +301,8 @@ const crawlDeepShape = {
     enabled: z.boolean().optional(),
     cookies_captured: z.number().optional()
   }).passthrough().optional(),
+  crawled_at: z.string().optional().describe('When the pages were actually fetched (ISO 8601)'),
+  cached: z.boolean().optional().describe('True when this response was replayed from an earlier crawl rather than crawled now; crawled_at gives its age'),
   _cost: costShape
 };
 

package/src/server/requestContext.js

@@ -0,0 +1,26 @@
+/**
+ * Per-request context for the HTTP transports.
+ *
+ * The streamable HTTP transport authenticates every request, but tool handlers
+ * are wrapped once at registration time (withAuth) with no per-request
+ * plumbing. AsyncLocalStorage bridges that gap: the transport runs each
+ * request inside a context, and withAuth reads it at invocation time.
+ *
+ * Today the only flag is `internal`: a request authenticated with the
+ * INTERNAL_PROXY_SECRET (the crawlforge-website REST proxy). Internal requests
+ * run tools normally but are billing-exempt — the website has already checked
+ * and charged the end user's credits, so metering here would double-bill.
+ *
+ * The flag lives on the request context, never on the MCP session: a session
+ * id created by an internal request grants nothing to a later request that
+ * authenticates by other means.
+ */
+
+import { AsyncLocalStorage } from 'node:async_hooks';
+
+export const requestContext = new AsyncLocalStorage();
+
+/** True when the current async context belongs to an internal-proxy request. */
+export function isInternalRequest() {
+  return requestContext.getStore()?.internal === true;
+}

package/src/server/transports/streamableHttp.js

@@ -28,8 +28,9 @@
 import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { createServer } from 'node:http';
-import { randomUUID } from 'node:crypto';
+import { createHash, randomUUID, timingSafeEqual } from 'node:crypto';
 import { readFileSync } from 'node:fs';
+import { requestContext } from '../requestContext.js';
 import { z } from 'zod';
 import { zodToJsonSchema } from 'zod-to-json-schema';
 
@@ -82,20 +83,30 @@ function buildToolCards(server) {
  * per session, not just one transport per session.
  *
  * connectStreamableHttp() only receives a single already-configured McpServer
- * (all 27 tools/resources/prompts registered by server.js before this runs),
+ * (all 28 tools/resources/prompts registered by server.js before this runs),
  * so instead of re-running that registration per session, this clones a
  * fresh McpServer and copies over the already-registered tool/resource/
  * prompt tables — plain config + handler-closure references, no per-connection
  * state — then re-runs the same internal handler-wiring methods McpServer
  * itself calls from registerTool/registerResource/registerPrompt. This
- * depends on @modelcontextprotocol/sdk 1.29.0's internal McpServer field
- * names (`_registered*`, `set*RequestHandlers`); re-check on SDK upgrades.
+ * depends on @modelcontextprotocol/sdk 1.30.0's internal McpServer/Server
+ * field names (`_registered*`, `set*RequestHandlers`, `_capabilities`,
+ * `_taskStore`); re-check on SDK upgrades.
  *
  * @param {import('@modelcontextprotocol/sdk/server/mcp.js').McpServer} templateServer
  */
 function cloneServerForSession(templateServer) {
   const low = templateServer.server;
-  const sessionServer = new McpServer(low._serverInfo, { instructions: low._instructions });
+  // capabilities + taskStore must survive the clone: the SDK's Protocol
+  // constructor wires the tasks/* request handlers only when options.taskStore
+  // is present, and without it a tools/call on any task-capable tool
+  // (crawl_deep, batch_scrape, deep_research, agent) throws
+  // 'No task store provided for task-capable tool.'
+  const sessionServer = new McpServer(low._serverInfo, {
+    instructions: low._instructions,
+    capabilities: low._capabilities,
+    taskStore: low._taskStore
+  });
 
   sessionServer._registeredTools = templateServer._registeredTools;
   sessionServer._registeredResources = templateServer._registeredResources;
@@ -156,7 +167,7 @@ export async function connectStreamableHttp(server, authManager, logger, options
     // CORS — Smithery + browser-based MCP clients
     res.setHeader('Access-Control-Allow-Origin', '*');
     res.setHeader('Access-Control-Allow-Methods', 'GET, POST, DELETE, OPTIONS');
-    res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Mcp-Session-Id, mcp-session-id, Authorization, X-API-Key');
+    res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Mcp-Session-Id, mcp-session-id, Authorization, X-API-Key, X-Internal-Secret');
     res.setHeader('Access-Control-Expose-Headers', 'Mcp-Session-Id, mcp-session-id');
 
     if (req.method === 'OPTIONS') {
@@ -231,7 +242,11 @@ export async function connectStreamableHttp(server, authManager, logger, options
 
     // MCP endpoint
     if (req.url === '/mcp' || req.url === '/' || req.url?.startsWith('/mcp?')) {
-      // Per-request auth (bypassed in creator mode)
+      // Per-request auth (bypassed in creator mode). `internal` marks a
+      // request from the website's REST proxy (INTERNAL_PROXY_SECRET): it is
+      // billing-exempt in withAuth because the website already charged the
+      // end user. Request-scoped only — never persisted on the session.
+      let internal = false;
       if (!authManager.isCreatorMode()) {
         const authResult = await authenticateRequest(req, authManager, oauthProvider);
         if (!authResult.ok) {
@@ -246,6 +261,7 @@ export async function connectStreamableHttp(server, authManager, logger, options
           }));
           return;
         }
+        internal = authResult.internal === true;
       }
 
       if (legacy) {
@@ -258,7 +274,7 @@ export async function connectStreamableHttp(server, authManager, logger, options
           sessionServer = cloneServerForSession(server);
           reqTransport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined });
           await sessionServer.connect(reqTransport);
-          await reqTransport.handleRequest(req, res);
+          await requestContext.run({ internal }, () => reqTransport.handleRequest(req, res));
         } catch (err) {
           logger.error('Legacy Streamable HTTP request failed', { error: err?.message });
           sendRpcError(res, 500, -32603, 'Internal server error');
@@ -279,7 +295,7 @@ export async function connectStreamableHttp(server, authManager, logger, options
       const existing = sessionIdHeader ? sessions.get(String(sessionIdHeader)) : undefined;
 
       if (existing) {
-        await existing.transport.handleRequest(req, res);
+        await requestContext.run({ internal }, () => existing.transport.handleRequest(req, res));
         return;
       }
 
@@ -312,7 +328,7 @@ export async function connectStreamableHttp(server, authManager, logger, options
 
       try {
         await sessionServer.connect(transport);
-        await transport.handleRequest(req, res);
+        await requestContext.run({ internal }, () => transport.handleRequest(req, res));
       } catch (err) {
         logger.error('Streamable HTTP session initialization failed', { error: err?.message });
         safeClose(transport);
@@ -357,14 +373,41 @@ export async function connectStreamableHttp(server, authManager, logger, options
  * Validate a request's credentials.
  *
  * Accepts:
+ *   - `X-Internal-Secret: <INTERNAL_PROXY_SECRET>` — server-to-server requests
+ *     from the crawlforge-website REST proxy. Returns { ok, internal: true };
+ *     internal requests are billing-exempt in withAuth (the website already
+ *     charged the end user). Only active when the env var is set.
  *   - `Authorization: Bearer <crawlforge-api-key>` (legacy static key)
  *   - `X-API-Key: <crawlforge-api-key>` (legacy static key)
  *   - `Authorization: Bearer <oauth-access-token>` if OAuth is enabled —
  *     the OAuth provider validates the token and maps it to the API key.
  *
- * @returns {Promise<{ok: true} | {ok: false, status: number, error: string, message: string, reason: string}>}
+ * @returns {Promise<{ok: true, internal?: boolean} | {ok: false, status: number, error: string, message: string, reason: string}>}
  */
 async function authenticateRequest(req, authManager, oauthProvider) {
+  // Internal proxy path first: presenting the header at all means the caller
+  // claims to be the website proxy, so a mismatch is a hard 401 rather than a
+  // fall-through to the key paths. Compare digests — timingSafeEqual on raw
+  // strings throws on length mismatch, which would leak length via timing.
+  const internalSecret = process.env.INTERNAL_PROXY_SECRET;
+  const providedSecret = (req.headers['x-internal-secret'] || '').toString();
+  if (providedSecret) {
+    if (internalSecret) {
+      const provided = createHash('sha256').update(providedSecret).digest();
+      const expected = createHash('sha256').update(internalSecret).digest();
+      if (timingSafeEqual(provided, expected)) {
+        return { ok: true, internal: true };
+      }
+    }
+    return {
+      ok: false,
+      status: 401,
+      error: 'Unauthorized',
+      message: 'Invalid internal secret.',
+      reason: 'invalid-internal-secret'
+    };
+  }
+
   const authHeader = (req.headers['authorization'] || '').toString();
   const apiKeyHeader = (req.headers['x-api-key'] || '').toString();
   const expectedKey = authManager.getConfig()?.apiKey;

package/src/server/withAuth.js

@@ -15,6 +15,7 @@
 
 import { createHash } from 'node:crypto';
 import { recordToolInvocation } from '../observability/tracing.js';
+import { isInternalRequest } from './requestContext.js';
 
 export function hashParams(params) {
   try {
@@ -36,7 +37,12 @@ export function makeWithAuth({ authManager, logger, metrics = null }) {
       const startTime = Date.now();
       const paramHash = hashParams(params);
       const creatorMode = authManager.isCreatorMode();
-      const creditCost = creatorMode ? 0 : authManager.getToolCost(toolName, params);
+      // Internal = a request from the website REST proxy (see requestContext).
+      // Billing-exempt like creator mode — the website already checked and
+      // charged the end user's credits — but auth still happened per-request.
+      const internal = isInternalRequest();
+      const billingExempt = creatorMode || internal;
+      const creditCost = billingExempt ? 0 : authManager.getToolCost(toolName, params);
       let outcome = 'pending';
       let thrown = null;
       // Only bill the error-path half-charge once the handler has actually run.
@@ -45,7 +51,11 @@ export function makeWithAuth({ authManager, logger, metrics = null }) {
       let handlerStarted = false;
 
       try {
-        if (!creatorMode) {
+        // billingExempt covers creator mode and authenticated internal-proxy
+        // requests (the website REST layer has already checked AND charged the
+        // end user's credits before forwarding — checking the static key's
+        // balance here would gate users on an unrelated account).
+        if (!billingExempt) {
           const hasCredits = await authManager.checkCredits(creditCost);
           if (!hasCredits) {
             outcome = 'insufficient_credits';
@@ -76,10 +86,16 @@ export function makeWithAuth({ authManager, logger, metrics = null }) {
           ? 0
           : (isErrorResult ? Math.max(1, Math.floor(creditCost * 0.5)) : creditCost);
 
-        // D3.5: Surface cost transparency in all tool responses
+        // D3.5: Surface cost transparency in all tool responses. For internal
+        // proxy requests the meaningful balance is the end user's, which only
+        // the website knows — report null rather than the static key's cache.
         try {
           const projection = authManager.projectCost(toolName, params);
-          const remainingCredits = creatorMode ? Infinity : (authManager.creditCache ? [...authManager.creditCache.values()][0] ?? null : null);
+          const remainingCredits = creatorMode
+            ? Infinity
+            : internal
+              ? null
+              : (authManager.creditCache ? [...authManager.creditCache.values()][0] ?? null : null);
           const costMeta = {
             projected: creditCost,
             actual: creatorMode ? 0 : charge,
@@ -134,7 +150,8 @@ export function makeWithAuth({ authManager, logger, metrics = null }) {
           durationMs,
           outcome,
           creditCost,
-          creatorMode
+          creatorMode,
+          internal
         });
 
         // Prometheus (no-op unless registry was supplied)
@@ -161,7 +178,8 @@ export function makeWithAuth({ authManager, logger, metrics = null }) {
           duration_ms: durationMs,
           outcome,
           credit_cost: creditCost,
-          creator_mode: creatorMode
+          creator_mode: creatorMode,
+          internal
         }, thrown);
       }
     };

package/src/skills/agent-skills/crawlforge-deep-research/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: crawlforge-deep-research
-description: "Runs multi-source web research and autonomous question-answering with CrawlForge's deep_research, agent, and search_web tools. Use when the user wants to research a topic, do a deep dive, compare competitors, gather facts with citations, answer a question from the web, search the web, or get a synthesized report from many sources. deep_research expands queries, fetches and dedupes sources, then synthesizes; agent autonomously plans searches and navigation from a plain-English prompt with no URLs needed; search_web returns ranked results. Caps costs via max_urls and confirms before expensive runs."
+description: "Runs multi-source web research and autonomous question-answering with CrawlForge's deep_research, agent, search_web, and reddit_search tools. Use when the user wants to research a topic, do a deep dive, compare competitors, gather facts with citations, answer a question from the web, search the web, or get a synthesized report from many sources, or search Reddit posts, comments, and threads. deep_research expands queries, fetches and dedupes sources, then synthesizes; agent autonomously plans searches and navigation from a plain-English prompt with no URLs needed; search_web returns ranked results; reddit_search searches Reddit via community archives (reddit.com blocks scrapers). Caps costs via max_urls and confirms before expensive runs."
 metadata:
   version: 4.8.0
   source: crawlforge-mcp-server
@@ -9,13 +9,15 @@ metadata:
 # CrawlForge Deep Research
 
 Answer questions and produce reports from many web sources using the CrawlForge
-MCP server. Three tools, from lightest to heaviest: `search_web` (ranked
-results), `agent` (autonomous plan-and-answer), `deep_research` (exhaustive
+MCP server. Four tools, from lightest to heaviest: `reddit_search` (Reddit
+posts/comments/threads), `search_web` (ranked results), `agent` (autonomous plan-and-answer), `deep_research` (exhaustive
 multi-source synthesis).
 
 ## When to use
 
 - "Search the web for X" / "find pages about X" → `search_web`
+- "Search Reddit for X" / "what does Reddit say about X" / "read this Reddit
+  thread" → `reddit_search`
 - "Answer this question from the web" / "what are the top 5 X" (no URLs given) → `agent`
 - "Research this topic in depth" / "compare competitors" / "give me a cited
   report" / "gather facts with sources" → `deep_research`
@@ -28,6 +30,7 @@ skill (`scrape` / `extract_content`) instead.
 | Need | Tool | Cost |
 |------|------|------|
 | A ranked list of result URLs + snippets | `search_web` | 5 |
+| Reddit posts, comments, or a full thread | `reddit_search` | 2 |
 | A direct answer, agent decides what to read | `agent` | 8 (scales) |
 | A synthesized, multi-source, cited report | `deep_research` | 10+ (scales) |
 
@@ -48,6 +51,26 @@ Returns titles, URLs, snippets. Supports `lang`, `site` (domain filter),
 `enable_ranking`, and `enable_deduplication`. CLI:
 `crawlforge search "MCP server tutorial" --limit 5`.
 
+## reddit_search (cost: 2)
+
+reddit.com 403-blocks direct scraping, so this queries the Arctic Shift and
+PullPush community archives instead (free, no Reddit credentials).
+
+```json
+{
+  "tool": "reddit_search",
+  "params": { "query": "best MCP servers", "subreddit": "ClaudeAI", "limit": 10 }
+}
+```
+
+Modes: `posts` (default), `comments`, and `thread` (a post plus its nested
+comment tree via `link_id`). Scope with `subreddit`/`author` for near-real-time
+results (Arctic Shift); an unscoped `query` searches all of Reddit via PullPush
+(best-effort — it rate-limits aggressively). `after`/`before` accept ISO dates,
+epoch seconds, or offsets like `"7d"`. Scores of posts under ~36h old read 0/1
+until the archive backfills. Space calls out rather than firing them in
+parallel — the archives shed load under concurrent bursts.
+
 ## agent — autonomous answer (cost: 8, scales with maxUrls)
 
 No URLs required. The agent plans search queries, fetches and filters relevant