firecrawl-mcp 1.4.2 → 1.6.0

package/README.md

@@ -6,7 +6,7 @@ Big thanks to [@vrknetha](https://github.com/vrknetha), [@cawstudios](https://ca
 
 ## Features
 
-- Scrape, crawl, search, extract and batch scrape support
+- Scrape, crawl, search, extract, deep research and batch scrape support
 - Web scraping with JS rendering
 - URL discovery and crawling
 - Web search with content extraction

package/dist/index.js

@@ -518,6 +518,28 @@ const DEEP_RESEARCH_TOOL = {
         required: ['query'],
     },
 };
+const GENERATE_LLMSTXT_TOOL = {
+    name: 'firecrawl_generate_llmstxt',
+    description: 'Generate standardized LLMs.txt file for a given URL, which provides context about how LLMs should interact with the website.',
+    inputSchema: {
+        type: 'object',
+        properties: {
+            url: {
+                type: 'string',
+                description: 'The URL to generate LLMs.txt from',
+            },
+            maxUrls: {
+                type: 'number',
+                description: 'Maximum number of URLs to process (1-100, default: 10)',
+            },
+            showFullText: {
+                type: 'boolean',
+                description: 'Whether to show the full LLMs-full.txt in the response',
+            },
+        },
+        required: ['url'],
+    },
+};
 // Type guards
 function isScrapeOptions(args) {
     return (typeof args === 'object' &&
@@ -563,6 +585,12 @@ function isExtractOptions(args) {
     return (Array.isArray(urls) &&
         urls.every((url) => typeof url === 'string'));
 }
+function isGenerateLLMsTextOptions(args) {
+    return (typeof args === 'object' &&
+        args !== null &&
+        'url' in args &&
+        typeof args.url === 'string');
+}
 // Server implementation
 const server = new Server({
     name: 'firecrawl-mcp',
@@ -699,6 +727,7 @@ server.setRequestHandler(ListToolsRequestSchema, async () => ({
         SEARCH_TOOL,
         EXTRACT_TOOL,
         DEEP_RESEARCH_TOOL,
+        GENERATE_LLMSTXT_TOOL,
     ],
 }));
 server.setRequestHandler(CallToolRequestSchema, async (request) => {
@@ -734,12 +763,36 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                     if ('success' in response && !response.success) {
                         throw new Error(response.error || 'Scraping failed');
                     }
-                    const content = 'markdown' in response
-                        ? response.markdown || response.html || response.rawHtml
-                        : null;
+                    // Format content based on requested formats
+                    const contentParts = [];
+                    if (options.formats?.includes('markdown') && response.markdown) {
+                        contentParts.push(response.markdown);
+                    }
+                    if (options.formats?.includes('html') && response.html) {
+                        contentParts.push(response.html);
+                    }
+                    if (options.formats?.includes('rawHtml') && response.rawHtml) {
+                        contentParts.push(response.rawHtml);
+                    }
+                    if (options.formats?.includes('links') && response.links) {
+                        contentParts.push(response.links.join('\n'));
+                    }
+                    if (options.formats?.includes('screenshot') && response.screenshot) {
+                        contentParts.push(response.screenshot);
+                    }
+                    if (options.formats?.includes('extract') && response.extract) {
+                        contentParts.push(JSON.stringify(response.extract, null, 2));
+                    }
+                    // Add warning to response if present
+                    if (response.warning) {
+                        server.sendLoggingMessage({
+                            level: 'warning',
+                            data: response.warning,
+                        });
+                    }
                     return {
                         content: [
-                            { type: 'text', text: content || 'No content available' },
+                            { type: 'text', text: contentParts.join('\n\n') || 'No content available' },
                         ],
                         isError: false,
                     };
@@ -1056,6 +1109,48 @@ ${result.markdown ? `\nContent:\n${result.markdown}` : ''}`)
                     };
                 }
             }
+            case 'firecrawl_generate_llmstxt': {
+                if (!isGenerateLLMsTextOptions(args)) {
+                    throw new Error('Invalid arguments for firecrawl_generate_llmstxt');
+                }
+                try {
+                    const { url, ...params } = args;
+                    const generateStartTime = Date.now();
+                    server.sendLoggingMessage({
+                        level: 'info',
+                        data: `Starting LLMs.txt generation for URL: ${url}`,
+                    });
+                    // Start the generation process
+                    const response = await withRetry(async () => client.generateLLMsText(url, params), 'LLMs.txt generation');
+                    if (!response.success) {
+                        throw new Error(response.error || 'LLMs.txt generation failed');
+                    }
+                    // Log performance metrics
+                    server.sendLoggingMessage({
+                        level: 'info',
+                        data: `LLMs.txt generation completed in ${Date.now() - generateStartTime}ms`,
+                    });
+                    // Format the response
+                    let resultText = '';
+                    if ('data' in response) {
+                        resultText = `LLMs.txt content:\n\n${response.data.llmstxt}`;
+                        if (args.showFullText && response.data.llmsfulltxt) {
+                            resultText += `\n\nLLMs-full.txt content:\n\n${response.data.llmsfulltxt}`;
+                        }
+                    }
+                    return {
+                        content: [{ type: 'text', text: resultText }],
+                        isError: false,
+                    };
+                }
+                catch (error) {
+                    const errorMessage = error instanceof Error ? error.message : String(error);
+                    return {
+                        content: [{ type: 'text', text: errorMessage }],
+                        isError: true,
+                    };
+                }
+            }
             default:
                 return {
                     content: [{ type: 'text', text: `Unknown tool: ${name}` }],
@@ -1104,6 +1199,10 @@ ${doc.metadata?.title ? `Title: ${doc.metadata.title}` : ''}`;
     })
         .join('\n\n');
 }
+// Add type guard for credit usage
+function hasCredits(response) {
+    return 'creditsUsed' in response && typeof response.creditsUsed === 'number';
+}
 // Server startup
 async function runServer() {
     try {
@@ -1130,7 +1229,3 @@ runServer().catch((error) => {
     console.error('Fatal error running server:', error);
     process.exit(1);
 });
-// Add type guard for credit usage
-function hasCredits(response) {
-    return 'creditsUsed' in response && typeof response.creditsUsed === 'number';
-}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "firecrawl-mcp",
-  "version": "1.4.2",
+  "version": "1.6.0",
   "description": "MCP server for FireCrawl web scraping integration. Supports both cloud and self-hosted instances. Features include web scraping, batch processing, structured data extraction, and LLM-powered content analysis.",
   "type": "module",
   "bin": {