firecrawl-mcp 1.5.0 → 1.7.0

package/dist/index.js

@@ -513,11 +513,33 @@ const DEEP_RESEARCH_TOOL = {
             maxUrls: {
                 type: 'number',
                 description: 'Maximum number of URLs to analyze (1-1000)',
-            }
+            },
         },
         required: ['query'],
     },
 };
+const GENERATE_LLMSTXT_TOOL = {
+    name: 'firecrawl_generate_llmstxt',
+    description: 'Generate standardized LLMs.txt file for a given URL, which provides context about how LLMs should interact with the website.',
+    inputSchema: {
+        type: 'object',
+        properties: {
+            url: {
+                type: 'string',
+                description: 'The URL to generate LLMs.txt from',
+            },
+            maxUrls: {
+                type: 'number',
+                description: 'Maximum number of URLs to process (1-100, default: 10)',
+            },
+            showFullText: {
+                type: 'boolean',
+                description: 'Whether to show the full LLMs-full.txt in the response',
+            },
+        },
+        required: ['url'],
+    },
+};
 // Type guards
 function isScrapeOptions(args) {
     return (typeof args === 'object' &&
@@ -563,10 +585,16 @@ function isExtractOptions(args) {
     return (Array.isArray(urls) &&
         urls.every((url) => typeof url === 'string'));
 }
+function isGenerateLLMsTextOptions(args) {
+    return (typeof args === 'object' &&
+        args !== null &&
+        'url' in args &&
+        typeof args.url === 'string');
+}
 // Server implementation
 const server = new Server({
     name: 'firecrawl-mcp',
-    version: '1.3.2',
+    version: '1.7.0',
 }, {
     capabilities: {
         tools: {},
@@ -607,6 +635,17 @@ const creditUsage = {
 function delay(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
 }
+let isStdioTransport = false;
+function safeLog(level, data) {
+    if (isStdioTransport) {
+        // For stdio transport, log to stderr to avoid protocol interference
+        console.error(`[${level}] ${typeof data === 'object' ? JSON.stringify(data) : data}`);
+    }
+    else {
+        // For other transport types, use the normal logging mechanism
+        server.sendLoggingMessage({ level, data });
+    }
+}
 // Add retry logic with exponential backoff
 async function withRetry(operation, context, attempt = 1) {
     try {
@@ -618,10 +657,7 @@ async function withRetry(operation, context, attempt = 1) {
         if (isRateLimit && attempt < CONFIG.retry.maxAttempts) {
             const delayMs = Math.min(CONFIG.retry.initialDelay *
                 Math.pow(CONFIG.retry.backoffFactor, attempt - 1), CONFIG.retry.maxDelay);
-            server.sendLoggingMessage({
-                level: 'warning',
-                data: `Rate limit hit for ${context}. Attempt ${attempt}/${CONFIG.retry.maxAttempts}. Retrying in ${delayMs}ms`,
-            });
+            safeLog('warning', `Rate limit hit for ${context}. Attempt ${attempt}/${CONFIG.retry.maxAttempts}. Retrying in ${delayMs}ms`);
             await delay(delayMs);
             return withRetry(operation, context, attempt + 1);
         }
@@ -632,22 +668,13 @@ async function withRetry(operation, context, attempt = 1) {
 async function updateCreditUsage(creditsUsed) {
     creditUsage.total += creditsUsed;
     // Log credit usage
-    server.sendLoggingMessage({
-        level: 'info',
-        data: `Credit usage: ${creditUsage.total} credits used total`,
-    });
+    safeLog('info', `Credit usage: ${creditUsage.total} credits used total`);
     // Check thresholds
     if (creditUsage.total >= CONFIG.credit.criticalThreshold) {
-        server.sendLoggingMessage({
-            level: 'error',
-            data: `CRITICAL: Credit usage has reached ${creditUsage.total}`,
-        });
+        safeLog('error', `CRITICAL: Credit usage has reached ${creditUsage.total}`);
     }
     else if (creditUsage.total >= CONFIG.credit.warningThreshold) {
-        server.sendLoggingMessage({
-            level: 'warning',
-            data: `WARNING: Credit usage has reached ${creditUsage.total}`,
-        });
+        safeLog('warning', `WARNING: Credit usage has reached ${creditUsage.total}`);
     }
 }
 // Initialize queue system
@@ -672,19 +699,13 @@ async function processBatchOperation(operation) {
         operation.result = response;
         // Log final credit usage for the batch
         if (!FIRECRAWL_API_URL) {
-            server.sendLoggingMessage({
-                level: 'info',
-                data: `Batch ${operation.id} completed. Total credits used: ${totalCreditsUsed}`,
-            });
+            safeLog('info', `Batch ${operation.id} completed. Total credits used: ${totalCreditsUsed}`);
         }
     }
     catch (error) {
         operation.status = 'failed';
         operation.error = error instanceof Error ? error.message : String(error);
-        server.sendLoggingMessage({
-            level: 'error',
-            data: `Batch ${operation.id} failed: ${operation.error}`,
-        });
+        safeLog('error', `Batch ${operation.id} failed: ${operation.error}`);
     }
 }
 // Tool handlers
@@ -699,6 +720,7 @@ server.setRequestHandler(ListToolsRequestSchema, async () => ({
         SEARCH_TOOL,
         EXTRACT_TOOL,
         DEEP_RESEARCH_TOOL,
+        GENERATE_LLMSTXT_TOOL,
     ],
 }));
 server.setRequestHandler(CallToolRequestSchema, async (request) => {
@@ -706,10 +728,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
     try {
         const { name, arguments: args } = request.params;
         // Log incoming request with timestamp
-        server.sendLoggingMessage({
-            level: 'info',
-            data: `[${new Date().toISOString()}] Received request for tool: ${name}`,
-        });
+        safeLog('info', `[${new Date().toISOString()}] Received request for tool: ${name}`);
         if (!args) {
             throw new Error('No arguments provided');
         }
@@ -721,16 +740,10 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                 const { url, ...options } = args;
                 try {
                     const scrapeStartTime = Date.now();
-                    server.sendLoggingMessage({
-                        level: 'info',
-                        data: `Starting scrape for URL: ${url} with options: ${JSON.stringify(options)}`,
-                    });
+                    safeLog('info', `Starting scrape for URL: ${url} with options: ${JSON.stringify(options)}`);
                     const response = await client.scrapeUrl(url, options);
                     // Log performance metrics
-                    server.sendLoggingMessage({
-                        level: 'info',
-                        data: `Scrape completed in ${Date.now() - scrapeStartTime}ms`,
-                    });
+                    safeLog('info', `Scrape completed in ${Date.now() - scrapeStartTime}ms`);
                     if ('success' in response && !response.success) {
                         throw new Error(response.error || 'Scraping failed');
                     }
@@ -756,14 +769,14 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                     }
                     // Add warning to response if present
                     if (response.warning) {
-                        server.sendLoggingMessage({
-                            level: 'warning',
-                            data: response.warning,
-                        });
+                        safeLog('warning', response.warning);
                     }
                     return {
                         content: [
-                            { type: 'text', text: contentParts.join('\n\n') || 'No content available' },
+                            {
+                                type: 'text',
+                                text: trimResponseText(contentParts.join('\n\n') || 'No content available'),
+                            },
                         ],
                         isError: false,
                     };
@@ -771,7 +784,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                 catch (error) {
                     const errorMessage = error instanceof Error ? error.message : String(error);
                     return {
-                        content: [{ type: 'text', text: errorMessage }],
+                        content: [{ type: 'text', text: trimResponseText(errorMessage) }],
                         isError: true,
                     };
                 }
@@ -789,7 +802,9 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                     throw new Error('No links received from FireCrawl API');
                 }
                 return {
-                    content: [{ type: 'text', text: response.links.join('\n') }],
+                    content: [
+                        { type: 'text', text: trimResponseText(response.links.join('\n')) },
+                    ],
                     isError: false,
                 };
             }
@@ -812,15 +827,12 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                     batchOperations.set(operationId, operation);
                     // Queue the operation
                     batchQueue.add(() => processBatchOperation(operation));
-                    server.sendLoggingMessage({
-                        level: 'info',
-                        data: `Queued batch operation ${operationId} with ${args.urls.length} URLs`,
-                    });
+                    safeLog('info', `Queued batch operation ${operationId} with ${args.urls.length} URLs`);
                     return {
                         content: [
                             {
                                 type: 'text',
-                                text: `Batch operation queued with ID: ${operationId}. Use firecrawl_check_batch_status to check progress.`,
+                                text: trimResponseText(`Batch operation queued with ID: ${operationId}. Use firecrawl_check_batch_status to check progress.`),
                             },
                         ],
                         isError: false,
@@ -831,7 +843,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                         ? error.message
                         : `Batch operation failed: ${JSON.stringify(error)}`;
                     return {
-                        content: [{ type: 'text', text: errorMessage }],
+                        content: [{ type: 'text', text: trimResponseText(errorMessage) }],
                         isError: true,
                     };
                 }
@@ -846,7 +858,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                         content: [
                             {
                                 type: 'text',
-                                text: `No batch operation found with ID: ${args.id}`,
+                                text: trimResponseText(`No batch operation found with ID: ${args.id}`),
                             },
                         ],
                         isError: true,
@@ -860,7 +872,7 @@ ${operation.result
                     ? `Results: ${JSON.stringify(operation.result, null, 2)}`
                     : ''}`;
                 return {
-                    content: [{ type: 'text', text: status }],
+                    content: [{ type: 'text', text: trimResponseText(status) }],
                     isError: false,
                 };
             }
@@ -881,7 +893,7 @@ ${operation.result
                     content: [
                         {
                             type: 'text',
-                            text: `Started crawl for ${url} with job ID: ${response.id}`,
+                            text: trimResponseText(`Started crawl for ${url} with job ID: ${response.id}`),
                         },
                     ],
                     isError: false,
@@ -902,7 +914,7 @@ Credits Used: ${response.creditsUsed}
 Expires At: ${response.expiresAt}
 ${response.data.length > 0 ? '\nResults:\n' + formatResults(response.data) : ''}`;
                 return {
-                    content: [{ type: 'text', text: status }],
+                    content: [{ type: 'text', text: trimResponseText(status) }],
                     isError: false,
                 };
             }
@@ -927,7 +939,7 @@ Description: ${result.description || 'No description'}
 ${result.markdown ? `\nContent:\n${result.markdown}` : ''}`)
                         .join('\n\n');
                     return {
-                        content: [{ type: 'text', text: results }],
+                        content: [{ type: 'text', text: trimResponseText(results) }],
                         isError: false,
                     };
                 }
@@ -936,7 +948,7 @@ ${result.markdown ? `\nContent:\n${result.markdown}` : ''}`)
                         ? error.message
                         : `Search failed: ${JSON.stringify(error)}`;
                     return {
-                        content: [{ type: 'text', text: errorMessage }],
+                        content: [{ type: 'text', text: trimResponseText(errorMessage) }],
                         isError: true,
                     };
                 }
@@ -947,16 +959,10 @@ ${result.markdown ? `\nContent:\n${result.markdown}` : ''}`)
                 }
                 try {
                     const extractStartTime = Date.now();
-                    server.sendLoggingMessage({
-                        level: 'info',
-                        data: `Starting extraction for URLs: ${args.urls.join(', ')}`,
-                    });
+                    safeLog('info', `Starting extraction for URLs: ${args.urls.join(', ')}`);
                     // Log if using self-hosted instance
                     if (FIRECRAWL_API_URL) {
-                        server.sendLoggingMessage({
-                            level: 'info',
-                            data: 'Using self-hosted instance for extraction',
-                        });
+                        safeLog('info', 'Using self-hosted instance for extraction');
                     }
                     const extractResponse = await withRetry(async () => client.extract(args.urls, {
                         prompt: args.prompt,
@@ -977,25 +983,19 @@ ${result.markdown ? `\nContent:\n${result.markdown}` : ''}`)
                         await updateCreditUsage(response.creditsUsed || 0);
                     }
                     // Log performance metrics
-                    server.sendLoggingMessage({
-                        level: 'info',
-                        data: `Extraction completed in ${Date.now() - extractStartTime}ms`,
-                    });
+                    safeLog('info', `Extraction completed in ${Date.now() - extractStartTime}ms`);
                     // Add warning to response if present
                     const result = {
                         content: [
                             {
                                 type: 'text',
-                                text: JSON.stringify(response.data, null, 2),
+                                text: trimResponseText(JSON.stringify(response.data, null, 2)),
                             },
                         ],
                         isError: false,
                     };
                     if (response.warning) {
-                        server.sendLoggingMessage({
-                            level: 'warning',
-                            data: response.warning,
-                        });
+                        safeLog('warning', response.warning);
                     }
                     return result;
                 }
@@ -1004,22 +1004,19 @@ ${result.markdown ? `\nContent:\n${result.markdown}` : ''}`)
                     // Special handling for self-hosted instance errors
                     if (FIRECRAWL_API_URL &&
                         errorMessage.toLowerCase().includes('not supported')) {
-                        server.sendLoggingMessage({
-                            level: 'error',
-                            data: 'Extraction is not supported by this self-hosted instance',
-                        });
+                        safeLog('error', 'Extraction is not supported by this self-hosted instance');
                         return {
                             content: [
                                 {
                                     type: 'text',
-                                    text: 'Extraction is not supported by this self-hosted instance. Please ensure LLM support is configured.',
+                                    text: trimResponseText('Extraction is not supported by this self-hosted instance. Please ensure LLM support is configured.'),
                                 },
                             ],
                             isError: true,
                         };
                     }
                     return {
-                        content: [{ type: 'text', text: errorMessage }],
+                        content: [{ type: 'text', text: trimResponseText(errorMessage) }],
                         isError: true,
                     };
                 }
@@ -1030,10 +1027,7 @@ ${result.markdown ? `\nContent:\n${result.markdown}` : ''}`)
                 }
                 try {
                     const researchStartTime = Date.now();
-                    server.sendLoggingMessage({
-                        level: 'info',
-                        data: `Starting deep research for query: ${args.query}`,
-                    });
+                    safeLog('info', `Starting deep research for query: ${args.query}`);
                     const response = await client.deepResearch(args.query, {
                         maxDepth: args.maxDepth,
                         timeLimit: args.timeLimit,
@@ -1041,23 +1035,14 @@ ${result.markdown ? `\nContent:\n${result.markdown}` : ''}`)
                     }, 
                     // Activity callback
                     (activity) => {
-                        server.sendLoggingMessage({
-                            level: 'info',
-                            data: `Research activity: ${activity.message} (Depth: ${activity.depth})`,
-                        });
+                        safeLog('info', `Research activity: ${activity.message} (Depth: ${activity.depth})`);
                     }, 
                     // Source callback
                     (source) => {
-                        server.sendLoggingMessage({
-                            level: 'info',
-                            data: `Research source found: ${source.url}${source.title ? ` - ${source.title}` : ''}`,
-                        });
+                        safeLog('info', `Research source found: ${source.url}${source.title ? ` - ${source.title}` : ''}`);
                     });
                     // Log performance metrics
-                    server.sendLoggingMessage({
-                        level: 'info',
-                        data: `Deep research completed in ${Date.now() - researchStartTime}ms`,
-                    });
+                    safeLog('info', `Deep research completed in ${Date.now() - researchStartTime}ms`);
                     if (!response.success) {
                         throw new Error(response.error || 'Deep research failed');
                     }
@@ -1068,42 +1053,82 @@ ${result.markdown ? `\nContent:\n${result.markdown}` : ''}`)
                         sources: response.data.sources,
                     };
                     return {
-                        content: [{ type: 'text', text: formattedResponse.finalAnalysis }],
+                        content: [
+                            {
+                                type: 'text',
+                                text: trimResponseText(formattedResponse.finalAnalysis),
+                            },
+                        ],
                         isError: false,
                     };
                 }
                 catch (error) {
                     const errorMessage = error instanceof Error ? error.message : String(error);
                     return {
-                        content: [{ type: 'text', text: errorMessage }],
+                        content: [{ type: 'text', text: trimResponseText(errorMessage) }],
+                        isError: true,
+                    };
+                }
+            }
+            case 'firecrawl_generate_llmstxt': {
+                if (!isGenerateLLMsTextOptions(args)) {
+                    throw new Error('Invalid arguments for firecrawl_generate_llmstxt');
+                }
+                try {
+                    const { url, ...params } = args;
+                    const generateStartTime = Date.now();
+                    safeLog('info', `Starting LLMs.txt generation for URL: ${url}`);
+                    // Start the generation process
+                    const response = await withRetry(async () => client.generateLLMsText(url, params), 'LLMs.txt generation');
+                    if (!response.success) {
+                        throw new Error(response.error || 'LLMs.txt generation failed');
+                    }
+                    // Log performance metrics
+                    safeLog('info', `LLMs.txt generation completed in ${Date.now() - generateStartTime}ms`);
+                    // Format the response
+                    let resultText = '';
+                    if ('data' in response) {
+                        resultText = `LLMs.txt content:\n\n${response.data.llmstxt}`;
+                        if (args.showFullText && response.data.llmsfulltxt) {
+                            resultText += `\n\nLLMs-full.txt content:\n\n${response.data.llmsfulltxt}`;
+                        }
+                    }
+                    return {
+                        content: [{ type: 'text', text: trimResponseText(resultText) }],
+                        isError: false,
+                    };
+                }
+                catch (error) {
+                    const errorMessage = error instanceof Error ? error.message : String(error);
+                    return {
+                        content: [{ type: 'text', text: trimResponseText(errorMessage) }],
                         isError: true,
                     };
                 }
             }
             default:
                 return {
-                    content: [{ type: 'text', text: `Unknown tool: ${name}` }],
+                    content: [
+                        { type: 'text', text: trimResponseText(`Unknown tool: ${name}`) },
+                    ],
                     isError: true,
                 };
         }
     }
     catch (error) {
         // Log detailed error information
-        server.sendLoggingMessage({
-            level: 'error',
-            data: {
-                message: `Request failed: ${error instanceof Error ? error.message : String(error)}`,
-                tool: request.params.name,
-                arguments: request.params.arguments,
-                timestamp: new Date().toISOString(),
-                duration: Date.now() - startTime,
-            },
+        safeLog('error', {
+            message: `Request failed: ${error instanceof Error ? error.message : String(error)}`,
+            tool: request.params.name,
+            arguments: request.params.arguments,
+            timestamp: new Date().toISOString(),
+            duration: Date.now() - startTime,
         });
         return {
             content: [
                 {
                     type: 'text',
-                    text: `Error: ${error instanceof Error ? error.message : String(error)}`,
+                    text: trimResponseText(`Error: ${error instanceof Error ? error.message : String(error)}`),
                 },
             ],
             isError: true,
@@ -1111,10 +1136,7 @@ ${result.markdown ? `\nContent:\n${result.markdown}` : ''}`)
     }
     finally {
         // Log request completion with performance metrics
-        server.sendLoggingMessage({
-            level: 'info',
-            data: `Request completed in ${Date.now() - startTime}ms`,
-        });
+        safeLog('info', `Request completed in ${Date.now() - startTime}ms`);
     }
 });
 // Helper function to format results
@@ -1128,21 +1150,29 @@ ${doc.metadata?.title ? `Title: ${doc.metadata.title}` : ''}`;
     })
         .join('\n\n');
 }
+// Add type guard for credit usage
+function hasCredits(response) {
+    return 'creditsUsed' in response && typeof response.creditsUsed === 'number';
+}
+// Utility function to trim trailing whitespace from text responses
+// This prevents Claude API errors with "final assistant content cannot end with trailing whitespace"
+function trimResponseText(text) {
+    return text.trim();
+}
 // Server startup
 async function runServer() {
     try {
         console.error('Initializing FireCrawl MCP Server...');
         const transport = new StdioServerTransport();
+        // Detect if we're using stdio transport
+        isStdioTransport = transport instanceof StdioServerTransport;
+        if (isStdioTransport) {
+            console.error('Running in stdio mode, logging will be directed to stderr');
+        }
         await server.connect(transport);
         // Now that we're connected, we can send logging messages
-        server.sendLoggingMessage({
-            level: 'info',
-            data: 'FireCrawl MCP Server initialized successfully',
-        });
-        server.sendLoggingMessage({
-            level: 'info',
-            data: `Configuration: API URL: ${FIRECRAWL_API_URL || 'default'}`,
-        });
+        safeLog('info', 'FireCrawl MCP Server initialized successfully');
+        safeLog('info', `Configuration: API URL: ${FIRECRAWL_API_URL || 'default'}`);
         console.error('FireCrawl MCP Server running on stdio');
     }
     catch (error) {
@@ -1154,7 +1184,3 @@ runServer().catch((error) => {
     console.error('Fatal error running server:', error);
     process.exit(1);
 });
-// Add type guard for credit usage
-function hasCredits(response) {
-    return 'creditsUsed' in response && typeof response.creditsUsed === 'number';
-}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "firecrawl-mcp",
-  "version": "1.5.0",
+  "version": "1.7.0",
   "description": "MCP server for FireCrawl web scraping integration. Supports both cloud and self-hosted instances. Features include web scraping, batch processing, structured data extraction, and LLM-powered content analysis.",
   "type": "module",
   "bin": {