@ariangibson/firecrawl-lite-mcp-server 1.1.2 → 1.4.0

package/dist/index.js

@@ -11,41 +11,17 @@ import { randomUUID } from 'node:crypto';
 // External dependencies
 import dotenv from 'dotenv';
 import axios from 'axios';
-import puppeteer from 'puppeteer';
+// Shared pure helpers (also covered by unit tests in tests/)
+import { isValidUrl, sanitizeUrl, validatePrompt, buildLlmRequestBody, parseLlmJson, } from './utils.js';
+import { loadConfig, createRotator } from './config.js';
+import { extractPageMetadata } from './htmlToMarkdown.js';
+import { createScraper } from './scraper.js';
+import { parseFirecrawlScrapeRequest, buildFirecrawlDocument, isFirecrawlRequestAuthorized, } from './firecrawlApi.js';
+import { createRequire } from 'node:module';
+const { version: PACKAGE_VERSION } = createRequire(import.meta.url)('../package.json');
 dotenv.config();
-// Constants
-const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
-const DEFAULT_VIEWPORT_WIDTH = 1920;
-const DEFAULT_VIEWPORT_HEIGHT = 1080;
-const DEFAULT_SCRAPE_DELAY_MIN = 1000;
-const DEFAULT_SCRAPE_DELAY_MAX = 3000;
-const DEFAULT_BATCH_DELAY_MIN = 2000;
-const DEFAULT_BATCH_DELAY_MAX = 5000;
-const DEFAULT_RETRY_ATTEMPTS = 3;
-const DEFAULT_RETRY_INITIAL_DELAY = 1000;
-const DEFAULT_RETRY_MAX_DELAY = 10000;
-const DEFAULT_RETRY_BACKOFF_FACTOR = 2;
 // Security constants
 const MAX_URLS_PER_REQUEST = 10;
-// Input validation utilities
-function isValidUrl(url) {
-    try {
-        const parsedUrl = new URL(url);
-        // Only allow http and https protocols
-        return ['http:', 'https:'].includes(parsedUrl.protocol);
-    }
-    catch {
-        return false;
-    }
-}
-function sanitizeUrl(url) {
-    // Remove any potentially dangerous characters
-    return url.trim().replace(/[<>'"]/g, '');
-}
-function validatePrompt(prompt) {
-    // Basic prompt validation - prevent extremely long prompts
-    return prompt.length > 0 && prompt.length < 10000;
-}
 const SCRAPE_TOOL = {
     name: 'scrape_page',
     description: 'Extract content from a single webpage',
@@ -97,11 +73,6 @@ const EXTRACT_DATA_TOOL = {
                 type: 'string',
                 description: 'Instructions for what data to extract'
             },
-            enableWebSearch: {
-                type: 'boolean',
-                description: 'Enable web search for additional context',
-                default: false
-            },
         },
         required: ['urls', 'prompt'],
     },
@@ -125,166 +96,36 @@ const EXTRACT_WITH_SCHEMA_TOOL = {
                 type: 'string',
                 description: 'Optional instructions for extraction'
             },
-            enableWebSearch: {
+        },
+        required: ['urls', 'schema'],
+    },
+};
+const SCREENSHOT_TOOL = {
+    name: 'screenshot',
+    description: 'Take a screenshot of a webpage using stealth browser',
+    inputSchema: {
+        type: 'object',
+        properties: {
+            url: { type: 'string', description: 'Webpage URL to screenshot' },
+            width: {
+                type: 'number',
+                description: 'Viewport width in pixels',
+                default: 1920
+            },
+            height: {
+                type: 'number',
+                description: 'Viewport height in pixels',
+                default: 1080
+            },
+            fullPage: {
                 type: 'boolean',
-                description: 'Enable web search for additional context',
+                description: 'Capture full page height',
                 default: false
             },
         },
-        required: ['urls', 'schema'],
+        required: ['url'],
     },
 };
-// Lightweight tool definitions for essential Firecrawl functionality
-// Local web scraping functions
-async function scrapeWebpage(url, onlyMainContent = true) {
-    // SECURITY: Validate and sanitize URL
-    if (!isValidUrl(url)) {
-        return {
-            url,
-            title: '',
-            content: '',
-            markdown: '',
-            html: '',
-            success: false,
-            error: 'Invalid URL format. Only HTTP and HTTPS URLs are allowed.'
-        };
-    }
-    const sanitizedUrl = sanitizeUrl(url);
-    let browser;
-    try {
-        // Get proxy configuration
-        const proxyUrl = CONFIG.proxy.url;
-        const proxyUsername = CONFIG.proxy.username;
-        const proxyPassword = CONFIG.proxy.password;
-        // Get scraping configuration
-        const customUserAgent = CONFIG.scraping.userAgent;
-        const viewportWidth = CONFIG.scraping.viewportWidth;
-        const viewportHeight = CONFIG.scraping.viewportHeight;
-        const delayMin = CONFIG.scraping.delayMin;
-        const delayMax = CONFIG.scraping.delayMax;
-        // Build Puppeteer launch options with enhanced anti-detection
-        // SECURITY: Removed --disable-web-security which is a major security risk
-        const launchOptions = {
-            headless: true,
-            args: [
-                '--no-sandbox',
-                '--disable-setuid-sandbox',
-                '--disable-dev-shm-usage',
-                '--disable-accelerated-2d-canvas',
-                '--no-first-run',
-                '--no-zygote',
-                '--disable-gpu',
-                '--disable-features=VizDisplayCompositor',
-                `--user-agent=${customUserAgent}`
-            ]
-        };
-        // Add proxy configuration if available
-        if (proxyUrl) {
-            launchOptions.args.push(`--proxy-server=${proxyUrl}`);
-            // If proxy requires authentication, we'll handle it in the page setup
-            if (proxyUsername && proxyPassword) {
-                console.error(`Using authenticated proxy: ${proxyUrl}`);
-            }
-            else {
-                console.error(`Using proxy: ${proxyUrl}`);
-            }
-        }
-        browser = await puppeteer.launch(launchOptions);
-        const page = await browser.newPage();
-        // Enhanced anti-detection setup
-        await page.setUserAgent(customUserAgent);
-        // Set viewport to common desktop size
-        await page.setViewport({ width: viewportWidth, height: viewportHeight });
-        // Add common browser properties to avoid detection
-        await page.evaluateOnNewDocument(() => {
-            // Override navigator properties
-            Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
-            Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] });
-            Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
-            // Mock common browser APIs
-            window.chrome = { runtime: {} };
-        });
-        // Handle proxy authentication if credentials are provided
-        if (proxyUrl && proxyUsername && proxyPassword) {
-            await page.authenticate({
-                username: proxyUsername,
-                password: proxyPassword
-            });
-        }
-        // Add random delay before navigation
-        const delay = Math.floor(Math.random() * (delayMax - delayMin)) + delayMin;
-        await new Promise(resolve => setTimeout(resolve, delay));
-        // SECURITY: Use sanitized URL and add timeout
-        await page.goto(sanitizedUrl, {
-            waitUntil: 'networkidle2',
-            timeout: 30000
-        });
-        // Wait additional time for dynamic content
-        await new Promise(resolve => setTimeout(resolve, 2000));
-        // Extract title
-        const title = await page.title();
-        // Extract content based on preference
-        let content = '';
-        let markdown = '';
-        if (onlyMainContent) {
-            // Try to extract main content using common selectors
-            const mainContent = await page.evaluate(() => {
-                const selectors = [
-                    'main',
-                    '[role="main"]',
-                    '.content',
-                    '.post-content',
-                    '.entry-content',
-                    'article',
-                    '.article-content',
-                    '#content',
-                    '.main-content'
-                ];
-                for (const selector of selectors) {
-                    const element = document.querySelector(selector);
-                    if (element && element.textContent && element.textContent.trim().length > 100) {
-                        return element.textContent.trim();
-                    }
-                }
-                // Fallback to body content
-                return document.body.textContent || '';
-            });
-            content = mainContent;
-            markdown = content;
-        }
-        else {
-            // Extract full page content
-            content = await page.evaluate(() => document.body.textContent || '');
-            markdown = content;
-        }
-        // Get HTML
-        const html = await page.content();
-        return {
-            url,
-            title,
-            content,
-            markdown,
-            html,
-            success: true
-        };
-    }
-    catch (error) {
-        return {
-            url,
-            title: '',
-            content: '',
-            markdown: '',
-            html: '',
-            success: false,
-            error: error instanceof Error ? error.message : String(error)
-        };
-    }
-    finally {
-        if (browser) {
-            await browser.close();
-        }
-    }
-}
 async function extractDataWithLLM(url, prompt, schema) {
     // SECURITY: Validate inputs
     if (!isValidUrl(url)) {
@@ -307,7 +148,7 @@ async function extractDataWithLLM(url, prompt, schema) {
     const sanitizedPrompt = prompt.trim();
     try {
         // First scrape the webpage
-        const scraped = await scrapeWebpage(url, true);
+        const scraped = await scraper.scrape(url, true);
         if (!scraped.success) {
             return {
                 url,
@@ -336,7 +177,7 @@ Webpage URL: ${sanitizedUrl}
 Webpage Title: ${scraped.title}
 
 Content:
-${scraped.content}
+${scraped.markdown || scraped.content}
 
 ${schema ? `Extract data according to this JSON schema: ${JSON.stringify(schema, null, 2)}` : ''}
 
@@ -344,8 +185,9 @@ User Request: ${sanitizedPrompt}
 
 Please provide the extracted data in JSON format. ${schema ? 'Ensure the response matches the provided schema.' : 'Structure the data logically based on the content and request.'}
 `;
-        // Get proxy configuration for LLM API calls
-        const proxyUrl = CONFIG.proxy.url;
+        // LLM API calls go out directly by default. Only use the (scraping) proxy
+        // for the LLM call when explicitly opted in via PROXY_LLM_API=true.
+        const proxyUrl = CONFIG.proxy.proxyLlmApi ? getNextProxy() : undefined;
         const proxyUsername = CONFIG.proxy.username;
         const proxyPassword = CONFIG.proxy.password;
         // Build axios configuration
@@ -369,29 +211,28 @@ Please provide the extracted data in JSON format. ${schema ? 'Ensure the respons
                 };
             }
             axiosConfig.proxy = proxyConfig;
-            console.error(`Using proxy for LLM API: ${proxyUrl}`);
+            // SECURITY: Never log proxy URLs that might contain credentials
+            console.error(`Using proxy for LLM API: [REDACTED]`);
         }
-        // Call LLM API with timeout for security
-        const response = await axios.post(`${LLM_PROVIDER_BASE_URL}/chat/completions`, {
-            model: LLM_MODEL,
-            messages: [
-                {
-                    role: 'user',
-                    content: extractionPrompt
-                }
-            ],
-            temperature: 0.1,
-            max_tokens: 2000
-        }, {
+        // Call LLM API with timeout for security.
+        // Optional tuning params (temperature, max_tokens, top_p, reasoning_effort)
+        // are sourced from CONFIG.llm so they can be set via env vars.
+        const requestBody = buildLlmRequestBody(LLM_MODEL, [
+            {
+                role: 'user',
+                content: extractionPrompt
+            }
+        ], CONFIG.llm);
+        const response = await axios.post(`${LLM_PROVIDER_BASE_URL}/chat/completions`, requestBody, {
             ...axiosConfig,
             timeout: 60000, // 60 second timeout for security
             maxContentLength: 10 * 1024 * 1024, // 10MB max response size
             maxBodyLength: 10 * 1024 * 1024
         });
         const llmResponse = response.data.choices[0].message.content;
-        // Try to parse JSON from the response
+        // Parse JSON from the response, tolerating markdown code fences and prose.
         try {
-            const extractedData = JSON.parse(llmResponse);
+            const extractedData = parseLlmJson(llmResponse);
             return {
                 url,
                 data: extractedData,
@@ -399,7 +240,7 @@ Please provide the extracted data in JSON format. ${schema ? 'Ensure the respons
             };
         }
         catch (parseError) {
-            // If JSON parsing fails, return the raw response
+            // If no JSON can be recovered, return the raw response
             return {
                 url,
                 data: { raw_response: llmResponse },
@@ -408,20 +249,40 @@ Please provide the extracted data in JSON format. ${schema ? 'Ensure the respons
         }
     }
     catch (error) {
-        // SECURITY: Prevent information disclosure in error messages
+        // SECURITY: Prevent information disclosure in the message returned to the
+        // client, but log full detail to stderr so operators can actually debug.
         const isAxiosError = axios.isAxiosError(error);
         let safeErrorMessage = 'An error occurred while processing the request';
         if (isAxiosError) {
-            // Only expose safe error information
-            if (error.response?.status === 401) {
+            const status = error.response?.status;
+            // Server-side diagnostics (stderr only — never returned to the client).
+            const responseBody = typeof error.response?.data === 'string'
+                ? error.response.data
+                : JSON.stringify(error.response?.data ?? {});
+            console.error(`LLM extract_data request failed: status=${status ?? 'n/a'} code=${error.code ?? 'n/a'} body=${responseBody.slice(0, 800)}`);
+            // HTTP status codes are not sensitive, so surface them to aid debugging.
+            if (status === 401) {
                 safeErrorMessage = 'Authentication failed with LLM provider';
             }
-            else if (error.response?.status === 429) {
+            else if (status === 429) {
                 safeErrorMessage = 'Rate limit exceeded with LLM provider';
             }
+            else if (status === 400) {
+                safeErrorMessage =
+                    'LLM provider rejected the request (HTTP 400) - check the model name and LLM_* tuning params';
+            }
             else if (error.code === 'ECONNABORTED') {
                 safeErrorMessage = 'Request timeout - LLM provider took too long to respond';
             }
+            else if (status) {
+                safeErrorMessage = `LLM provider returned an error (HTTP ${status})`;
+            }
+            else {
+                safeErrorMessage = `LLM request failed (${error.code ?? 'network error'})`;
+            }
+        }
+        else {
+            console.error('extract_data unexpected error:', error);
         }
         return {
             url: sanitizedUrl,
@@ -438,6 +299,12 @@ function isScrapeOptions(args) {
         'url' in args &&
         typeof args.url === 'string');
 }
+function isScreenshotOptions(args) {
+    return (typeof args === 'object' &&
+        args !== null &&
+        'url' in args &&
+        typeof args.url === 'string');
+}
 function isBatchScrapeOptions(args) {
     return (typeof args === 'object' &&
         args !== null &&
@@ -452,46 +319,32 @@ function isExtractOptions(args) {
         Array.isArray(args.urls) &&
         typeof args.prompt === 'string');
 }
+// For schema-based extraction the prompt is optional (the schema drives it).
+function isExtractWithSchemaOptions(args) {
+    return (typeof args === 'object' &&
+        args !== null &&
+        'urls' in args &&
+        'schema' in args &&
+        Array.isArray(args.urls) &&
+        args.schema != null &&
+        typeof args.schema === 'object');
+}
 // Remove all complex tools - keep only essential ones above
 // Server implementation
 const server = new Server({
     name: 'firecrawl-lite-mcp-server',
-    version: '1.0.0',
+    version: PACKAGE_VERSION,
 }, {
     capabilities: {
         tools: {},
     },
 });
-// Configuration for retries and monitoring
-const CONFIG = {
-    scraping: {
-        userAgent: process.env.SCRAPE_USER_AGENT || DEFAULT_USER_AGENT,
-        viewportWidth: Number(process.env.SCRAPE_VIEWPORT_WIDTH) || DEFAULT_VIEWPORT_WIDTH,
-        viewportHeight: Number(process.env.SCRAPE_VIEWPORT_HEIGHT) || DEFAULT_VIEWPORT_HEIGHT,
-        delayMin: Number(process.env.SCRAPE_DELAY_MIN) || DEFAULT_SCRAPE_DELAY_MIN,
-        delayMax: Number(process.env.SCRAPE_DELAY_MAX) || DEFAULT_SCRAPE_DELAY_MAX,
-        batchDelayMin: Number(process.env.SCRAPE_BATCH_DELAY_MIN) || DEFAULT_BATCH_DELAY_MIN,
-        batchDelayMax: Number(process.env.SCRAPE_BATCH_DELAY_MAX) || DEFAULT_BATCH_DELAY_MAX,
-    },
-    retry: {
-        maxAttempts: Number(process.env.FIRECRAWL_RETRY_MAX_ATTEMPTS) || DEFAULT_RETRY_ATTEMPTS,
-        initialDelay: Number(process.env.FIRECRAWL_RETRY_INITIAL_DELAY) || DEFAULT_RETRY_INITIAL_DELAY,
-        maxDelay: Number(process.env.FIRECRAWL_RETRY_MAX_DELAY) || DEFAULT_RETRY_MAX_DELAY,
-        backoffFactor: Number(process.env.FIRECRAWL_RETRY_BACKOFF_FACTOR) || DEFAULT_RETRY_BACKOFF_FACTOR,
-    },
-    llm: {
-        apiKey: process.env.LLM_API_KEY,
-        providerBaseUrl: process.env.LLM_PROVIDER_BASE_URL,
-        model: process.env.LLM_MODEL,
-    },
-    proxy: {
-        url: process.env.PROXY_SERVER_URL,
-        username: process.env.PROXY_SERVER_USERNAME,
-        password: process.env.PROXY_SERVER_PASSWORD,
-    },
-};
+const CONFIG = loadConfig();
+// Proxy rotation for LLM API calls (only used when PROXY_LLM_API=true).
+const getNextProxy = createRotator(CONFIG.proxy.urls);
+// Scraper: browser sessions, rotation, retries, HTML -> Markdown.
+const scraper = createScraper(CONFIG);
 // Get LLM configuration
-const LLM_API_KEY = CONFIG.llm.apiKey;
 const LLM_PROVIDER_BASE_URL = CONFIG.llm.providerBaseUrl;
 const LLM_MODEL = CONFIG.llm.model;
 // Add utility function for delay
@@ -517,6 +370,7 @@ server.setRequestHandler(ListToolsRequestSchema, async function listToolsRequest
             BATCH_SCRAPE_TOOL,
             EXTRACT_DATA_TOOL,
             EXTRACT_WITH_SCHEMA_TOOL,
+            SCREENSHOT_TOOL,
         ],
     };
 });
@@ -538,12 +392,44 @@ server.setRequestHandler(CallToolRequestSchema, async function callToolRequestHa
                 if (!isValidUrl(args.url)) {
                     throw new Error('Invalid URL format. Only HTTP and HTTPS URLs are allowed.');
                 }
-                const result = await scrapeWebpage(args.url, args.onlyMainContent !== false);
+                const result = await scraper.scrape(args.url, args.onlyMainContent !== false);
                 return {
                     content: [{ type: 'text', text: result.success ? result.markdown : `Error: ${result.error}` }],
                     isError: !result.success,
                 };
             }
+            case 'screenshot': {
+                if (!isScreenshotOptions(args)) {
+                    throw new Error('Invalid arguments for screenshot');
+                }
+                // SECURITY: Validate URL before processing
+                if (!isValidUrl(args.url)) {
+                    throw new Error('Invalid URL format. Only HTTP and HTTPS URLs are allowed.');
+                }
+                const result = await scraper.screenshot(args.url, {
+                    width: args.width || 1920,
+                    height: args.height || 1080,
+                    fullPage: args.fullPage || false,
+                });
+                if (result.success && result.dataUrl) {
+                    return {
+                        content: [{
+                                type: 'text',
+                                text: `Screenshot captured successfully!\n\nMetadata:\n- Format: ${result.metadata?.format}\n- Size: ${result.metadata?.sizeKB}KB\n- Dimensions: ${result.metadata?.dimensions}\n- Full Page: ${result.metadata?.fullPage}\n- URL: ${result.metadata?.url}\n- Timestamp: ${result.metadata?.timestamp}\n\nBase64 Data URL:\n${result.dataUrl}`
+                            }],
+                        isError: false,
+                    };
+                }
+                else {
+                    return {
+                        content: [{
+                                type: 'text',
+                                text: `Screenshot failed: ${result.error}`
+                            }],
+                        isError: true,
+                    };
+                }
+            }
             case 'batch_scrape': {
                 if (!isBatchScrapeOptions(args)) {
                     throw new Error('Invalid arguments for batch_scrape: urls array required');
@@ -560,7 +446,7 @@ server.setRequestHandler(CallToolRequestSchema, async function callToolRequestHa
                 const results = [];
                 for (const url of args.urls) {
                     try {
-                        const result = await scrapeWebpage(url, args.onlyMainContent !== false);
+                        const result = await scraper.scrape(url, args.onlyMainContent !== false);
                         results.push({
                             url,
                             success: result.success,
@@ -627,8 +513,8 @@ server.setRequestHandler(CallToolRequestSchema, async function callToolRequestHa
                 };
             }
             case 'extract_with_schema': {
-                if (!isExtractOptions(args) || !args.schema) {
-                    throw new Error('Invalid arguments for extract_with_schema: urls array, schema, and prompt required');
+                if (!isExtractWithSchemaOptions(args)) {
+                    throw new Error('Invalid arguments for extract_with_schema: urls array and schema object are required');
                 }
                 // Security validations
                 if (!Array.isArray(args.urls) || args.urls.length === 0) {
@@ -652,14 +538,18 @@ server.setRequestHandler(CallToolRequestSchema, async function callToolRequestHa
                     }
                     sanitizedUrls.push(sanitizedUrl);
                 }
-                // Validate and sanitize prompt
-                if (typeof args.prompt !== 'string') {
-                    throw new Error('Invalid prompt: must be a string');
-                }
-                if (!validatePrompt(args.prompt)) {
-                    throw new Error('Invalid prompt: must be between 1 and 10,000 characters');
+                // Prompt is optional for schema-based extraction; the schema drives
+                // the extraction. Validate it only if the caller supplied one.
+                let sanitizedPrompt = 'Extract data matching the provided JSON schema.';
+                if (args.prompt !== undefined) {
+                    if (typeof args.prompt !== 'string') {
+                        throw new Error('Invalid prompt: must be a string');
+                    }
+                    if (!validatePrompt(args.prompt)) {
+                        throw new Error('Invalid prompt: must be between 1 and 10,000 characters');
+                    }
+                    sanitizedPrompt = args.prompt.trim();
                 }
-                const sanitizedPrompt = args.prompt.trim();
                 // Validate schema (basic validation)
                 if (typeof args.schema !== 'object' || args.schema === null) {
                     throw new Error('Invalid schema: must be a valid object');
@@ -722,16 +612,6 @@ server.setRequestHandler(CallToolRequestSchema, async function callToolRequestHa
         safeLog('info', `Request completed in ${Date.now() - startTime}ms`);
     }
 });
-// Helper function to format results
-function formatResults(data) {
-    return data
-        .map((doc) => {
-        const content = doc.markdown || doc.content || 'No content';
-        return `Title: ${doc.title}
-Content: ${content.substring(0, 100)}${content.length > 100 ? '...' : ''}`;
-    })
-        .join('\n\n');
-}
 // Utility function to trim trailing whitespace from text responses
 // This prevents Claude API errors with "final assistant content cannot end with trailing whitespace"
 function trimResponseText(text) {
@@ -758,123 +638,234 @@ async function runLocalServer() {
         process.exit(1);
     }
 }
-async function runSSELocalServer() {
-    let transport = null;
-    const app = express();
-    app.get('/sse', async (req, res) => {
-        transport = new SSEServerTransport(`/messages`, res);
-        res.on('close', () => {
-            transport = null;
-        });
-        await server.connect(transport);
-    });
-    // Endpoint for the client to POST messages
-    // Remove express.json() middleware - let the transport handle the body
-    app.post('/messages', (req, res) => {
-        if (transport) {
-            transport.handlePostMessage(req, res);
-        }
-    });
-    const PORT = process.env.PORT || 3000;
-    console.log('Starting server on port', PORT);
-    try {
-        app.listen(PORT, () => {
-            console.log(`MCP SSE Server listening on http://localhost:${PORT}`);
-            console.log(`SSE endpoint: http://localhost:${PORT}/sse`);
-            console.log(`Message endpoint: http://localhost:${PORT}/messages`);
-        });
-    }
-    catch (error) {
-        console.error('Error starting server:', error);
-    }
-}
 async function runHTTPStreamableServer() {
     const app = express();
     app.use(express.json());
     // Health check endpoint
-    app.get('/health', (req, res) => {
+    app.get('/health', (_req, res) => {
         res.status(200).json({
             status: 'OK',
             server: 'Firecrawl Lite MCP Server',
-            version: '1.0.0',
-            timestamp: new Date().toISOString()
+            version: PACKAGE_VERSION,
+            timestamp: new Date().toISOString(),
+            endpoints: {
+                mcp: CONFIG.endpoints.enableHttpStreamableEndpoint ? 'enabled' : 'disabled',
+                sse: CONFIG.endpoints.enableSseEndpoint ? 'enabled' : 'disabled',
+                firecrawlApi: CONFIG.endpoints.enableFirecrawlApi ? 'enabled' : 'disabled'
+            }
         });
     });
     const transports = {};
-    // A single endpoint handles all MCP requests.
-    app.all('/mcp', async (req, res) => {
-        try {
-            const sessionId = req.headers['mcp-session-id'];
-            let transport;
-            if (sessionId && transports[sessionId]) {
-                transport = transports[sessionId];
+    let sseTransport = null;
+    // MCP endpoint - only if enabled
+    if (CONFIG.endpoints.enableHttpStreamableEndpoint) {
+        app.all('/mcp', async (req, res) => {
+            try {
+                const sessionId = req.headers['mcp-session-id'];
+                let transport;
+                if (sessionId && transports[sessionId]) {
+                    transport = transports[sessionId];
+                }
+                else if (!sessionId &&
+                    req.method === 'POST' &&
+                    req.body &&
+                    typeof req.body === 'object' &&
+                    req.body.method === 'initialize') {
+                    transport = new StreamableHTTPServerTransport({
+                        sessionIdGenerator: () => {
+                            const id = randomUUID();
+                            return id;
+                        },
+                        onsessioninitialized: (sid) => {
+                            transports[sid] = transport;
+                        },
+                    });
+                    transport.onclose = () => {
+                        const sid = transport.sessionId;
+                        if (sid && transports[sid]) {
+                            delete transports[sid];
+                        }
+                    };
+                    console.log('Creating server instance');
+                    console.log('Connecting transport to server');
+                    await server.connect(transport);
+                    await transport.handleRequest(req, res, req.body);
+                    return;
+                }
+                else {
+                    res.status(400).json({
+                        jsonrpc: '2.0',
+                        error: {
+                            code: -32000,
+                            message: 'Invalid or missing session ID',
+                        },
+                        id: null,
+                    });
+                    return;
+                }
+                await transport.handleRequest(req, res, req.body);
+            }
+            catch (error) {
+                if (!res.headersSent) {
+                    res.status(500).json({
+                        jsonrpc: '2.0',
+                        error: {
+                            code: -32603,
+                            message: 'Internal server error',
+                        },
+                        id: null,
+                    });
+                }
             }
-            else if (!sessionId &&
-                req.method === 'POST' &&
-                req.body &&
-                typeof req.body === 'object' &&
-                req.body.method === 'initialize') {
-                transport = new StreamableHTTPServerTransport({
-                    sessionIdGenerator: () => {
-                        const id = randomUUID();
-                        return id;
-                    },
-                    onsessioninitialized: (sid) => {
-                        transports[sid] = transport;
-                    },
+        });
+    }
+    // SSE endpoint - only if enabled  
+    if (CONFIG.endpoints.enableSseEndpoint) {
+        // Map to store SSE transports per session
+        const sseTransports = new Map();
+        app.get('/sse', async (req, res) => {
+            try {
+                const sessionId = req.query.sessionId || randomUUID();
+                console.log(`SSE connection established for session: ${sessionId}`);
+                const transport = new SSEServerTransport(`/messages?sessionId=${sessionId}`, res);
+                sseTransports.set(sessionId, transport);
+                res.on('close', () => {
+                    console.log(`SSE connection closed for session: ${sessionId}`);
+                    sseTransports.delete(sessionId);
+                    transport.close?.();
+                });
+                res.on('error', (error) => {
+                    console.error(`SSE error for session ${sessionId}:`, error);
+                    sseTransports.delete(sessionId);
+                    transport.close?.();
                 });
-                transport.onclose = () => {
-                    const sid = transport.sessionId;
-                    if (sid && transports[sid]) {
-                        delete transports[sid];
-                    }
-                };
-                console.log('Creating server instance');
-                console.log('Connecting transport to server');
                 await server.connect(transport);
-                await transport.handleRequest(req, res, req.body);
-                return;
             }
-            else {
-                res.status(400).json({
-                    jsonrpc: '2.0',
-                    error: {
-                        code: -32000,
-                        message: 'Invalid or missing session ID',
-                    },
-                    id: null,
-                });
+            catch (error) {
+                console.error('SSE endpoint error:', error);
+                res.status(500).json({ error: 'Failed to establish SSE connection' });
+            }
+        });
+        app.post('/messages', async (req, res) => {
+            try {
+                const sessionId = req.query.sessionId;
+                // First try exact match, then fallback to any available transport
+                let transport = sessionId ? sseTransports.get(sessionId) : null;
+                if (!transport) {
+                    transport = Array.from(sseTransports.values())[0];
+                }
+                if (transport) {
+                    console.log(`Routing message from session ${sessionId} to transport`);
+                    await transport.handlePostMessage(req, res, req.body);
+                }
+                else {
+                    console.error('No SSE transport available. Active sessions:', Array.from(sseTransports.keys()));
+                    res.status(503).json({
+                        error: 'SSE transport not available. Connect to /sse first.',
+                        sessionId: sessionId,
+                        activeSessions: Array.from(sseTransports.keys())
+                    });
+                }
+            }
+            catch (error) {
+                console.error('Messages endpoint error:', error);
+                res.status(500).json({ error: 'Failed to handle message' });
+            }
+        });
+    }
+    // Firecrawl-compatible REST API - only if enabled.
+    // Implements the subset of the Firecrawl v2 API that agent frameworks use
+    // for page extraction (the firecrawl SDK's `scrape`). Point FIRECRAWL_API_URL
+    // at this server and it behaves like a self-hosted Firecrawl instance.
+    if (CONFIG.endpoints.enableFirecrawlApi) {
+        const requireFirecrawlAuth = (req, res) => {
+            if (isFirecrawlRequestAuthorized(req.headers.authorization, CONFIG.endpoints.firecrawlApiKey)) {
+                return true;
+            }
+            res.status(401).json({ success: false, error: 'Unauthorized: invalid or missing API key' });
+            return false;
+        };
+        const handleFirecrawlScrape = async (req, res) => {
+            if (!requireFirecrawlAuth(req, res))
+                return;
+            const parsed = parseFirecrawlScrapeRequest(req.body);
+            if (!parsed.ok || !parsed.request) {
+                res.status(400).json({ success: false, error: parsed.error });
                 return;
             }
-            await transport.handleRequest(req, res, req.body);
-        }
-        catch (error) {
-            if (!res.headersSent) {
+            const { url, formats, onlyMainContent } = parsed.request;
+            try {
+                const result = await scraper.scrape(url, onlyMainContent);
+                if (!result.success) {
+                    res.status(500).json({ success: false, error: result.error || 'Scrape failed' });
+                    return;
+                }
+                const meta = extractPageMetadata(result.html, url);
+                res.status(200).json({ success: true, data: buildFirecrawlDocument(result, formats, meta) });
+            }
+            catch (error) {
+                console.error('Firecrawl API scrape error:', error);
                 res.status(500).json({
-                    jsonrpc: '2.0',
-                    error: {
-                        code: -32603,
-                        message: 'Internal server error',
-                    },
-                    id: null,
+                    success: false,
+                    error: error instanceof Error ? error.message : String(error),
                 });
             }
-        }
-    });
-    const PORT = 3000;
+        };
+        app.post('/v2/scrape', handleFirecrawlScrape);
+        app.post('/v1/scrape', handleFirecrawlScrape);
+        // Web search is intentionally out of scope: this server renders pages, it
+        // is not a search engine. Return a clear error so clients can fall back
+        // to a dedicated search backend (DuckDuckGo, SearXNG, Brave, ...).
+        const handleFirecrawlSearch = (req, res) => {
+            if (!requireFirecrawlAuth(req, res))
+                return;
+            res.status(501).json({
+                success: false,
+                error: 'Search is not supported by Firecrawl Lite. Configure a dedicated search backend ' +
+                    '(e.g. in Hermes Agent set web.search_backend to ddgs, searxng, or brave-free) ' +
+                    'and use this server for extraction only.',
+            });
+        };
+        app.post('/v2/search', handleFirecrawlSearch);
+        app.post('/v1/search', handleFirecrawlSearch);
+    }
+    const PORT = CONFIG.endpoints.port;
     const appServer = app.listen(PORT, () => {
-        console.log(`MCP Streamable HTTP Server listening on port ${PORT}`);
+        console.log(`🚀 Firecrawl Lite MCP Server v${PACKAGE_VERSION} listening on port ${PORT}`);
+        console.log(`📊 Health endpoint: http://localhost:${PORT}/health`);
+        if (CONFIG.endpoints.enableHttpStreamableEndpoint) {
+            console.log(`🔌 MCP endpoint: http://localhost:${PORT}/mcp`);
+        }
+        if (CONFIG.endpoints.enableSseEndpoint) {
+            console.log(`📡 SSE endpoint: http://localhost:${PORT}/sse`);
+            console.log(`💬 Messages endpoint: http://localhost:${PORT}/messages`);
+        }
+        if (CONFIG.endpoints.enableFirecrawlApi) {
+            console.log(`🔥 Firecrawl-compatible API: http://localhost:${PORT}/v2/scrape${CONFIG.endpoints.firecrawlApiKey ? ' (bearer auth enabled)' : ' (no auth)'}`);
+        }
     });
     process.on('SIGINT', async () => {
         console.log('Shutting down server...');
+        // Close MCP transports
         for (const sessionId in transports) {
             try {
-                console.log(`Closing transport for session ${sessionId}`);
+                console.log(`Closing MCP transport for session ${sessionId}`);
                 await transports[sessionId].close();
                 delete transports[sessionId];
             }
             catch (error) {
-                console.error(`Error closing transport for session ${sessionId}:`, error);
+                console.error(`Error closing MCP transport for session ${sessionId}:`, error);
+            }
+        }
+        // Close SSE transport if active
+        if (sseTransport) {
+            try {
+                console.log('Closing SSE transport');
+                await sseTransport.close();
+                sseTransport = null;
+            }
+            catch (error) {
+                console.error('Error closing SSE transport:', error);
             }
         }
         appServer.close(() => {
@@ -883,8 +874,20 @@ async function runHTTPStreamableServer() {
         });
     });
 }
-// Server startup - standalone MCP server
-runLocalServer().catch((error) => {
-    console.error('Fatal error running server:', error);
-    process.exit(1);
-});
+// Server startup - conditional based on enabled endpoints
+if (CONFIG.endpoints.enableHttpStreamableEndpoint ||
+    CONFIG.endpoints.enableSseEndpoint ||
+    CONFIG.endpoints.enableFirecrawlApi) {
+    console.error('Starting HTTP server...');
+    runHTTPStreamableServer().catch((error) => {
+        console.error('Fatal error running HTTP server:', error);
+        process.exit(1);
+    });
+}
+else {
+    console.error('Starting stdio MCP Server...');
+    runLocalServer().catch((error) => {
+        console.error('Fatal error running server:', error);
+        process.exit(1);
+    });
+}