@ariangibson/firecrawl-lite-mcp-server 1.1.2 → 1.4.0

package/dist/browser.js

@@ -0,0 +1,145 @@
+// Browser session management shared by scraping and screenshots.
+//
+// Owns everything that happens between "we have a URL" and "we have a
+// navigated, stealth-configured page": launch flags, proxy and user-agent
+// setup, anti-detection shims, pre-navigation delay, and guaranteed browser
+// cleanup. Callers supply only the work to do on the page.
+//
+// The browser itself is injected (`BrowserDeps.launchBrowser`) so the whole
+// flow can be exercised in tests with a stub; the default launches stealth
+// Puppeteer.
+// --- Default (real) dependencies -------------------------------------------
+// Puppeteer will be loaded dynamically to handle mixed module issues
+let puppeteer;
+// Initialize puppeteer modules and ensure Chrome is available
+export async function initializePuppeteer() {
+    if (puppeteer)
+        return;
+    const puppeteerExtra = await import('puppeteer-extra');
+    const stealthPlugin = await import('puppeteer-extra-plugin-stealth');
+    puppeteer = puppeteerExtra.default;
+    // Configure puppeteer-extra with stealth plugin
+    puppeteer.use(stealthPlugin.default());
+    // Check if Chrome is available, install if needed
+    try {
+        await puppeteer.executablePath();
+    }
+    catch {
+        console.error('Chrome not found, attempting to install...');
+        const { execSync } = await import('child_process');
+        try {
+            execSync('npx puppeteer browsers install chrome', { stdio: 'inherit' });
+            console.error('Chrome installation completed successfully');
+        }
+        catch {
+            console.error('Failed to install Chrome automatically. Please run: npx puppeteer browsers install chrome');
+            throw new Error('Chrome browser not found and automatic installation failed');
+        }
+    }
+}
+export async function launchPuppeteer(options) {
+    await initializePuppeteer();
+    return puppeteer.launch(options);
+}
+export function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+export const defaultBrowserDeps = { launchBrowser: launchPuppeteer, sleep };
+// Puppeteer launch flags with anti-detection tweaks.
+// SECURITY: --disable-web-security is deliberately absent; it is a major risk.
+export function buildLaunchArgs(userAgent, proxyUrl) {
+    const args = [
+        '--no-sandbox',
+        '--disable-setuid-sandbox',
+        '--disable-dev-shm-usage',
+        '--disable-accelerated-2d-canvas',
+        '--no-first-run',
+        '--no-zygote',
+        '--disable-gpu',
+        '--disable-features=VizDisplayCompositor',
+        '--disable-blink-features=AutomationControlled',
+        '--disable-extensions-except',
+        '--disable-plugins-discovery',
+        '--no-default-browser-check',
+        '--no-experiments',
+        '--disable-default-apps',
+        '--disable-sync',
+        '--disable-translate',
+        '--hide-scrollbars',
+        '--mute-audio',
+        '--no-pings',
+        '--no-session-id',
+        '--disable-background-timer-throttling',
+        '--disable-backgrounding-occluded-windows',
+        '--disable-renderer-backgrounding',
+        `--user-agent=${userAgent}`,
+    ];
+    if (proxyUrl)
+        args.push(`--proxy-server=${proxyUrl}`);
+    return args;
+}
+export function randomBetween(minMs, maxMs) {
+    return Math.floor(Math.random() * (maxMs - minMs)) + minMs;
+}
+// Launch a stealth browser, navigate to `url`, run `work` against the page,
+// and always close the browser afterwards. Errors from navigation or `work`
+// propagate to the caller.
+export async function withBrowserPage(url, options, work, deps = defaultBrowserDeps) {
+    const { userAgent, viewport, proxy, delayMin, delayMax } = options;
+    const logSuffix = options.logLabel ? ` ${options.logLabel}` : '';
+    const hasProxyAuth = !!(proxy.url && proxy.username && proxy.password);
+    if (proxy.url) {
+        // SECURITY: Never log proxy URLs that might contain credentials
+        console.error(hasProxyAuth ? `Using authenticated proxy${logSuffix}: [REDACTED]` : `Using proxy${logSuffix}: ${proxy.url}`);
+    }
+    const browser = await deps.launchBrowser({
+        headless: 'new', // Use new headless mode for better stealth
+        args: buildLaunchArgs(userAgent, proxy.url),
+    });
+    try {
+        const page = await browser.newPage();
+        await page.setUserAgent(userAgent);
+        await page.setViewport(viewport);
+        // Add common browser properties to avoid detection
+        await page.evaluateOnNewDocument(() => {
+            Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
+            Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] });
+            Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
+            window.chrome = { runtime: {} };
+        });
+        if (hasProxyAuth) {
+            await page.authenticate({ username: proxy.username, password: proxy.password });
+        }
+        await deps.sleep(randomBetween(delayMin, delayMax));
+        await page.goto(url, { waitUntil: 'networkidle2', timeout: 30000 });
+        // Simulate human-like behavior: pause 2-5s after load
+        await deps.sleep(randomBetween(2000, 5000));
+        return await work(page);
+    }
+    finally {
+        await browser.close();
+    }
+}
+// Run `attempt` up to `maxAttempts` times until it returns `success: true`.
+// Each attempt picks up the next proxy/user agent via the caller. A thrown
+// error or an unsuccessful result both count as a failed attempt. When every
+// attempt fails, `onFailure` builds the caller's failure result.
+export async function withRetries(attempt, options, onFailure) {
+    const { maxAttempts, label, url } = options;
+    let lastError = null;
+    for (let n = 1; n <= maxAttempts; n++) {
+        try {
+            const result = await attempt();
+            if (result.success)
+                return result;
+            lastError = result.error || `Unknown ${label.toLowerCase()} error`;
+        }
+        catch (error) {
+            lastError = error instanceof Error ? error.message : String(error);
+        }
+        if (n < maxAttempts) {
+            console.error(`${label} attempt ${n} failed for ${url}, trying next proxy...`);
+        }
+    }
+    return onFailure(`Failed after ${maxAttempts} attempts. Last error: ${lastError}`);
+}

package/dist/config.js

@@ -0,0 +1,67 @@
+// Server configuration, resolved once from the environment.
+//
+// This is the single answer to "what does the server think its settings are".
+// Every env var the server reads is parsed here, with its default, so the
+// rest of the code never touches process.env. Pure: pass any env record in.
+import { DEFAULT_USER_AGENT, parseProxyUrls, parseUserAgents, parseLlmConfig, } from './utils.js';
+export const DEFAULT_VIEWPORT_WIDTH = 1920;
+export const DEFAULT_VIEWPORT_HEIGHT = 1080;
+export const DEFAULT_SCRAPE_DELAY_MIN = 1000;
+export const DEFAULT_SCRAPE_DELAY_MAX = 3000;
+export const DEFAULT_BATCH_DELAY_MIN = 2000;
+export const DEFAULT_BATCH_DELAY_MAX = 5000;
+// Max time to wait for the DOM to stop changing after load. Early-exits when
+// stable, so this only caps pages that keep mutating. Bump it (e.g. 12000) for
+// sites that inject content via long setTimeouts.
+export const DEFAULT_SETTLE_MAX_MS = 3000;
+export const DEFAULT_RETRY_ATTEMPTS = 3;
+export const DEFAULT_PORT = 3000;
+function positiveNumber(raw, fallback) {
+    const n = Number(raw);
+    return raw !== undefined && raw !== '' && Number.isFinite(n) && n > 0 ? n : fallback;
+}
+function flag(raw) {
+    return raw === 'true';
+}
+export function loadConfig(env = process.env) {
+    return {
+        scraping: {
+            userAgents: parseUserAgents(env.SCRAPE_USER_AGENT || '', DEFAULT_USER_AGENT),
+            viewportWidth: positiveNumber(env.SCRAPE_VIEWPORT_WIDTH, DEFAULT_VIEWPORT_WIDTH),
+            viewportHeight: positiveNumber(env.SCRAPE_VIEWPORT_HEIGHT, DEFAULT_VIEWPORT_HEIGHT),
+            delayMin: positiveNumber(env.SCRAPE_DELAY_MIN, DEFAULT_SCRAPE_DELAY_MIN),
+            delayMax: positiveNumber(env.SCRAPE_DELAY_MAX, DEFAULT_SCRAPE_DELAY_MAX),
+            batchDelayMin: positiveNumber(env.SCRAPE_BATCH_DELAY_MIN, DEFAULT_BATCH_DELAY_MIN),
+            batchDelayMax: positiveNumber(env.SCRAPE_BATCH_DELAY_MAX, DEFAULT_BATCH_DELAY_MAX),
+            settleMaxMs: positiveNumber(env.SCRAPE_SETTLE_MAX_MS, DEFAULT_SETTLE_MAX_MS),
+        },
+        retry: {
+            maxAttempts: positiveNumber(env.FIRECRAWL_RETRY_MAX_ATTEMPTS, DEFAULT_RETRY_ATTEMPTS),
+        },
+        llm: parseLlmConfig(env),
+        proxy: {
+            urls: parseProxyUrls(env.PROXY_SERVER_URL || ''),
+            username: env.PROXY_SERVER_USERNAME,
+            password: env.PROXY_SERVER_PASSWORD,
+            proxyLlmApi: flag(env.PROXY_LLM_API),
+        },
+        endpoints: {
+            enableHttpStreamableEndpoint: flag(env.ENABLE_HTTP_STREAMABLE_ENDPOINT),
+            enableSseEndpoint: flag(env.ENABLE_SSE_ENDPOINT),
+            enableFirecrawlApi: flag(env.ENABLE_FIRECRAWL_API),
+            firecrawlApiKey: env.FIRECRAWL_API_KEY?.trim() || undefined,
+            port: positiveNumber(env.PORT, DEFAULT_PORT),
+        },
+    };
+}
+// Round-robin over a list. Returns `undefined` forever for an empty list.
+export function createRotator(items) {
+    let index = 0;
+    return () => {
+        if (items.length === 0)
+            return undefined;
+        const item = items[index];
+        index = (index + 1) % items.length;
+        return item;
+    };
+}

package/dist/firecrawlApi.js

@@ -0,0 +1,86 @@
+// Firecrawl-compatible REST API helpers.
+//
+// Agent frameworks such as Hermes Agent talk to a "self-hosted Firecrawl"
+// through the official firecrawl SDK, which only needs `POST /v2/scrape`
+// to return `{ success: true, data: Document }`. These pure helpers parse
+// that request and shape our scrape result into a Firecrawl Document, so
+// the server can pose as a Firecrawl instance without any Firecrawl
+// infrastructure. Kept free of side effects so they can be unit tested.
+import { isValidUrl } from './utils.js';
+const KNOWN_FORMATS = ['markdown', 'html', 'rawHtml', 'links', 'screenshot'];
+// Normalise the `formats` array. Firecrawl accepts both plain strings
+// (`"markdown"`) and object entries (`{ type: "markdown" }`); unknown formats
+// are ignored rather than rejected so newer SDKs keep working.
+function parseFormats(raw) {
+    if (!Array.isArray(raw))
+        return ['markdown'];
+    const formats = [];
+    for (const entry of raw) {
+        const type = typeof entry === 'string'
+            ? entry
+            : entry && typeof entry === 'object' && typeof entry.type === 'string'
+                ? entry.type
+                : undefined;
+        if (type && KNOWN_FORMATS.includes(type) && !formats.includes(type)) {
+            formats.push(type);
+        }
+    }
+    return formats.length > 0 ? formats : ['markdown'];
+}
+export function parseFirecrawlScrapeRequest(body) {
+    if (!body || typeof body !== 'object') {
+        return { ok: false, error: 'Request body must be a JSON object' };
+    }
+    const { url, formats, onlyMainContent } = body;
+    if (typeof url !== 'string' || url.trim().length === 0) {
+        return { ok: false, error: 'Missing required field: url' };
+    }
+    const trimmed = url.trim();
+    if (!isValidUrl(trimmed)) {
+        return { ok: false, error: 'Invalid URL. Only http and https URLs are supported.' };
+    }
+    return {
+        ok: true,
+        request: {
+            url: trimmed,
+            formats: parseFormats(formats),
+            onlyMainContent: onlyMainContent !== false,
+        },
+    };
+}
+// Shape a scrape result into a Firecrawl v2 Document. Only the requested
+// formats are populated (like Firecrawl) so large HTML isn't shipped to
+// clients that asked for markdown only.
+export function buildFirecrawlDocument(result, formats, extra = {}) {
+    const doc = {};
+    if (formats.includes('markdown'))
+        doc.markdown = result.markdown;
+    if (formats.includes('html'))
+        doc.html = result.html;
+    if (formats.includes('rawHtml'))
+        doc.rawHtml = result.html;
+    if (formats.includes('links'))
+        doc.links = extra.links ?? [];
+    const metadata = {
+        title: result.title,
+        sourceURL: result.url,
+        url: result.url,
+        statusCode: 200,
+    };
+    if (extra.description)
+        metadata.description = extra.description;
+    if (extra.language)
+        metadata.language = extra.language;
+    doc.metadata = metadata;
+    return doc;
+}
+// Optional bearer-token auth for the REST API. When no key is configured the
+// endpoint is open (matching self-hosted Firecrawl with USE_DB_AUTHENTICATION=false).
+export function isFirecrawlRequestAuthorized(authorizationHeader, expectedKey) {
+    if (!expectedKey)
+        return true;
+    if (!authorizationHeader)
+        return false;
+    const match = /^Bearer\s+(.+)$/i.exec(authorizationHeader.trim());
+    return !!match && match[1].trim() === expectedKey;
+}

package/dist/htmlToMarkdown.js

@@ -0,0 +1,263 @@
+// HTML cleaning and Markdown conversion.
+//
+// Clean-room implementation using only permissively-licensed libraries
+// (cheerio: MIT, turndown: MIT, turndown-plugin-gfm: MIT). Deliberately does
+// NOT copy code from the AGPL-licensed Firecrawl project — only well-known,
+// standard techniques.
+// Use cheerio's slim build: it excludes the undici-based `fromURL` helper
+// (which we don't use). The full build pulls undici, which references the
+// `File` global and crashes on Node < 20 at import time.
+import { load } from 'cheerio/slim';
+import TurndownService from 'turndown';
+import { gfm } from 'turndown-plugin-gfm';
+// Semantic containers that usually wrap the primary article/content.
+const MAIN_CONTENT_SELECTORS = [
+    'main',
+    'article',
+    '[role="main"]',
+    '#main',
+    '#content',
+    '.main-content',
+    '.post-content',
+    '.entry-content',
+    '.article-content',
+    '.article-body',
+];
+// Elements that never carry readable content — always removed.
+const STRIP_SELECTORS = [
+    'script',
+    'style',
+    'noscript',
+    'template',
+    'svg',
+    'iframe',
+    'head',
+    'meta',
+    'link',
+    'base',
+];
+// Structural / chrome elements removed when onlyMainContent is requested.
+// These are common, generic selectors (not a copied list).
+const NON_MAIN_SELECTORS = [
+    'header',
+    'footer',
+    'nav',
+    'aside',
+    'form',
+    '.header',
+    '.footer',
+    '.nav',
+    '.navbar',
+    '.navigation',
+    '.menu',
+    '.sidebar',
+    '.aside',
+    '.ad',
+    '.ads',
+    '.advert',
+    '.advertisement',
+    '.social',
+    '.share',
+    '.sharing',
+    '.newsletter',
+    '.subscribe',
+    '.cookie',
+    '.cookie-banner',
+    '.consent',
+    '.popup',
+    '.modal',
+    '.overlay',
+    '.breadcrumb',
+    '.breadcrumbs',
+    '.pagination',
+    '.related',
+    '.comments',
+    '#header',
+    '#footer',
+    '#nav',
+    '#sidebar',
+    '#comments',
+    '[role="banner"]',
+    '[role="navigation"]',
+    '[role="complementary"]',
+    '[role="contentinfo"]',
+    '[aria-hidden="true"]',
+];
+// Find the semantic container that holds the bulk of the page's text. Returns
+// the element only when it clearly dominates the body (avoids scoping to a tiny
+// or empty <main>), otherwise undefined so the caller falls back to chrome
+// stripping.
+function pickMainContainer($) {
+    const bodyLen = $('body').text().replace(/\s+/g, ' ').trim().length;
+    if (bodyLen === 0)
+        return undefined;
+    let best;
+    let bestLen = 0;
+    for (const selector of MAIN_CONTENT_SELECTORS) {
+        $(selector).each((_, el) => {
+            const len = $(el).text().replace(/\s+/g, ' ').trim().length;
+            if (len > bestLen) {
+                bestLen = len;
+                best = el;
+            }
+        });
+    }
+    // Require the container to hold a meaningful share of the page text.
+    if (best && bestLen >= 200 && bestLen / bodyLen >= 0.4) {
+        return best;
+    }
+    return undefined;
+}
+// Load HTML into cheerio and strip noise, optionally reduce to main content,
+// and resolve relative links/images to absolute URLs.
+function clean(html, options) {
+    const $ = load(html);
+    $(STRIP_SELECTORS.join(',')).remove();
+    // Recover lazy-loaded images: many sites put a placeholder in src and the
+    // real URL in a data-* attribute. Promote it before the base64 cull below.
+    $('img').each((_, el) => {
+        const $img = $(el);
+        const src = $img.attr('src');
+        const lazy = $img.attr('data-src') ||
+            $img.attr('data-original') ||
+            $img.attr('data-lazy-src') ||
+            $img.attr('data-lazy') ||
+            $img.attr('data-srcset')?.split(',')[0]?.trim().split(' ')[0];
+        if ((!src || src.startsWith('data:')) && lazy) {
+            $img.attr('src', lazy);
+        }
+    });
+    // Drop inline base64 / data-URI images — they bloat the output enormously
+    // and carry no value for an LLM reader.
+    $('img[src^="data:"]').remove();
+    $('img[srcset^="data:"]').removeAttr('srcset');
+    // Drop images with no usable source (icons/placeholders) so they don't leave
+    // stray "!" markers in the output.
+    $('img:not([src]), img[src=""]').remove();
+    // Flatten layout tables (no <th> header row, or role="presentation"). The GFM
+    // plugin only converts true data tables and leaves the rest as raw HTML, so
+    // we collapse non-data tables to plain divs to keep that HTML out of the
+    // Markdown. Genuine data tables (with <th>) are left for GFM to convert.
+    $('table').each((_, el) => {
+        const $t = $(el);
+        const isPresentation = ($t.attr('role') || '').toLowerCase() === 'presentation';
+        const hasHeader = $t.find('tr').first().find('th').length > 0;
+        if (isPresentation || !hasHeader) {
+            const rows = $t
+                .find('tr')
+                .map((_, r) => `<div>${$(r)
+                .find('td,th')
+                .map((_, c) => $(c).html() ?? '')
+                .get()
+                .join(' ')}</div>`)
+                .get()
+                .join('');
+            $t.replaceWith(rows || '');
+        }
+    });
+    if (options.onlyMainContent) {
+        // Prefer a clear semantic content container when one holds the bulk of the
+        // page text; otherwise fall back to stripping generic chrome from the body.
+        const main = pickMainContainer($);
+        if (main) {
+            const mainHtml = $.html(main);
+            $('body').empty().append(mainHtml);
+        }
+        $(NON_MAIN_SELECTORS.join(',')).remove();
+    }
+    if (options.baseUrl) {
+        $('a[href]').each((_, el) => {
+            const href = $(el).attr('href');
+            if (href) {
+                try {
+                    $(el).attr('href', new URL(href, options.baseUrl).href);
+                }
+                catch {
+                    /* leave as-is if it can't be resolved */
+                }
+            }
+        });
+        $('img[src]').each((_, el) => {
+            const src = $(el).attr('src');
+            if (src) {
+                try {
+                    $(el).attr('src', new URL(src, options.baseUrl).href);
+                }
+                catch {
+                    /* leave as-is */
+                }
+            }
+        });
+    }
+    return $;
+}
+function makeTurndown() {
+    const td = new TurndownService({
+        headingStyle: 'atx',
+        hr: '---',
+        bulletListMarker: '-',
+        codeBlockStyle: 'fenced',
+        emDelimiter: '_',
+    });
+    // GFM support: tables, strikethrough, task lists.
+    td.use(gfm);
+    // Drop anything that survived but carries no readable value.
+    td.remove(['script', 'style', 'noscript', 'iframe', 'form']);
+    return td;
+}
+// Tidy up common turndown artifacts: empty links/images, stray escapes, and
+// runs of blank lines.
+function tidyMarkdown(md) {
+    return md
+        .replace(/!\[[^\]]*\]\(\s*\)/g, '') // images with empty src
+        .replace(/\[\s*\]\([^)]*\)/g, '') // links with empty text
+        .replace(/\[([^\]]+)\]\(\s*\)/g, '$1') // links with empty href -> text
+        .replace(/^[ \t]*!+[ \t]*$/gm, '') // stray image markers left on their own line
+        .replace(/^#{1,6}[ \t]*$/gm, '') // empty headings
+        .replace(/[ \t]+$/gm, '') // trailing whitespace
+        .replace(/\n{3,}/g, '\n\n') // collapse blank lines
+        .trim();
+}
+// Convert HTML to Markdown. Returns trimmed Markdown text.
+export function htmlToMarkdown(html, options = {}) {
+    const $ = clean(html, options);
+    const body = $('body').html() ?? $.html() ?? '';
+    if (!body.trim())
+        return '';
+    const markdown = makeTurndown().turndown(body);
+    return tidyMarkdown(markdown);
+}
+// Extract readable plain text (used for the `content` field).
+export function htmlToText(html, options = {}) {
+    const $ = clean(html, options);
+    const text = ($('body').text() || $.root().text() || '')
+        .replace(/[ \t]+/g, ' ')
+        .replace(/ *\n */g, '\n')
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+    return text;
+}
+// Pull lightweight page metadata (description, language, absolute links) from
+// rendered HTML. Used to enrich Firecrawl-compatible API responses.
+export function extractPageMetadata(html, baseUrl) {
+    const $ = load(html);
+    const description = $('meta[name="description"]').attr('content')?.trim() ||
+        $('meta[property="og:description"]').attr('content')?.trim() ||
+        undefined;
+    const language = $('html').attr('lang')?.trim() || undefined;
+    const links = new Set();
+    $('a[href]').each((_, el) => {
+        const href = $(el).attr('href');
+        if (!href)
+            return;
+        try {
+            const abs = new URL(href, baseUrl).toString();
+            if (abs.startsWith('http://') || abs.startsWith('https://'))
+                links.add(abs);
+        }
+        catch {
+            // ignore unparseable hrefs
+        }
+    });
+    return { description, language, links: Array.from(links) };
+}