@ariangibson/firecrawl-lite-mcp-server 1.1.2 → 1.4.0

package/dist/scraper.js

@@ -0,0 +1,138 @@
+// The scraper: everything behind "give me this page as Markdown / as a PNG".
+//
+// Interface: createScraper(config, deps?) -> { scrape, screenshot }.
+// Behind it: URL validation, proxy and user-agent rotation, retries, the
+// browser session, scroll/settle heuristics, and HTML -> Markdown. Tests
+// exercise all of that through the same two methods callers use, by passing
+// a stub `launchBrowser` and fake `sleep`/`now`.
+import { withBrowserPage, withRetries, randomBetween, defaultBrowserDeps, } from './browser.js';
+import { createRotator } from './config.js';
+import { htmlToMarkdown, htmlToText } from './htmlToMarkdown.js';
+import { DEFAULT_USER_AGENT, isValidUrl, sanitizeUrl } from './utils.js';
+const INVALID_URL_ERROR = 'Invalid URL format. Only HTTP and HTTPS URLs are allowed.';
+function emptyScrape(url, error) {
+    return { url, title: '', content: '', markdown: '', html: '', success: false, error };
+}
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+// Poll until the rendered text stops growing (or maxWaitMs elapses), so pages
+// that inject content after load — setTimeout, XHR/AJAX, lazy hydration — are
+// captured. Fast/static pages return after the first couple of stable reads.
+export async function waitForContentToSettle(page, maxWaitMs, deps) {
+    const intervalMs = 600;
+    const requiredStableReads = 2;
+    let lastLength = -1;
+    let stableReads = 0;
+    const start = deps.now();
+    while (deps.now() - start < maxWaitMs) {
+        let length = 0;
+        try {
+            length = await page.evaluate(() => document.body?.innerText?.length || 0);
+        }
+        catch {
+            break; // navigation/detached — stop waiting
+        }
+        if (length === lastLength) {
+            if (++stableReads >= requiredStableReads)
+                return;
+        }
+        else {
+            stableReads = 0;
+            lastLength = length;
+        }
+        await deps.sleep(intervalMs);
+    }
+}
+export function createScraper(config, overrides = {}) {
+    const deps = { ...defaultBrowserDeps, now: Date.now, ...overrides };
+    // Each browser session advances the proxy / user-agent rotation.
+    const nextProxy = createRotator(config.proxy.urls);
+    const nextUserAgent = createRotator(config.scraping.userAgents);
+    function nextSession(overrides = {}) {
+        return {
+            userAgent: nextUserAgent() ?? DEFAULT_USER_AGENT,
+            viewport: { width: config.scraping.viewportWidth, height: config.scraping.viewportHeight },
+            proxy: { url: nextProxy(), username: config.proxy.username, password: config.proxy.password },
+            delayMin: config.scraping.delayMin,
+            delayMax: config.scraping.delayMax,
+            ...overrides,
+        };
+    }
+    // Single scrape attempt using the next proxy/user agent in rotation
+    async function scrapeOnce(url, onlyMainContent) {
+        if (!isValidUrl(url))
+            return emptyScrape(url, INVALID_URL_ERROR);
+        const sanitizedUrl = sanitizeUrl(url);
+        try {
+            return await withBrowserPage(sanitizedUrl, nextSession(), async (page) => {
+                // Scroll to the bottom to trigger lazy-loaded / scroll-triggered content,
+                // then back up (also looks human-like).
+                await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
+                await deps.sleep(randomBetween(500, 1500));
+                await page.evaluate(() => window.scrollTo(0, 0));
+                // Wait for the DOM to stabilize. Adapts to AJAX/lazy content (a stable page
+                // exits in ~1-2s) while capping the wait for never-settling SPAs. Tunable
+                // via SCRAPE_SETTLE_MAX_MS for sites with long setTimeout-injected content.
+                await waitForContentToSettle(page, config.scraping.settleMaxMs, deps);
+                const title = await page.title();
+                // Get the fully rendered HTML, then derive clean Markdown and text from it.
+                const html = await page.content();
+                const markdown = htmlToMarkdown(html, { onlyMainContent, baseUrl: url });
+                const content = htmlToText(html, { onlyMainContent, baseUrl: url });
+                return { url, title, content, markdown, html, success: true };
+            }, deps);
+        }
+        catch (error) {
+            return emptyScrape(url, errorMessage(error));
+        }
+    }
+    // Single screenshot attempt using the next proxy/user agent in rotation
+    async function screenshotOnce(url, width, height, fullPage) {
+        if (!isValidUrl(url))
+            return { success: false, error: INVALID_URL_ERROR };
+        const sanitizedUrl = sanitizeUrl(url);
+        const session = nextSession({ viewport: { width, height }, logLabel: 'for screenshot' });
+        try {
+            return await withBrowserPage(sanitizedUrl, session, async (page) => {
+                // Simulate human scrolling behavior, then scroll back up slightly
+                await page.evaluate(() => {
+                    window.scrollBy(0, Math.floor(Math.random() * 300) + 100);
+                });
+                await deps.sleep(randomBetween(500, 1500));
+                await page.evaluate(() => {
+                    window.scrollBy(0, -Math.floor(Math.random() * 100) - 50);
+                });
+                // Final wait for any dynamic content
+                await deps.sleep(1000);
+                // Return as base64 for remote deployment compatibility
+                const screenshotBuffer = Buffer.from(await page.screenshot({ fullPage, type: 'png' }));
+                const base64Data = screenshotBuffer.toString('base64');
+                return {
+                    success: true,
+                    dataUrl: `data:image/png;base64,${base64Data}`,
+                    base64: base64Data,
+                    metadata: {
+                        format: 'png',
+                        sizeKB: Math.round(screenshotBuffer.length / 1024),
+                        dimensions: `${width}x${height}`,
+                        fullPage,
+                        timestamp: new Date(deps.now()).toISOString(),
+                        url: sanitizedUrl,
+                    },
+                };
+            }, deps);
+        }
+        catch (error) {
+            return { success: false, error: errorMessage(error) };
+        }
+    }
+    return {
+        scrape(url, onlyMainContent = true) {
+            return withRetries(() => scrapeOnce(url, onlyMainContent), { maxAttempts: config.retry.maxAttempts, label: 'Scrape', url }, (error) => emptyScrape(url, error));
+        },
+        screenshot(url, { width = 1920, height = 1080, fullPage = false } = {}) {
+            return withRetries(() => screenshotOnce(url, width, height, fullPage), { maxAttempts: config.retry.maxAttempts, label: 'Screenshot', url }, (error) => ({ success: false, error }));
+        },
+    };
+}

package/dist/utils.js

@@ -0,0 +1,124 @@
+// Pure, side-effect-free helpers shared by the server and unit tests.
+// Keeping these out of index.ts means they can be imported and tested
+// without booting the HTTP/MCP server.
+export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+// Default LLM request parameters (used when the matching env var is unset)
+export const DEFAULT_LLM_TEMPERATURE = 0.1;
+export const DEFAULT_LLM_MAX_TOKENS = 2000;
+// Input validation utilities
+export function isValidUrl(url) {
+    try {
+        const parsedUrl = new URL(url);
+        // Only allow http and https protocols
+        return ['http:', 'https:'].includes(parsedUrl.protocol);
+    }
+    catch {
+        return false;
+    }
+}
+export function sanitizeUrl(url) {
+    // Remove any potentially dangerous characters
+    return url.trim().replace(/[<>'"]/g, '');
+}
+export function validatePrompt(prompt) {
+    // Basic prompt validation - prevent extremely long prompts
+    return prompt.length > 0 && prompt.length < 10000;
+}
+// Parse proxy URL with range support (e.g. https://example.com:10001-10010)
+export function parseProxyUrls(proxyUrl) {
+    if (!proxyUrl)
+        return [];
+    // Check for port range syntax: https://example.com:10001-10010
+    const rangeMatch = proxyUrl.match(/^(https?:\/\/[^:]+):(\d+)-(\d+)$/);
+    if (rangeMatch) {
+        const [, baseUrl, startPort, endPort] = rangeMatch;
+        const start = parseInt(startPort, 10);
+        const end = parseInt(endPort, 10);
+        const proxies = [];
+        for (let port = start; port <= end; port++) {
+            proxies.push(`${baseUrl}:${port}`);
+        }
+        return proxies;
+    }
+    // Single proxy URL
+    return [proxyUrl];
+}
+// Parse user agent env value, which may be a JSON array or a single string
+export function parseUserAgents(userAgentEnv, defaultUserAgent = DEFAULT_USER_AGENT) {
+    if (!userAgentEnv)
+        return [defaultUserAgent];
+    // Try to parse as JSON array first
+    try {
+        const parsed = JSON.parse(userAgentEnv);
+        if (Array.isArray(parsed) && parsed.length > 0) {
+            const filtered = parsed.filter((ua) => typeof ua === 'string' && ua.trim().length > 0);
+            return filtered.length > 0 ? filtered : [defaultUserAgent];
+        }
+    }
+    catch {
+        // Not JSON, treat as single user agent
+    }
+    // Single user agent string
+    return [userAgentEnv.trim()];
+}
+// Read LLM settings from the environment, applying sensible defaults.
+// Optional tuning params (reasoning effort, top_p) are left undefined when
+// unset so they are omitted from the request rather than sent as null.
+export function parseLlmConfig(env = process.env) {
+    const maxTokens = Number(env.LLM_MAX_TOKENS);
+    const temperature = Number(env.LLM_TEMPERATURE);
+    const topP = Number(env.LLM_TOP_P);
+    return {
+        apiKey: env.LLM_API_KEY,
+        providerBaseUrl: env.LLM_PROVIDER_BASE_URL,
+        model: env.LLM_MODEL,
+        reasoningEffort: env.LLM_REASONING_EFFORT?.trim() || undefined,
+        maxTokens: env.LLM_MAX_TOKENS && Number.isFinite(maxTokens) && maxTokens > 0
+            ? maxTokens
+            : DEFAULT_LLM_MAX_TOKENS,
+        temperature: env.LLM_TEMPERATURE !== undefined && Number.isFinite(temperature)
+            ? temperature
+            : DEFAULT_LLM_TEMPERATURE,
+        topP: env.LLM_TOP_P !== undefined && Number.isFinite(topP) ? topP : undefined,
+    };
+}
+// Parse the JSON object an LLM returns, tolerating common formatting quirks:
+// markdown code fences (```json ... ```), surrounding prose, and leading/
+// trailing whitespace. Throws if no valid JSON can be recovered.
+export function parseLlmJson(text) {
+    const trimmed = text.trim();
+    // Strip a wrapping markdown code fence if present (```json ... ``` or ``` ... ```).
+    const fenceMatch = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i);
+    const unfenced = fenceMatch ? fenceMatch[1].trim() : trimmed;
+    try {
+        return JSON.parse(unfenced);
+    }
+    catch {
+        // Fall back to the first balanced {...} or [...] span in the text.
+        const start = unfenced.search(/[{[]/);
+        const lastObj = unfenced.lastIndexOf('}');
+        const lastArr = unfenced.lastIndexOf(']');
+        const end = Math.max(lastObj, lastArr);
+        if (start !== -1 && end > start) {
+            return JSON.parse(unfenced.slice(start, end + 1));
+        }
+        throw new Error('No valid JSON found in LLM response');
+    }
+}
+// Build the chat-completions request body, only including optional tuning
+// parameters when they have been explicitly configured.
+export function buildLlmRequestBody(model, messages, llm) {
+    const body = {
+        model,
+        messages,
+        temperature: llm.temperature,
+        max_tokens: llm.maxTokens,
+    };
+    if (llm.topP !== undefined) {
+        body.top_p = llm.topP;
+    }
+    if (llm.reasoningEffort) {
+        body.reasoning_effort = llm.reasoningEffort;
+    }
+    return body;
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@ariangibson/firecrawl-lite-mcp-server",
-  "version": "1.1.2",
-  "description": "Privacy-first, standalone MCP server for web scraping and data extraction using local browser automation and your own LLM API key",
+  "version": "1.4.0",
+  "description": "Privacy-first, single-process web scraping for AI agents: MCP server plus a Firecrawl-compatible REST API, powered by local browser automation and your own LLM key",
   "type": "module",
   "bin": {
     "firecrawl-lite-mcp-server": "dist/index.js"
@@ -13,26 +13,32 @@
     "build": "tsc",
     "start": "node dist/index.js",
     "dev": "tsc && node dist/index.js",
-    "lint": "tsc --noEmit"
+    "test": "tsx --test tests/*.test.ts",
+    "lint": "tsc --noEmit",
+    "postinstall": "node -e \"try { require('puppeteer').executablePath(); } catch (e) { console.log('Installing Chrome for Puppeteer...'); require('child_process').execSync('npx puppeteer browsers install chrome', {stdio: 'inherit'}); }\""
   },
   "license": "MIT",
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.17.3",
-    "@types/cheerio": "^0.22.35",
     "axios": "^1.11.0",
     "cheerio": "^1.1.2",
     "dotenv": "^16.4.7",
     "express": "^5.1.0",
     "puppeteer": "^24.17.1",
-    "ws": "^8.18.1"
+    "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2",
+    "turndown": "^7.2.4",
+    "turndown-plugin-gfm": "^1.0.2"
   },
   "devDependencies": {
     "@types/express": "^5.0.1",
-    "@types/node": "^20.10.5",
+    "@types/node": "^22.0.0",
+    "@types/turndown": "^5.0.6",
+    "tsx": "^4.19.2",
     "typescript": "^5.9.2"
   },
   "engines": {
-    "node": ">=18.0.0"
+    "node": ">=20.0.0"
   },
   "repository": {
     "type": "git",