@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/server/routes/map.js

@@ -0,0 +1,116 @@
+/**
+ * First-class /v1/map endpoint
+ *
+ * Native WebPeel URL discovery API — returns our standard {success, links, count} format.
+ * Distinct from the Firecrawl-compatible compat.ts /v1/map route.
+ *
+ * POST /v1/map — Discover all URLs on a domain
+ */
+import { Router } from 'express';
+import '../types.js'; // Augments Express.Request with requestId
+import { mapDomain } from '../../core/map.js';
+import { validateUrlForSSRF, SSRFError } from '../middleware/url-validator.js';
+export function createMapRouter() {
+    const router = Router();
+    /**
+     * POST /v1/map
+     *
+     * Discover all URLs on a domain by combining sitemap discovery and link crawling.
+     *
+     * Body:
+     *   url             {string}   Required. Starting URL / domain root.
+     *   search          {string}   Optional. Filter URLs by relevance to this query.
+     *   maxUrls         {number}   Max URLs to return (default: 5000).
+     *   includePatterns {string[]} Regex patterns — only include matching URLs.
+     *   excludePatterns {string[]} Regex patterns — exclude matching URLs.
+     *
+     * Response:
+     *   { success: true, links: [{url, title?, description?}], count: number }
+     */
+    router.post('/', async (req, res) => {
+        try {
+            const { url, search, maxUrls = 5000, includePatterns, excludePatterns, } = req.body ?? {};
+            // Validate URL
+            if (!url || typeof url !== 'string') {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Missing or invalid "url" parameter.',
+                        docs: 'https://webpeel.dev/docs/api-reference#map',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            try {
+                new URL(url);
+            }
+            catch {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Invalid URL format.',
+                        docs: 'https://webpeel.dev/docs/api-reference#map',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            // SECURITY: Validate URL to prevent SSRF
+            try {
+                validateUrlForSSRF(url);
+            }
+            catch (error) {
+                if (error instanceof SSRFError) {
+                    res.status(400).json({
+                        success: false,
+                        error: {
+                            type: 'blocked_url',
+                            message: 'Cannot map localhost, private networks, or non-HTTP URLs.',
+                            docs: 'https://webpeel.dev/docs/api-reference#map',
+                        },
+                        requestId: req.requestId,
+                    });
+                    return;
+                }
+                throw error;
+            }
+            const mapOptions = {
+                maxUrls,
+                search: typeof search === 'string' ? search : undefined,
+            };
+            if (Array.isArray(includePatterns) && includePatterns.length > 0) {
+                mapOptions.includePatterns = includePatterns;
+            }
+            if (Array.isArray(excludePatterns) && excludePatterns.length > 0) {
+                mapOptions.excludePatterns = excludePatterns;
+            }
+            const result = await mapDomain(url, mapOptions);
+            // Return native format: { url, title?, description? }
+            // mapDomain returns urls as string[], so wrap each in an object.
+            const links = result.urls.map((u) => ({ url: u }));
+            res.json({
+                success: true,
+                links,
+                count: links.length,
+                elapsed: result.elapsed,
+                requestId: req.requestId,
+            });
+        }
+        catch (error) {
+            console.error('POST /v1/map error:', error);
+            res.status(500).json({
+                success: false,
+                error: {
+                    type: 'internal_error',
+                    message: 'An unexpected error occurred.',
+                    docs: 'https://webpeel.dev/docs/api-reference#errors',
+                },
+                requestId: req.requestId,
+            });
+        }
+    });
+    return router;
+}

package/dist/server/routes/mcp.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Hosted MCP endpoint — POST /mcp, POST /v2/mcp, POST /:apiKey/v2/mcp
+ *
+ * Thin HTTP/SSE transport wrapper. All tool logic lives in the shared handler
+ * registry at src/mcp/handlers/. This file handles:
+ *   - Express routing and auth
+ *   - MCP Streamable HTTP transport setup
+ *   - Passing McpContext (accountId, pool) to handlers
+ */
+import { Router } from 'express';
+import type { AuthStore } from '../auth-store.js';
+import '../types.js';
+import type { Pool } from 'pg';
+export declare function createMcpRouter(_authStore?: AuthStore, pool?: Pool | null): Router;

package/dist/server/routes/mcp.js

@@ -0,0 +1,197 @@
+/**
+ * Hosted MCP endpoint — POST /mcp, POST /v2/mcp, POST /:apiKey/v2/mcp
+ *
+ * Thin HTTP/SSE transport wrapper. All tool logic lives in the shared handler
+ * registry at src/mcp/handlers/. This file handles:
+ *   - Express routing and auth
+ *   - MCP Streamable HTTP transport setup
+ *   - Passing McpContext (accountId, pool) to handlers
+ */
+import { Router } from 'express';
+import '../types.js'; // Augments Express.Request with requestId
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
+import { CallToolRequestSchema, ListToolsRequestSchema, ListPromptsRequestSchema, GetPromptRequestSchema, } from '@modelcontextprotocol/sdk/types.js';
+import { readFileSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { getHandler } from '../../mcp/handlers/index.js';
+import { toolDefinitions } from '../../mcp/handlers/definitions.js';
+// Read version from package.json
+let pkgVersion = '0.7.0';
+try {
+    const __filename = fileURLToPath(import.meta.url);
+    const __dirname = dirname(__filename);
+    const pkg = JSON.parse(readFileSync(join(__dirname, '..', '..', '..', 'package.json'), 'utf-8'));
+    pkgVersion = pkg.version;
+}
+catch { /* fallback */ }
+// ---------------------------------------------------------------------------
+// Helper
+// ---------------------------------------------------------------------------
+function safeStringify(obj) {
+    try {
+        return JSON.stringify(obj, null, 2);
+    }
+    catch {
+        return JSON.stringify({ error: 'serialization_error', message: 'Failed to serialize result' });
+    }
+}
+// ---------------------------------------------------------------------------
+// Create a fresh MCP server instance (stateless — one per request)
+// ---------------------------------------------------------------------------
+function createMcpServer(pool, req) {
+    const mcpServer = new Server({ name: 'webpeel', version: pkgVersion }, { capabilities: { tools: {}, prompts: {} } });
+    mcpServer.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: toolDefinitions }));
+    // MCP Prompts — pre-built templates for common web data tasks
+    const mcpPrompts = [
+        {
+            name: 'research',
+            description: 'Research a topic by searching the web and synthesizing findings from multiple sources.',
+            arguments: [{ name: 'topic', description: 'Topic to research', required: true }],
+        },
+        {
+            name: 'extract-product',
+            description: 'Extract structured product data (name, price, rating, reviews) from any e-commerce URL.',
+            arguments: [{ name: 'url', description: 'Product page URL (Amazon, Best Buy, etc.)', required: true }],
+        },
+        {
+            name: 'monitor-price',
+            description: 'Set up price monitoring for a product page. Tracks changes and reports diffs.',
+            arguments: [{ name: 'url', description: 'Product URL to monitor', required: true }],
+        },
+        {
+            name: 'summarize',
+            description: 'Fetch a web page and provide a concise summary of its content.',
+            arguments: [{ name: 'url', description: 'URL to summarize', required: true }],
+        },
+    ];
+    mcpServer.setRequestHandler(ListPromptsRequestSchema, async () => ({ prompts: mcpPrompts }));
+    mcpServer.setRequestHandler(GetPromptRequestSchema, async (request) => {
+        const { name, arguments: args } = request.params;
+        const promptMap = {
+            'research': (a) => ({ messages: [{ role: 'user', content: { type: 'text', text: `Research "${a.topic}" using webpeel_find with depth="deep", then summarize the key findings with sources.` } }] }),
+            'extract-product': (a) => ({ messages: [{ role: 'user', content: { type: 'text', text: `Use webpeel_extract on ${a.url} with fields=['name','price','rating','reviews','availability'] and return the structured data.` } }] }),
+            'monitor-price': (a) => ({ messages: [{ role: 'user', content: { type: 'text', text: `Use webpeel_monitor on ${a.url} with selector=".price" to track price changes. Report the current price and set up monitoring.` } }] }),
+            'summarize': (a) => ({ messages: [{ role: 'user', content: { type: 'text', text: `Use webpeel_read on ${a.url} with budget=500 to get a concise version, then summarize the key points in 3-4 sentences.` } }] }),
+        };
+        const handler = promptMap[name];
+        if (!handler)
+            throw new Error(`Prompt not found: ${name}`);
+        return handler((args || {}));
+    });
+    mcpServer.setRequestHandler(CallToolRequestSchema, async (request) => {
+        const { name, arguments: rawArgs } = request.params;
+        const args = (rawArgs ?? {});
+        // Build context: auth + pool for HTTP-specific features (webpeel_watch)
+        const accountId = req?.auth?.keyInfo?.accountId ||
+            req?.user?.userId ||
+            'anonymous';
+        const ctx = { accountId, pool: pool ?? undefined };
+        try {
+            const handler = getHandler(name);
+            if (!handler)
+                throw new Error(`Unknown tool: ${name}`);
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            return (await handler(args, ctx));
+        }
+        catch (error) {
+            const err = error;
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: safeStringify({ error: err.name || 'Error', message: err.message || 'Unknown error' }),
+                    },
+                ],
+                isError: true,
+            };
+        }
+    });
+    return mcpServer;
+}
+// ---------------------------------------------------------------------------
+// Shared MCP request handler
+// ---------------------------------------------------------------------------
+async function handleMcpPost(req, res, pool) {
+    // Allow unauthenticated access to discovery methods (initialize, tools/list)
+    // so MCP marketplaces (Smithery, Glama) and clients can discover our tools.
+    // Actual tool execution (tools/call) still works without auth but uses free-tier limits.
+    const mcpAuthId = req.auth?.keyInfo?.accountId || req.user?.userId;
+    const body = req.body;
+    const method = body?.method || (Array.isArray(body) ? body[0]?.method : undefined);
+    const isDiscovery = method === 'initialize' || method === 'tools/list' || method === 'notifications/initialized';
+    // Only block if no auth AND it's not a discovery method
+    if (!mcpAuthId && !isDiscovery) {
+        // Still allow tools/call without auth — it'll use anonymous rate limits
+        // This enables free-tier MCP usage without signup
+    }
+    try {
+        const mcpServer = createMcpServer(pool, req);
+        const transport = new StreamableHTTPServerTransport({
+            sessionIdGenerator: undefined, // stateless
+        });
+        await mcpServer.connect(transport);
+        await transport.handleRequest(
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        req, res, req.body);
+        transport.close().catch(() => { });
+        mcpServer.close().catch(() => { });
+    }
+    catch (error) {
+        console.error('MCP endpoint error:', error);
+        if (!res.headersSent) {
+            res.status(500).json({
+                success: false,
+                error: {
+                    type: 'internal_error',
+                    message: 'Internal error',
+                    docs: 'https://webpeel.dev/docs/errors#internal_error',
+                },
+                requestId: req.requestId,
+            });
+        }
+    }
+}
+function mcpMethodNotAllowed(req, res) {
+    res.status(405).json({ success: false, error: { type: 'method_not_allowed', message: 'Method not allowed. Use POST to send MCP JSON-RPC messages.', hint: 'Send a POST request with a JSON-RPC body', docs: 'https://webpeel.dev/docs/errors#method_not_allowed' }, requestId: req.requestId });
+}
+function mcpDeleteOk(_req, res) {
+    res.status(200).json({ ok: true });
+}
+// ---------------------------------------------------------------------------
+// Express router
+// ---------------------------------------------------------------------------
+export function createMcpRouter(_authStore, pool) {
+    const router = Router();
+    const boundHandler = (req, res) => handleMcpPost(req, res, pool);
+    // GET /.well-known/mcp/server-card.json — MCP server discovery (Smithery, Glama, etc.)
+    router.get('/.well-known/mcp/server-card.json', (_req, res) => {
+        res.json({
+            name: 'WebPeel',
+            description: 'The web data platform for AI agents — fetch, search, crawl, extract, monitor, screenshot. 55+ domain extractors, 65-98% token savings.',
+            version: '0.21.87',
+            tools_count: 7,
+            homepage: 'https://webpeel.dev',
+            documentation: 'https://webpeel.dev/docs/mcp',
+            mcp_endpoint: 'https://api.webpeel.dev/mcp',
+            authentication: { type: 'optional', description: 'Bearer token optional. Works without auth using free-tier limits.' },
+        });
+    });
+    // POST /mcp — legacy path
+    router.post('/mcp', boundHandler);
+    router.get('/mcp', mcpMethodNotAllowed);
+    router.delete('/mcp', mcpDeleteOk);
+    // POST /v2/mcp — canonical v2 path; auth via Authorization: Bearer <key> header
+    router.post('/v2/mcp', boundHandler);
+    router.get('/v2/mcp', mcpMethodNotAllowed);
+    router.delete('/v2/mcp', mcpDeleteOk);
+    // SECURITY: /:apiKey/v2/mcp — BLOCKED. API keys in URLs are insecure.
+    const mcpInsecureAuthHandler = (req, res) => {
+        res.status(400).json({ success: false, error: { type: 'insecure_auth', message: 'API keys in URLs are insecure.', hint: 'Use the Authorization header instead: Authorization: Bearer wp_your_key', docs: 'https://webpeel.dev/docs/api-reference#authentication' }, requestId: req.requestId });
+    };
+    router.post('/:apiKey/v2/mcp', mcpInsecureAuthHandler);
+    router.get('/:apiKey/v2/mcp', mcpInsecureAuthHandler);
+    router.delete('/:apiKey/v2/mcp', mcpInsecureAuthHandler);
+    return router;
+}

package/dist/server/routes/metrics.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Prometheus metrics endpoint for WebPeel.
+ *
+ * Exposed at GET /metrics (behind auth — admin only).
+ *
+ * Metrics:
+ * - webpeel_requests_total — counter by method, path, status
+ * - webpeel_request_duration_seconds — histogram by method, status
+ * - webpeel_active_requests — gauge of in-flight requests
+ * - webpeel_fetch_duration_seconds — histogram of peel() execution time
+ * - webpeel_fetch_method_total — counter by fetch method (simple/browser/stealth)
+ * - webpeel_errors_total — counter by error code
+ * - webpeel_queue_jobs — gauge of queued/active/failed jobs
+ * - webpeel_memory_usage_bytes — gauge from system monitor
+ */
+import { Router, Request, Response, NextFunction } from 'express';
+import client from 'prom-client';
+export declare const httpRequestsTotal: client.Counter<"path" | "status" | "method">;
+export declare const httpRequestDuration: client.Histogram<"status" | "method">;
+export declare const activeRequests: client.Gauge<string>;
+export declare const fetchDuration: client.Histogram<"method" | "success">;
+export declare const fetchMethodCounter: client.Counter<"method">;
+export declare const errorsTotal: client.Counter<"code">;
+export declare const queueJobs: client.Gauge<"state" | "queue">;
+export declare const memoryUsageBytes: client.Gauge<"type">;
+/**
+ * Express middleware that records request metrics.
+ * Add EARLY in the middleware chain (after request ID, before routes).
+ */
+export declare function metricsMiddleware(): (req: Request, res: Response, next: NextFunction) => void;
+/** Record a peel() operation result */
+export declare function recordFetch(method: string, durationMs: number, success: boolean): void;
+/** Record an error by code */
+export declare function recordError(code: string): void;
+/** Update memory gauge */
+export declare function updateMemoryMetrics(): void;
+export declare function createMetricsRouter(): Router;

package/dist/server/routes/metrics.js

@@ -0,0 +1,149 @@
+/**
+ * Prometheus metrics endpoint for WebPeel.
+ *
+ * Exposed at GET /metrics (behind auth — admin only).
+ *
+ * Metrics:
+ * - webpeel_requests_total — counter by method, path, status
+ * - webpeel_request_duration_seconds — histogram by method, status
+ * - webpeel_active_requests — gauge of in-flight requests
+ * - webpeel_fetch_duration_seconds — histogram of peel() execution time
+ * - webpeel_fetch_method_total — counter by fetch method (simple/browser/stealth)
+ * - webpeel_errors_total — counter by error code
+ * - webpeel_queue_jobs — gauge of queued/active/failed jobs
+ * - webpeel_memory_usage_bytes — gauge from system monitor
+ */
+import { Router } from 'express';
+import client from 'prom-client';
+// Create a custom registry (don't use default to avoid conflicts)
+const register = new client.Registry();
+// Default metrics (CPU, memory, event loop, etc.)
+client.collectDefaultMetrics({ register, prefix: 'webpeel_' });
+// ─── Custom metrics ──────────────────────────────────────────────────────────
+export const httpRequestsTotal = new client.Counter({
+    name: 'webpeel_http_requests_total',
+    help: 'Total HTTP requests',
+    labelNames: ['method', 'path', 'status'],
+    registers: [register],
+});
+export const httpRequestDuration = new client.Histogram({
+    name: 'webpeel_http_request_duration_seconds',
+    help: 'HTTP request duration in seconds',
+    labelNames: ['method', 'status'],
+    buckets: [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60],
+    registers: [register],
+});
+export const activeRequests = new client.Gauge({
+    name: 'webpeel_active_requests',
+    help: 'Number of active (in-flight) requests',
+    registers: [register],
+});
+export const fetchDuration = new client.Histogram({
+    name: 'webpeel_fetch_duration_seconds',
+    help: 'Duration of peel() fetch operations',
+    labelNames: ['method', 'success'],
+    buckets: [0.1, 0.5, 1, 2, 5, 10, 30, 60, 120],
+    registers: [register],
+});
+export const fetchMethodCounter = new client.Counter({
+    name: 'webpeel_fetch_method_total',
+    help: 'Fetch method used',
+    labelNames: ['method'],
+    registers: [register],
+});
+export const errorsTotal = new client.Counter({
+    name: 'webpeel_errors_total',
+    help: 'Total errors by error code',
+    labelNames: ['code'],
+    registers: [register],
+});
+export const queueJobs = new client.Gauge({
+    name: 'webpeel_queue_jobs',
+    help: 'Queue job counts',
+    labelNames: ['queue', 'state'],
+    registers: [register],
+});
+export const memoryUsageBytes = new client.Gauge({
+    name: 'webpeel_memory_usage_bytes',
+    help: 'Application memory usage in bytes',
+    labelNames: ['type'],
+    registers: [register],
+});
+// ─── Metrics collection middleware ───────────────────────────────────────────
+/**
+ * Express middleware that records request metrics.
+ * Add EARLY in the middleware chain (after request ID, before routes).
+ */
+export function metricsMiddleware() {
+    return (req, res, next) => {
+        const start = process.hrtime.bigint();
+        activeRequests.inc();
+        res.on('finish', () => {
+            activeRequests.dec();
+            const durationNs = Number(process.hrtime.bigint() - start);
+            const durationS = durationNs / 1e9;
+            // Normalize path to avoid high-cardinality labels
+            const normalizedPath = normalizePath(req.path);
+            httpRequestsTotal.inc({
+                method: req.method,
+                path: normalizedPath,
+                status: res.statusCode.toString(),
+            });
+            httpRequestDuration.observe({ method: req.method, status: res.statusCode.toString() }, durationS);
+        });
+        next();
+    };
+}
+/** Normalize request path to reduce cardinality (UUIDs, IDs → :id) */
+function normalizePath(path) {
+    return path
+        .replace(/\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, '/:id')
+        .replace(/\/wp_[a-z]+_[a-f0-9]+/gi, '/:key')
+        .replace(/\/\d+/g, '/:id')
+        .replace(/\?.*$/, ''); // strip query params
+}
+// ─── Helper functions for recording metrics from other modules ───────────────
+/** Record a peel() operation result */
+export function recordFetch(method, durationMs, success) {
+    fetchDuration.observe({ method, success: success.toString() }, durationMs / 1000);
+    fetchMethodCounter.inc({ method });
+}
+/** Record an error by code */
+export function recordError(code) {
+    errorsTotal.inc({ code });
+}
+/** Update memory gauge */
+export function updateMemoryMetrics() {
+    const mem = process.memoryUsage();
+    memoryUsageBytes.set({ type: 'rss' }, mem.rss);
+    memoryUsageBytes.set({ type: 'heapUsed' }, mem.heapUsed);
+    memoryUsageBytes.set({ type: 'heapTotal' }, mem.heapTotal);
+    memoryUsageBytes.set({ type: 'external' }, mem.external);
+}
+// Update memory metrics every 15s
+setInterval(updateMemoryMetrics, 15_000);
+// ─── Router ──────────────────────────────────────────────────────────────────
+export function createMetricsRouter() {
+    const router = Router();
+    router.get('/metrics', async (req, res) => {
+        // Only allow admin tier to access metrics
+        if (req.auth?.tier !== 'admin') {
+            // Also allow unauthenticated access from localhost/K8s internal (Prometheus scraper)
+            const ip = req.ip || '';
+            const isInternal = ip === '127.0.0.1' ||
+                ip === '::1' ||
+                ip.startsWith('10.') ||
+                ip.startsWith('172.');
+            if (!isInternal) {
+                res.status(403).json({
+                    success: false,
+                    error: { type: 'forbidden', message: 'Metrics require admin access' },
+                });
+                return;
+            }
+        }
+        res.set('Content-Type', register.contentType);
+        res.end(await register.metrics());
+    });
+    return router;
+}

package/dist/server/routes/oauth.d.ts

@@ -0,0 +1,9 @@
+/**
+ * OAuth authentication routes
+ * Handles OAuth login from Auth.js (GitHub, Google)
+ */
+import { Router } from 'express';
+/**
+ * Create OAuth routes
+ */
+export declare function createOAuthRouter(): Router;