@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/server/routes/crawl.js

@@ -0,0 +1,287 @@
+/**
+ * First-class /v1/crawl endpoint
+ *
+ * Native WebPeel crawl API — returns our standard {success, data, requestId} format.
+ * Distinct from the Firecrawl-compatible compat.ts routes which return Firecrawl's format.
+ *
+ * POST /v1/crawl  — Start an async crawl job
+ * GET  /v1/crawl/:id — Poll crawl job status
+ */
+import { Router } from 'express';
+import '../types.js'; // Augments Express.Request with requestId
+import { crawl } from '../../core/crawler.js';
+import { validateUrlForSSRF, SSRFError } from '../middleware/url-validator.js';
+import crypto from 'crypto';
+export function createCrawlRouter(jobQueue) {
+    const router = Router();
+    /**
+     * POST /v1/crawl
+     *
+     * Start an async crawl job. Returns a job ID immediately; poll GET /v1/crawl/:id for status.
+     * With stream:true, keeps the connection open and sends SSE events per page.
+     *
+     * Body:
+     *   url            {string}   Required. Starting URL.
+     *   maxPages       {number}   Max pages to crawl (default: 10).
+     *   maxDepth       {number}   Max link depth (default: 2).
+     *   includePatterns {string[]} Regex patterns — only crawl matching URLs.
+     *   excludePatterns {string[]} Regex patterns — skip matching URLs.
+     *   formats        {string[]} Content formats: 'markdown' | 'text' (default: ['markdown']).
+     *   webhook        {object}   Optional webhook to POST results to when done.
+     *   stream         {boolean}  If true, respond with SSE events (start → progress → done).
+     */
+    router.post('/', async (req, res) => {
+        try {
+            const { url, maxPages = 10, maxDepth = 2, includePatterns = [], excludePatterns = [], webhook, stream, } = req.body ?? {};
+            // Validate URL
+            if (!url || typeof url !== 'string') {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Missing or invalid "url" parameter.',
+                        docs: 'https://webpeel.dev/docs/api-reference#crawl',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            try {
+                new URL(url);
+            }
+            catch {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Invalid URL format.',
+                        docs: 'https://webpeel.dev/docs/api-reference#crawl',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            // SECURITY: Validate URL to prevent SSRF
+            try {
+                validateUrlForSSRF(url);
+            }
+            catch (error) {
+                if (error instanceof SSRFError) {
+                    res.status(400).json({
+                        success: false,
+                        error: {
+                            type: 'blocked_url',
+                            message: 'Cannot crawl localhost, private networks, or non-HTTP URLs.',
+                            docs: 'https://webpeel.dev/docs/api-reference#crawl',
+                        },
+                        requestId: req.requestId,
+                    });
+                    return;
+                }
+                throw error;
+            }
+            const ownerId = req.auth?.keyInfo?.accountId;
+            // ── Streaming mode (SSE) — keep connection open ──────────────────────
+            if (stream === true) {
+                res.setHeader('Content-Type', 'text/event-stream');
+                res.setHeader('Cache-Control', 'no-cache');
+                res.setHeader('Connection', 'keep-alive');
+                res.setHeader('X-Accel-Buffering', 'no');
+                res.flushHeaders();
+                const jobId = crypto.randomUUID();
+                // Send start event (total unknown until crawl runs)
+                res.write(`event: start\ndata: ${JSON.stringify({ id: jobId, url, maxPages, requestId: req.requestId })}\n\n`);
+                const crawlOptions = {
+                    maxPages,
+                    maxDepth,
+                    tier: req.auth?.tier,
+                    onProgress: (progress) => {
+                        const total = progress.crawled + progress.queued;
+                        res.write(`event: progress\ndata: ${JSON.stringify({
+                            id: jobId,
+                            completed: progress.crawled,
+                            total,
+                            queued: progress.queued,
+                            currentUrl: progress.currentUrl,
+                        })}\n\n`);
+                    },
+                };
+                if (Array.isArray(includePatterns) && includePatterns.length > 0) {
+                    crawlOptions.includePatterns = includePatterns;
+                }
+                if (Array.isArray(excludePatterns) && excludePatterns.length > 0) {
+                    crawlOptions.excludePatterns = excludePatterns;
+                }
+                try {
+                    const results = await crawl(url, crawlOptions);
+                    const data = results.map(r => ({
+                        url: r.url,
+                        title: r.title,
+                        content: r.markdown,
+                        links: r.links,
+                        elapsed: r.elapsed,
+                    }));
+                    res.write(`event: done\ndata: ${JSON.stringify({
+                        id: jobId,
+                        total: results.length,
+                        completed: results.length,
+                        results: data,
+                        requestId: req.requestId,
+                    })}\n\n`);
+                    // Fire webhook if configured
+                    if (webhook) {
+                        Promise.resolve(jobQueue.createJob('crawl', webhook, ownerId)).then((job) => {
+                            jobQueue.updateJob(job.id, {
+                                status: 'completed',
+                                data,
+                                total: results.length,
+                                completed: results.length,
+                                creditsUsed: results.length,
+                            });
+                        }).catch(() => { });
+                    }
+                }
+                catch (error) {
+                    res.write(`event: error\ndata: ${JSON.stringify({
+                        id: jobId,
+                        message: error.message || 'Crawl failed',
+                        requestId: req.requestId,
+                    })}\n\n`);
+                }
+                res.end();
+                return;
+            }
+            const job = await jobQueue.createJob('crawl', webhook, ownerId);
+            // Start crawl in background
+            setImmediate(async () => {
+                try {
+                    jobQueue.updateJob(job.id, { status: 'processing' });
+                    const crawlOptions = {
+                        maxPages,
+                        maxDepth,
+                        tier: req.auth?.tier,
+                        onProgress: (progress) => {
+                            const total = progress.crawled + progress.queued;
+                            jobQueue.updateJob(job.id, {
+                                total,
+                                completed: progress.crawled,
+                                creditsUsed: progress.crawled,
+                            });
+                        },
+                    };
+                    if (Array.isArray(includePatterns) && includePatterns.length > 0) {
+                        crawlOptions.includePatterns = includePatterns;
+                    }
+                    if (Array.isArray(excludePatterns) && excludePatterns.length > 0) {
+                        crawlOptions.excludePatterns = excludePatterns;
+                    }
+                    const results = await crawl(url, crawlOptions);
+                    // Native format: our standard CrawlResult shape
+                    const data = results.map(r => ({
+                        url: r.url,
+                        title: r.title,
+                        content: r.markdown,
+                        links: r.links,
+                        elapsed: r.elapsed,
+                    }));
+                    jobQueue.updateJob(job.id, {
+                        status: 'completed',
+                        data,
+                        total: results.length,
+                        completed: results.length,
+                        creditsUsed: results.length,
+                    });
+                }
+                catch (error) {
+                    jobQueue.updateJob(job.id, {
+                        status: 'failed',
+                        error: error.message || 'Unknown error',
+                    });
+                }
+            });
+            res.status(202).json({
+                success: true,
+                data: {
+                    jobId: job.id,
+                    status: 'queued',
+                },
+                requestId: req.requestId,
+            });
+        }
+        catch (error) {
+            console.error('POST /v1/crawl error:', error);
+            res.status(500).json({
+                success: false,
+                error: {
+                    type: 'internal_error',
+                    message: 'An unexpected error occurred.',
+                    docs: 'https://webpeel.dev/docs/api-reference#errors',
+                },
+                requestId: req.requestId,
+            });
+        }
+    });
+    /**
+     * GET /v1/crawl/:id
+     *
+     * Poll crawl job status. Returns job progress and results when complete.
+     */
+    router.get('/:id', async (req, res) => {
+        try {
+            const id = req.params['id'];
+            const job = await jobQueue.getJob(id);
+            if (!job) {
+                res.status(404).json({
+                    success: false,
+                    error: {
+                        type: 'not_found',
+                        message: `Crawl job '${id}' not found or expired.`,
+                        docs: 'https://webpeel.dev/docs/api-reference#crawl',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            // SECURITY: Only the job owner can read results
+            const requestOwnerId = req.auth?.keyInfo?.accountId;
+            if (job.ownerId && requestOwnerId && job.ownerId !== requestOwnerId) {
+                res.status(404).json({
+                    success: false,
+                    error: {
+                        type: 'not_found',
+                        message: `Crawl job '${id}' not found or expired.`,
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            res.json({
+                success: true,
+                data: {
+                    jobId: job.id,
+                    status: job.status,
+                    completed: job.completed || 0,
+                    total: job.total || 0,
+                    creditsUsed: job.creditsUsed || 0,
+                    expiresAt: job.expiresAt,
+                    results: job.status === 'completed' ? (job.data || []) : undefined,
+                    error: job.status === 'failed' ? job.error : undefined,
+                },
+                requestId: req.requestId,
+            });
+        }
+        catch (error) {
+            console.error('GET /v1/crawl/:id error:', error);
+            res.status(500).json({
+                success: false,
+                error: {
+                    type: 'internal_error',
+                    message: 'An unexpected error occurred.',
+                    docs: 'https://webpeel.dev/docs/api-reference#errors',
+                },
+                requestId: req.requestId,
+            });
+        }
+    });
+    return router;
+}

package/dist/server/routes/deep-fetch.d.ts

@@ -0,0 +1,8 @@
+/**
+ * POST /v1/deep-fetch
+ *
+ * Deep web intelligence endpoint: search + fetch + synthesize + structure.
+ * Body: { query, count?, format?, maxChars? }
+ */
+import { Router } from 'express';
+export declare function createDeepFetchRouter(): Router;

package/dist/server/routes/deep-fetch.js

@@ -0,0 +1,57 @@
+/**
+ * POST /v1/deep-fetch
+ *
+ * Deep web intelligence endpoint: search + fetch + synthesize + structure.
+ * Body: { query, count?, format?, maxChars? }
+ */
+import { Router } from 'express';
+import { deepFetch } from '../../core/deep-fetch.js';
+export function createDeepFetchRouter() {
+    const router = Router();
+    router.post('/v1/deep-fetch', async (req, res) => {
+        // Deprecation notice — prefer /v1/search?depth=deep
+        res.setHeader('X-Deprecated', 'true');
+        res.setHeader('X-Deprecated-Use', '/v1/search?depth=deep');
+        // AUTH: require authentication (global middleware sets req.auth)
+        const dfAuthId = req.auth?.keyInfo?.accountId || req.user?.userId;
+        if (!dfAuthId) {
+            res.status(401).json({ success: false, error: { type: 'authentication_required', message: 'API key required. Get one at https://app.webpeel.dev/keys', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/errors#authentication_required' }, requestId: req.requestId });
+            return;
+        }
+        try {
+            const body = req.body;
+            const query = body.query;
+            if (!query || typeof query !== 'string' || !query.trim()) {
+                res.status(400).json({ success: false, error: { type: 'bad_request', message: 'Missing required field: query', hint: 'Include a "query" string in the request body', docs: 'https://webpeel.dev/docs/errors#bad_request' }, requestId: req.requestId });
+                return;
+            }
+            const options = {
+                query: query.trim(),
+                count: typeof body.count === 'number' ? Math.min(Math.max(body.count, 1), 10) : 5,
+                format: ['merged', 'structured', 'comparison'].includes(body.format)
+                    ? body.format
+                    : 'merged',
+                maxChars: typeof body.maxChars === 'number' ? body.maxChars : 32000,
+            };
+            const result = await deepFetch(options);
+            res.json({
+                ...result,
+                content: result.merged || '', // expose as `content` for consistency
+            });
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : 'Unknown error';
+            console.error('[deep-fetch] error:', message);
+            res.status(500).json({
+                success: false,
+                error: {
+                    type: 'internal_error',
+                    message,
+                    docs: 'https://webpeel.dev/docs/errors#internal_error',
+                },
+                requestId: req.requestId,
+            });
+        }
+    });
+    return router;
+}

package/dist/server/routes/deep-research.d.ts

@@ -0,0 +1,11 @@
+/**
+ * POST /v1/deep-research
+ *
+ * Multi-step search agent that turns one question into a comprehensive,
+ * cited research report. Supports SSE streaming.
+ *
+ * Auth: API key required (full or read scope)
+ * Body: DeepResearchRequest
+ */
+import { Router } from 'express';
+export declare function createDeepResearchRouter(): Router;

package/dist/server/routes/deep-research.js

@@ -0,0 +1,232 @@
+/**
+ * POST /v1/deep-research
+ *
+ * Multi-step search agent that turns one question into a comprehensive,
+ * cited research report. Supports SSE streaming.
+ *
+ * Auth: API key required (full or read scope)
+ * Body: DeepResearchRequest
+ */
+import { Router } from 'express';
+import { runDeepResearch, } from '../../core/deep-research.js';
+import { isFreeTierLimitError, getDefaultLLMConfig, } from '../../core/llm-provider.js';
+const VALID_PROVIDERS = [
+    'cloudflare',
+    'openai',
+    'anthropic',
+    'google',
+    'ollama',
+    'cerebras',
+];
+function parseIntOrUndefined(v) {
+    if (typeof v === 'number')
+        return v;
+    if (typeof v === 'string' && /^\d+$/.test(v))
+        return parseInt(v, 10);
+    return undefined;
+}
+export function createDeepResearchRouter() {
+    const router = Router();
+    router.post('/v1/deep-research', async (req, res) => {
+        // AUTH: require authentication
+        const authId = req.auth?.keyInfo?.accountId || req.user?.userId;
+        if (!authId) {
+            res.status(401).json({
+                success: false,
+                error: {
+                    type: 'authentication_required',
+                    message: 'API key required. Get one at https://app.webpeel.dev/keys',
+                    hint: 'Get a free API key at https://app.webpeel.dev/keys',
+                    docs: 'https://webpeel.dev/docs/errors#authentication_required',
+                },
+                requestId: req.requestId,
+            });
+            return;
+        }
+        const { question, llm: llmRaw, maxRounds, maxSources, stream, } = req.body;
+        // ── Validation ──────────────────────────────────────────────────────────
+        if (!question || typeof question !== 'string' || question.trim().length === 0) {
+            res.status(400).json({
+                success: false,
+                error: {
+                    type: 'invalid_request',
+                    message: 'Missing or invalid "question" parameter',
+                    hint: 'Include a non-empty "question" string in the request body',
+                    docs: 'https://webpeel.dev/docs/errors#invalid_request',
+                },
+                requestId: req.requestId,
+            });
+            return;
+        }
+        if (question.length > 5000) {
+            res.status(400).json({
+                success: false,
+                error: {
+                    type: 'invalid_request',
+                    message: '"question" too long (max 5000 characters)',
+                    hint: 'Keep the question under 5000 characters',
+                    docs: 'https://webpeel.dev/docs/errors#invalid_request',
+                },
+                requestId: req.requestId,
+            });
+            return;
+        }
+        // ── LLM config ──────────────────────────────────────────────────────────
+        let llmConfig;
+        if (llmRaw && typeof llmRaw === 'object') {
+            const llmObj = llmRaw;
+            const provider = llmObj.provider;
+            if (provider && !VALID_PROVIDERS.includes(provider)) {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: `Invalid "llm.provider". Must be one of: ${VALID_PROVIDERS.join(', ')}`,
+                        hint: `Supported providers: ${VALID_PROVIDERS.join(', ')}`,
+                        docs: 'https://webpeel.dev/docs/errors#invalid_request',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            llmConfig = {
+                provider: (provider || 'openai'),
+                apiKey: typeof llmObj.apiKey === 'string' ? llmObj.apiKey : undefined,
+                model: typeof llmObj.model === 'string' ? llmObj.model : undefined,
+                endpoint: typeof llmObj.endpoint === 'string' ? llmObj.endpoint : undefined,
+            };
+        }
+        // BYOK-only: require user to provide LLM config with an API key (or use Ollama)
+        if (!llmConfig) {
+            // Check for server-side fallback keys
+            const fallback = getDefaultLLMConfig();
+            if (fallback.provider === 'cloudflare') {
+                // No server-side keys configured and user didn't provide one
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'llm_required',
+                        message: 'Deep research requires an LLM provider. Pass your API key in the "llm" field.',
+                        hint: 'Example: { "llm": { "provider": "openai", "apiKey": "sk-..." } }. Supported: openai, anthropic, google, ollama.',
+                        docs: 'https://webpeel.dev/docs/deep-research',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            llmConfig = fallback;
+        }
+        const resolvedMaxRounds = Math.min(Math.max(parseIntOrUndefined(maxRounds) ?? 3, 1), 5);
+        const resolvedMaxSources = Math.min(Math.max(parseIntOrUndefined(maxSources) ?? 20, 5), 30);
+        const shouldStream = stream === true;
+        // ── Abort signal from request close ─────────────────────────────────────
+        const ac = new AbortController();
+        res.on('close', () => ac.abort());
+        // ── Streaming ───────────────────────────────────────────────────────────
+        if (shouldStream) {
+            res.setHeader('Content-Type', 'text/event-stream');
+            res.setHeader('Cache-Control', 'no-cache');
+            res.setHeader('Connection', 'keep-alive');
+            res.setHeader('X-Accel-Buffering', 'no');
+            res.flushHeaders();
+            const sendEvent = (data) => {
+                if (!res.writableEnded) {
+                    res.write(`data: ${JSON.stringify(data)}\n\n`);
+                }
+            };
+            const deepReq = {
+                question: question.trim(),
+                llm: llmConfig,
+                maxRounds: resolvedMaxRounds,
+                maxSources: resolvedMaxSources,
+                stream: true,
+                onChunk: (text) => {
+                    sendEvent({ type: 'chunk', text });
+                },
+                onProgress: (event) => {
+                    sendEvent({ eventType: 'progress', ...event });
+                },
+                signal: ac.signal,
+            };
+            try {
+                const result = await runDeepResearch(deepReq);
+                sendEvent({
+                    type: 'done',
+                    citations: result.citations,
+                    sourcesUsed: result.sourcesUsed,
+                    roundsCompleted: result.roundsCompleted,
+                    totalSearchQueries: result.totalSearchQueries,
+                    llmProvider: result.llmProvider,
+                    tokensUsed: result.tokensUsed,
+                    elapsed: result.elapsed,
+                });
+            }
+            catch (err) {
+                if (isFreeTierLimitError(err)) {
+                    sendEvent({
+                        type: 'error',
+                        error: err.error,
+                        message: err.message,
+                    });
+                }
+                else {
+                    const errMsg = err instanceof Error ? err.message : 'Unknown error';
+                    sendEvent({ type: 'error', message: errMsg });
+                }
+            }
+            res.end();
+            return;
+        }
+        // ── Non-streaming ───────────────────────────────────────────────────────
+        const deepReq = {
+            question: question.trim(),
+            llm: llmConfig,
+            maxRounds: resolvedMaxRounds,
+            maxSources: resolvedMaxSources,
+            stream: false,
+            signal: ac.signal,
+        };
+        try {
+            const result = await runDeepResearch(deepReq);
+            res.json({
+                success: true,
+                report: result.report,
+                citations: result.citations,
+                sourcesUsed: result.sourcesUsed,
+                roundsCompleted: result.roundsCompleted,
+                totalSearchQueries: result.totalSearchQueries,
+                llmProvider: result.llmProvider,
+                tokensUsed: result.tokensUsed,
+                elapsed: result.elapsed,
+            });
+        }
+        catch (err) {
+            if (isFreeTierLimitError(err)) {
+                res.status(429).json({
+                    success: false,
+                    error: {
+                        type: err.error,
+                        message: err.message,
+                        hint: 'Provide your own API key in the "llm" config object for unlimited use.',
+                        docs: 'https://webpeel.dev/docs/deep-research#free-tier',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            const errMsg = err instanceof Error ? err.message : 'Unknown error';
+            console.error('[deep-research] Error:', errMsg);
+            res.status(500).json({
+                success: false,
+                error: {
+                    type: 'deep_research_failed',
+                    message: 'Deep research failed. Please try again.',
+                    hint: errMsg,
+                    docs: 'https://webpeel.dev/docs/errors#deep_research_failed',
+                },
+                requestId: req.requestId,
+            });
+        }
+    });
+    return router;
+}

package/dist/server/routes/demo.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Demo endpoint — GET /v1/demo?url=<encoded_url>
+ *
+ * Unauthenticated endpoint for the WebPeel landing page hero demo.
+ * Returns a truncated fetch result for allowed domains only.
+ *
+ * Security:
+ * - Domain allowlist (no arbitrary URLs)
+ * - Separate rate limiter: 3 req/min, 30 req/day per IP
+ * - SSRF validation via validateUrl()
+ * - HTTP-only fetch (no Puppeteer/browser rendering)
+ * - 5s timeout, 3000 char content truncation
+ * - CORS: only webpeel.dev + localhost
+ * - In-memory cache: 10 min per URL
+ */
+import { Router } from 'express';
+import { RateLimiter } from '../middleware/rate-limit.js';
+export interface DemoRouterOptions {
+    /** Inject custom per-minute rate limiter (useful for testing) */
+    perMinute?: RateLimiter;
+    /** Inject custom per-day rate limiter (useful for testing) */
+    perDay?: RateLimiter;
+}
+export declare function createDemoRouter(options?: DemoRouterOptions): Router;