@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/server/routes/playground.js

@@ -0,0 +1,283 @@
+/**
+ * Playground endpoint — GET /v1/playground?url=<encoded_url>
+ *                       GET /v1/playground/search?q=<query>
+ *
+ * Unauthenticated endpoints for the WebPeel playground page.
+ * Lets visitors try the product without signing up.
+ *
+ * Security:
+ * - CORS-locked to webpeel.dev and localhost
+ * - IP-based rate limit: 10 requests per 15 minutes (shared across /fetch and /search)
+ * - Simple HTTP-only fetch (no browser rendering)
+ * - 5-second timeout
+ * - Content truncated to 5,000 chars
+ * - No screenshots
+ */
+import { Router } from 'express';
+import { peel } from '../../index.js';
+import { getBestSearchProvider } from '../../core/search-provider.js';
+import { createLogger } from '../logger.js';
+const log = createLogger('playground');
+// ── IP-based rate limiter ─────────────────────────────────────────────────────
+const MAX_PER_WINDOW = 10;
+const WINDOW_MS = 15 * 60 * 1000; // 15 minutes
+const ipHits = new Map();
+function checkRateLimit(ip) {
+    const now = Date.now();
+    const entry = ipHits.get(ip);
+    if (!entry || now > entry.resetAt) {
+        const resetAt = now + WINDOW_MS;
+        ipHits.set(ip, { count: 1, resetAt });
+        return { allowed: true, remaining: MAX_PER_WINDOW - 1, resetAt };
+    }
+    if (entry.count >= MAX_PER_WINDOW) {
+        return { allowed: false, remaining: 0, resetAt: entry.resetAt };
+    }
+    entry.count++;
+    return { allowed: true, remaining: MAX_PER_WINDOW - entry.count, resetAt: entry.resetAt };
+}
+// Clean up expired entries every 5 minutes
+setInterval(() => {
+    const now = Date.now();
+    for (const [ip, entry] of ipHits) {
+        if (now > entry.resetAt)
+            ipHits.delete(ip);
+    }
+}, 5 * 60 * 1000).unref();
+// ── CORS helper ───────────────────────────────────────────────────────────────
+function setCorsHeaders(req, res) {
+    const origin = req.headers.origin || '';
+    if (origin === 'https://webpeel.dev' ||
+        /^http:\/\/localhost(:\d+)?$/.test(origin) ||
+        /^http:\/\/127\.0\.0\.1(:\d+)?$/.test(origin)) {
+        res.setHeader('Access-Control-Allow-Origin', origin);
+        res.setHeader('Vary', 'Origin');
+    }
+    else if (!origin) {
+        // Allow curl and server-to-server (no Origin header)
+        res.setHeader('Access-Control-Allow-Origin', '*');
+    }
+    res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS');
+    res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
+}
+// ── IP extraction ─────────────────────────────────────────────────────────────
+function getClientIp(req) {
+    const forwardedFor = req.headers['x-forwarded-for'];
+    const firstForwardedIp = typeof forwardedFor === 'string'
+        ? forwardedFor.split(',')[0].trim()
+        : Array.isArray(forwardedFor) ? forwardedFor[0] : undefined;
+    return req.headers['cf-connecting-ip']
+        || firstForwardedIp
+        || req.headers['x-real-ip']
+        || req.ip
+        || req.socket?.remoteAddress
+        || 'unknown';
+}
+// ── CORS origin check ─────────────────────────────────────────────────────────
+function isAllowedOrigin(req) {
+    const origin = req.headers.origin || req.headers.referer || '';
+    if (!origin)
+        return true; // Allow curl / server-to-server (no Origin header)
+    return (origin === 'https://webpeel.dev' ||
+        origin.startsWith('https://webpeel.dev/') ||
+        /^http:\/\/localhost(:\d+)?/.test(origin) ||
+        /^http:\/\/127\.0\.0\.1(:\d+)?/.test(origin));
+}
+// ── Router ────────────────────────────────────────────────────────────────────
+const MAX_CONTENT_LENGTH = 5000;
+const FETCH_TIMEOUT_MS = 5000;
+const SIGN_UP_URL = 'https://app.webpeel.dev';
+export function createPlaygroundRouter() {
+    const router = Router();
+    // ── CORS preflight ─────────────────────────────────────────────────────────
+    router.options('/', (req, res) => {
+        setCorsHeaders(req, res);
+        res.status(204).end();
+    });
+    router.options('/search', (req, res) => {
+        setCorsHeaders(req, res);
+        res.status(204).end();
+    });
+    // ── GET /v1/playground?url=... ─────────────────────────────────────────────
+    router.get('/', async (req, res) => {
+        setCorsHeaders(req, res);
+        // CORS check
+        if (!isAllowedOrigin(req)) {
+            res.status(403).json({
+                success: false,
+                error: {
+                    type: 'cors_denied',
+                    message: 'Playground is only available from webpeel.dev',
+                    hint: `Sign up at ${SIGN_UP_URL} for full API access.`,
+                },
+            });
+            return;
+        }
+        const url = (req.query.url || '').trim();
+        if (!url) {
+            res.status(400).json({
+                success: false,
+                error: {
+                    type: 'missing_url',
+                    message: 'URL parameter is required',
+                    hint: 'GET /v1/playground?url=https://example.com',
+                },
+            });
+            return;
+        }
+        // Basic URL validation
+        let parsedUrl;
+        try {
+            parsedUrl = new URL(url);
+        }
+        catch {
+            res.status(400).json({
+                success: false,
+                error: {
+                    type: 'invalid_url',
+                    message: 'Invalid URL format',
+                    hint: 'Ensure the URL is well-formed: https://example.com',
+                },
+            });
+            return;
+        }
+        if (!['http:', 'https:'].includes(parsedUrl.protocol)) {
+            res.status(400).json({
+                success: false,
+                error: {
+                    type: 'invalid_url',
+                    message: 'Only HTTP and HTTPS URLs are allowed',
+                },
+            });
+            return;
+        }
+        // Rate limit by IP
+        const ip = getClientIp(req);
+        const rl = checkRateLimit(ip);
+        res.setHeader('X-RateLimit-Limit', String(MAX_PER_WINDOW));
+        res.setHeader('X-RateLimit-Remaining', String(rl.remaining));
+        res.setHeader('X-RateLimit-Reset', String(Math.ceil(rl.resetAt / 1000)));
+        if (!rl.allowed) {
+            res.status(429).json({
+                success: false,
+                error: {
+                    type: 'rate_limited',
+                    message: 'Playground limit reached (10 requests per 15 minutes)',
+                    hint: `Sign up for a free API key for unlimited access: ${SIGN_UP_URL}`,
+                },
+                playground: true,
+            });
+            return;
+        }
+        try {
+            log.info('Playground fetch', { url, ip });
+            const startMs = Date.now();
+            const result = await peel(url, {
+                timeout: FETCH_TIMEOUT_MS,
+                render: false,
+                noEscalate: true,
+            });
+            const fullContent = result.content || '';
+            const content = fullContent.slice(0, MAX_CONTENT_LENGTH);
+            const truncated = fullContent.length > MAX_CONTENT_LENGTH;
+            res.json({
+                success: true,
+                url: result.url,
+                title: result.title,
+                content,
+                tokens: result.tokens,
+                method: result.method,
+                elapsed: Date.now() - startMs,
+                truncated,
+                ...(truncated && {
+                    upgrade: `Full content available with a free API key → ${SIGN_UP_URL}`,
+                }),
+                playground: true,
+                rateLimitRemaining: rl.remaining,
+            });
+        }
+        catch (err) {
+            log.warn('Playground fetch error', { url, error: err?.message });
+            res.status(502).json({
+                success: false,
+                error: {
+                    type: 'fetch_failed',
+                    message: err?.message || 'Failed to fetch URL',
+                    hint: 'Check that the URL is publicly accessible.',
+                },
+                playground: true,
+            });
+        }
+    });
+    // ── GET /v1/playground/search?q=... ───────────────────────────────────────
+    router.get('/search', async (req, res) => {
+        setCorsHeaders(req, res);
+        // CORS check
+        if (!isAllowedOrigin(req)) {
+            res.status(403).json({
+                success: false,
+                error: {
+                    type: 'cors_denied',
+                    message: 'Playground is only available from webpeel.dev',
+                },
+            });
+            return;
+        }
+        const q = (req.query.q || '').trim();
+        if (!q) {
+            res.status(400).json({
+                success: false,
+                error: {
+                    type: 'missing_query',
+                    message: 'Query parameter is required',
+                    hint: 'GET /v1/playground/search?q=your+query',
+                },
+            });
+            return;
+        }
+        // Rate limit by IP (shared counter with fetch)
+        const ip = getClientIp(req);
+        const rl = checkRateLimit(ip);
+        res.setHeader('X-RateLimit-Limit', String(MAX_PER_WINDOW));
+        res.setHeader('X-RateLimit-Remaining', String(rl.remaining));
+        res.setHeader('X-RateLimit-Reset', String(Math.ceil(rl.resetAt / 1000)));
+        if (!rl.allowed) {
+            res.status(429).json({
+                success: false,
+                error: {
+                    type: 'rate_limited',
+                    message: 'Playground limit reached (10 requests per 15 minutes)',
+                    hint: `Sign up for a free API key for unlimited access: ${SIGN_UP_URL}`,
+                },
+                playground: true,
+            });
+            return;
+        }
+        try {
+            log.info('Playground search', { q, ip });
+            const startMs = Date.now();
+            const { provider, apiKey } = getBestSearchProvider();
+            const results = await provider.searchWeb(q, { count: 5, apiKey });
+            res.json({
+                success: true,
+                query: q,
+                results,
+                elapsed: Date.now() - startMs,
+                playground: true,
+                rateLimitRemaining: rl.remaining,
+            });
+        }
+        catch (err) {
+            log.warn('Playground search error', { q, error: err?.message });
+            res.status(502).json({
+                success: false,
+                error: {
+                    type: 'search_failed',
+                    message: err?.message || 'Search failed',
+                },
+                playground: true,
+            });
+        }
+    });
+    return router;
+}

package/dist/server/routes/reader.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Zero-auth URL-prefix reader API — Jina Reader style
+ *
+ * GET /r/https://example.com  → returns markdown (no auth, no signup)
+ * GET /s/search+query         → searches web and returns fetched results
+ *
+ * Headers (Jina-compatible):
+ *   x-respond-with: markdown | text | html | screenshot
+ *   x-timeout: seconds (default 10, max 15)
+ *   x-target-selector: CSS selector to target
+ *   x-wait-for-selector: CSS selector to wait for (triggers browser rendering)
+ *   x-with-generated-alt: true to caption images
+ *   x-no-cache: true to bypass cache
+ *
+ * Rate limit: 20 requests per 15 minutes per IP (no auth required)
+ */
+import { Router } from 'express';
+export declare function createReaderRouter(): Router;

package/dist/server/routes/reader.js

@@ -0,0 +1,192 @@
+/**
+ * Zero-auth URL-prefix reader API — Jina Reader style
+ *
+ * GET /r/https://example.com  → returns markdown (no auth, no signup)
+ * GET /s/search+query         → searches web and returns fetched results
+ *
+ * Headers (Jina-compatible):
+ *   x-respond-with: markdown | text | html | screenshot
+ *   x-timeout: seconds (default 10, max 15)
+ *   x-target-selector: CSS selector to target
+ *   x-wait-for-selector: CSS selector to wait for (triggers browser rendering)
+ *   x-with-generated-alt: true to caption images
+ *   x-no-cache: true to bypass cache
+ *
+ * Rate limit: 20 requests per 15 minutes per IP (no auth required)
+ */
+import { Router } from 'express';
+import { peel } from '../../index.js';
+import { createLogger } from '../../core/logger.js';
+import { validateUrlForSSRF, SSRFError } from '../middleware/url-validator.js';
+import crypto from 'crypto';
+const log = createLogger('reader');
+// IP-based rate limiting (same approach as playground)
+const ipHits = new Map();
+const RATE_LIMIT = 20; // 20 requests per window
+const RATE_WINDOW = 15 * 60 * 1000; // 15 minutes
+function checkRateLimit(ip) {
+    const now = Date.now();
+    const entry = ipHits.get(ip);
+    if (!entry || entry.resetAt < now) {
+        ipHits.set(ip, { count: 1, resetAt: now + RATE_WINDOW });
+        return { allowed: true, remaining: RATE_LIMIT - 1 };
+    }
+    entry.count++;
+    if (entry.count > RATE_LIMIT) {
+        return { allowed: false, remaining: 0 };
+    }
+    return { allowed: true, remaining: RATE_LIMIT - entry.count };
+}
+export function createReaderRouter() {
+    const router = Router();
+    // GET /r/https://example.com — fetch any URL, return markdown
+    // Also supports /r/http://example.com
+    router.get('/r/*', async (req, res) => {
+        // Express wildcard: req.params[0] gives everything after /r/
+        const targetUrl = req.params[0] || req.url.replace(/^\/r\//, '');
+        if (!targetUrl || !/^https?:\/\//i.test(targetUrl)) {
+            return res.status(400).json({
+                success: false,
+                error: {
+                    type: 'invalid_url',
+                    message: 'Prepend /r/ to a full URL. Example: /r/https://example.com',
+                },
+            });
+        }
+        // SECURITY: SSRF validation — block private IPs, localhost, etc.
+        try {
+            validateUrlForSSRF(targetUrl);
+        }
+        catch (err) {
+            if (err instanceof SSRFError) {
+                return res.status(400).json({
+                    success: false,
+                    error: { type: 'ssrf_blocked', message: err.message },
+                });
+            }
+            throw err;
+        }
+        const ip = req.headers['x-forwarded-for']?.split(',')[0]?.trim() ||
+            req.ip ||
+            'unknown';
+        const { allowed, remaining } = checkRateLimit(ip);
+        res.setHeader('X-RateLimit-Remaining', remaining.toString());
+        if (!allowed) {
+            return res.status(429).json({
+                success: false,
+                error: {
+                    type: 'rate_limited',
+                    message: 'Rate limit exceeded. 20 requests per 15 minutes without auth.',
+                },
+            });
+        }
+        try {
+            // Parse request headers for options (Jina-compatible)
+            const format = req.headers['x-respond-with'] || 'markdown';
+            const timeoutSec = parseInt(req.headers['x-timeout'] || '10', 10);
+            const timeout = Math.min(timeoutSec * 1000, 15000); // cap at 15s
+            const targetSelector = req.headers['x-target-selector'];
+            const waitForSelector = req.headers['x-wait-for-selector'];
+            const withCaptions = req.headers['x-with-generated-alt'] === 'true';
+            const result = await peel(targetUrl, {
+                timeout,
+                render: !!waitForSelector || !!targetSelector,
+                noEscalate: !waitForSelector, // no browser escalation unless explicitly needed
+                captionImages: withCaptions,
+                selector: targetSelector,
+                waitSelector: waitForSelector,
+            });
+            // Cache-Control: this endpoint is public and heavily cacheable.
+            // Cloudflare edge caches for 2 min; serves stale for up to 10 min while revalidating.
+            res.setHeader('Cache-Control', 'public, s-maxage=120, stale-while-revalidate=600');
+            // Vary on Accept so different content-type representations are cached separately.
+            res.setHeader('Vary', 'Accept');
+            // Return based on format
+            const responseFormat = format.toLowerCase();
+            if (responseFormat === 'text') {
+                res.setHeader('Content-Type', 'text/plain; charset=utf-8');
+                return res.send(result.content || '');
+            }
+            else if (responseFormat === 'html') {
+                res.setHeader('Content-Type', 'text/html; charset=utf-8');
+                return res.send(result.html || result.rawHtml || '');
+            }
+            else if (responseFormat === 'screenshot') {
+                return res.json({
+                    success: true,
+                    screenshot: result.screenshot || null,
+                    url: targetUrl,
+                });
+            }
+            else {
+                // Default: markdown as plain text (like Jina)
+                res.setHeader('Content-Type', 'text/plain; charset=utf-8');
+                const header = `Title: ${result.title || ''}\nURL: ${targetUrl}\nTokens: ${result.tokens || 0}\n\n`;
+                return res.send(header + (result.content || ''));
+            }
+        }
+        catch (err) {
+            log.error('Reader error:', err.message);
+            return res.status(500).json({
+                success: false,
+                error: { type: 'fetch_failed', message: err.message },
+                requestId: crypto.randomUUID(),
+            });
+        }
+    });
+    // GET /s/query — search web and return fetched results
+    router.get('/s/*', async (req, res) => {
+        const query = decodeURIComponent(req.params[0] || '');
+        if (!query.trim()) {
+            return res.status(400).json({
+                success: false,
+                error: {
+                    type: 'missing_query',
+                    message: 'Prepend /s/ to your search query. Example: /s/stripe pricing plans',
+                },
+            });
+        }
+        const ip = req.headers['x-forwarded-for']?.split(',')[0]?.trim() ||
+            req.ip ||
+            'unknown';
+        const { allowed, remaining } = checkRateLimit(ip);
+        res.setHeader('X-RateLimit-Remaining', remaining.toString());
+        if (!allowed) {
+            return res.status(429).json({
+                success: false,
+                error: { type: 'rate_limited', message: 'Rate limit exceeded.' },
+            });
+        }
+        try {
+            const { getBestSearchProvider } = await import('../../core/search-provider.js');
+            const { provider, apiKey } = await getBestSearchProvider();
+            const results = await provider.searchWeb(query, { count: 5, apiKey });
+            // Fetch top 3 results (5K char limit each to keep responses manageable)
+            const fetched = await Promise.all(results.slice(0, 3).map(async (r) => {
+                try {
+                    const page = await peel(r.url, { timeout: 5000, noEscalate: true });
+                    return {
+                        title: r.title || page.title,
+                        url: r.url,
+                        content: (page.content || '').slice(0, 5000),
+                    };
+                }
+                catch {
+                    return { title: r.title, url: r.url, content: r.snippet || '' };
+                }
+            }));
+            res.setHeader('Content-Type', 'text/plain; charset=utf-8');
+            const output = fetched
+                .map((r, i) => `## Result ${i + 1}: ${r.title}\nURL: ${r.url}\n\n${r.content}`)
+                .join('\n\n---\n\n');
+            return res.send(`Search: ${query}\nResults: ${fetched.length}\n\n${output}`);
+        }
+        catch (err) {
+            return res.status(500).json({
+                success: false,
+                error: { type: 'search_failed', message: err.message },
+            });
+        }
+    });
+    return router;
+}

package/dist/server/routes/research.d.ts

@@ -0,0 +1,14 @@
+/**
+ * POST /v1/research
+ *
+ * Lightweight research endpoint that chains search → fetch → compile.
+ * Default: uses WebPeel's self-hosted LLM (Ollama on Hetzner) for synthesis.
+ * Override: users can pass their own LLM config (BYOK) via the `llm` body param.
+ *
+ * Auth: API key required (full or read scope)
+ * Body: ResearchRequest
+ */
+import { Router } from 'express';
+export declare function expandQuery(query: string): string[];
+export declare function extractKeyFacts(content: string, query: string, maxFacts?: number): string[];
+export declare function createResearchRouter(): Router;