@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/server/middleware/rate-limit.js

@@ -0,0 +1,270 @@
+/**
+ * Redis-backed rate limiting middleware.
+ *
+ * Uses `rate-limiter-flexible` for atomic Redis operations — works correctly
+ * across all 6 K8s API pods (shared state via Redis).
+ *
+ * Falls back to in-memory when REDIS_URL is not set (CLI/local dev).
+ *
+ * Features:
+ * - Per-key (API key or IP) rate limiting
+ * - Tier-based limits: free=50/hr, pro=100/hr, max=500/hr
+ * - Route-based cost weighting: crawl=5x, render=3x, batch/screenshot=2x
+ * - Per-IP rate limiting ON TOP of API key limits (abuse prevention)
+ * - Exempt paths for health/docs endpoints
+ */
+import '../types.js';
+// ─── Tier burst limits ───────────────────────────────────────────────────────
+const TIER_BURST_LIMITS = {
+    free: 50,
+    pro: 100,
+    max: 500,
+    admin: 999999,
+};
+/** Global per-IP limit regardless of API key (prevents shared-key abuse) */
+const GLOBAL_IP_LIMIT = 1000; // per hour
+// ─── Exempt paths ────────────────────────────────────────────────────────────
+const EXEMPT_PATHS = [
+    '/health',
+    '/ready',
+    '/openapi.json',
+    '/openapi.yaml',
+    '/docs',
+    '/v1/usage',
+    '/v1/me',
+    '/v1/keys',
+    '/v1/activity',
+    '/v1/stats',
+];
+// ─── RateLimiter class (Redis or in-memory) ──────────────────────────────────
+export class RateLimiter {
+    keyLimiter; // RateLimiterRedis or RateLimiterMemory
+    ipLimiter; // separate limiter for per-IP limits
+    windowMs;
+    initialized = false;
+    initPromise;
+    constructor(windowMs = 3_600_000) {
+        this.windowMs = windowMs;
+        this.initPromise = this.init();
+    }
+    async init() {
+        const duration = Math.floor(this.windowMs / 1000); // seconds
+        try {
+            // Try Redis first
+            const redisUrl = process.env.REDIS_URL;
+            if (redisUrl) {
+                // Dynamic import to avoid hard dependency
+                const [{ RateLimiterRedis }, IoRedisModule] = await Promise.all([
+                    import('rate-limiter-flexible'),
+                    import('ioredis'),
+                ]);
+                const IoRedis = IoRedisModule.default ?? IoRedisModule;
+                const parsed = new URL(redisUrl);
+                const redisClient = new IoRedis({
+                    host: parsed.hostname,
+                    port: parseInt(parsed.port || '6379', 10),
+                    password: process.env.REDIS_PASSWORD || undefined,
+                    db: parseInt(parsed.pathname?.slice(1) || '0', 10) || 0,
+                    enableOfflineQueue: false,
+                    maxRetriesPerRequest: 1,
+                    lazyConnect: true,
+                });
+                await redisClient.connect();
+                // Key-based limiter (per API key or unauthenticated IP)
+                this.keyLimiter = new RateLimiterRedis({
+                    storeClient: redisClient,
+                    keyPrefix: 'rl:key',
+                    points: 500, // max tier limit — actual check done per-tier
+                    duration,
+                    blockDuration: 0, // don't block, just check
+                });
+                // IP-based limiter (global per-IP cap, on top of key limits)
+                this.ipLimiter = new RateLimiterRedis({
+                    storeClient: redisClient,
+                    keyPrefix: 'rl:ip',
+                    points: GLOBAL_IP_LIMIT,
+                    duration,
+                    blockDuration: 0,
+                });
+                console.log('[rate-limit] Redis-backed rate limiting active (shared across pods)');
+                this.initialized = true;
+                return;
+            }
+        }
+        catch (err) {
+            console.warn('[rate-limit] Failed to init Redis rate limiter, falling back to in-memory:', err.message);
+        }
+        // Fallback: in-memory
+        const { RateLimiterMemory } = await import('rate-limiter-flexible');
+        this.keyLimiter = new RateLimiterMemory({
+            keyPrefix: 'rl:key',
+            points: 500,
+            duration: Math.floor(this.windowMs / 1000),
+        });
+        this.ipLimiter = new RateLimiterMemory({
+            keyPrefix: 'rl:ip',
+            points: GLOBAL_IP_LIMIT,
+            duration: Math.floor(this.windowMs / 1000),
+        });
+        console.log('[rate-limit] In-memory rate limiting active (single-pod only)');
+        this.initialized = true;
+    }
+    async waitForInit() {
+        if (!this.initialized)
+            await this.initPromise;
+    }
+    /**
+     * Check if request is allowed.
+     * Returns { allowed, remaining, retryAfter? }
+     */
+    async checkLimit(identifier, limit, cost = 1) {
+        await this.waitForInit();
+        try {
+            const res = await this.keyLimiter.consume(identifier, cost);
+            // rate-limiter-flexible tracks points consumed; we need to check against the per-tier limit
+            const consumed = res.consumedPoints;
+            if (consumed > limit) {
+                // Over the tier limit — reject
+                const msBeforeNext = res.msBeforeNext;
+                return {
+                    allowed: false,
+                    remaining: 0,
+                    retryAfter: Math.ceil(msBeforeNext / 1000),
+                };
+            }
+            return {
+                allowed: true,
+                remaining: Math.max(0, limit - consumed),
+            };
+        }
+        catch (rejRes) {
+            // RateLimiterRes when rate limited
+            if (rejRes && rejRes.msBeforeNext !== undefined) {
+                return {
+                    allowed: false,
+                    remaining: 0,
+                    retryAfter: Math.ceil(rejRes.msBeforeNext / 1000),
+                };
+            }
+            // Unexpected error — allow through (fail-open)
+            console.error('[rate-limit] Error checking rate limit:', rejRes);
+            return { allowed: true, remaining: limit };
+        }
+    }
+    /**
+     * Check per-IP limit (global cap regardless of API key).
+     */
+    async checkIpLimit(ip, cost = 1) {
+        await this.waitForInit();
+        try {
+            const res = await this.ipLimiter.consume(ip, cost);
+            return {
+                allowed: true,
+                remaining: Math.max(0, GLOBAL_IP_LIMIT - res.consumedPoints),
+            };
+        }
+        catch (rejRes) {
+            if (rejRes && rejRes.msBeforeNext !== undefined) {
+                return {
+                    allowed: false,
+                    remaining: 0,
+                    retryAfter: Math.ceil(rejRes.msBeforeNext / 1000),
+                };
+            }
+            return { allowed: true, remaining: GLOBAL_IP_LIMIT };
+        }
+    }
+    /**
+     * Backward compat: cleanup is no-op for Redis (TTL handles it).
+     * Kept for in-memory fallback and for app.ts which calls it on an interval.
+     */
+    cleanup() {
+        // No-op for Redis; RateLimiterMemory handles its own cleanup
+    }
+}
+// ─── Middleware ───────────────────────────────────────────────────────────────
+export function createRateLimitMiddleware(limiter) {
+    return async (req, res, next) => {
+        try {
+            // Skip rate limiting for exempt endpoints
+            if (EXEMPT_PATHS.some(p => req.path === p || req.path.startsWith(p + '/'))) {
+                return next();
+            }
+            // Resolve real client IP (Cloudflare → x-forwarded-for → x-real-ip → req.ip)
+            const forwardedFor = req.headers['x-forwarded-for'];
+            const firstForwardedIp = typeof forwardedFor === 'string'
+                ? forwardedFor.split(',')[0].trim()
+                : Array.isArray(forwardedFor) ? forwardedFor[0] : undefined;
+            const clientIp = req.headers['cf-connecting-ip']
+                || firstForwardedIp
+                || req.headers['x-real-ip']
+                || req.ip
+                || 'unknown';
+            // Key-based identifier: prefer API key, fall back to IP
+            const keyIdentifier = req.auth?.keyInfo?.key || `ip:${clientIp}`;
+            // Tier-based limit
+            const tier = req.auth?.tier || 'free';
+            const limit = TIER_BURST_LIMITS[tier] || 50;
+            // Route-based cost weighting
+            let cost = 1;
+            const path = req.path;
+            if (path.includes('/crawl') || path.includes('/map'))
+                cost = 5;
+            else if (path.includes('/batch'))
+                cost = 2;
+            else if (path.includes('/screenshot'))
+                cost = 2;
+            else if (req.query.render === 'true' || req.body?.render === true)
+                cost = 3;
+            // Check 1: Per-key rate limit
+            const keyResult = await limiter.checkLimit(keyIdentifier, limit, cost);
+            // Check 2: Per-IP rate limit (on top of key limit, prevents shared-key abuse)
+            const ipResult = await limiter.checkIpLimit(clientIp, cost);
+            // Use the more restrictive result
+            const allowed = keyResult.allowed && ipResult.allowed;
+            const remaining = Math.min(keyResult.remaining, ipResult.remaining);
+            const retryAfter = !allowed
+                ? Math.max(keyResult.retryAfter || 0, ipResult.retryAfter || 0)
+                : undefined;
+            // Set rate limit headers
+            const now = Date.now();
+            const resetTimestamp = Math.ceil((now + limiter.windowMs) / 1000);
+            res.setHeader('X-RateLimit-Limit', limit.toString());
+            res.setHeader('X-RateLimit-Remaining', Math.max(0, remaining).toString());
+            res.setHeader('X-RateLimit-Reset', resetTimestamp.toString());
+            if (req.auth?.tier) {
+                res.setHeader('X-WebPeel-Plan', req.auth.tier);
+            }
+            if (!allowed) {
+                const retryAfterSecs = retryAfter || 60;
+                res.setHeader('Retry-After', retryAfterSecs.toString());
+                const upgradeHint = tier === 'free'
+                    ? ' Upgrade to Pro ($9/mo) for 100/hr burst limit → https://webpeel.dev/pricing'
+                    : tier === 'pro'
+                        ? ' Upgrade to Max ($29/mo) for 500/hr burst limit → https://webpeel.dev/pricing'
+                        : '';
+                // Determine which limit was hit
+                const reason = !keyResult.allowed
+                    ? `Hourly rate limit exceeded (${limit} requests/hr on ${tier} plan)`
+                    : `Global IP rate limit exceeded (${GLOBAL_IP_LIMIT} requests/hr)`;
+                res.status(429).json({
+                    success: false,
+                    error: {
+                        type: 'rate_limited',
+                        message: `${reason}. Try again in ${retryAfterSecs}s.`,
+                        hint: `Retry after ${retryAfterSecs} seconds.${upgradeHint}`,
+                        docs: 'https://webpeel.dev/docs/errors#rate-limited',
+                    },
+                    metadata: { requestId: req.requestId },
+                });
+                return;
+            }
+            next();
+        }
+        catch (_error) {
+            // Fail-open: if rate limiter errors, allow the request through
+            console.error('[rate-limit] Middleware error, failing open:', _error);
+            next();
+        }
+    };
+}

package/dist/server/middleware/scope-guard.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Scope enforcement middleware for API key permission scoping.
+ *
+ * Keys have one of three scopes:
+ *   'full'       — all endpoints (default)
+ *   'read'       — read/fetch operations only
+ *   'restricted' — /v1/scrape only (for limited sharing)
+ *
+ * JWT-authenticated requests (dashboard sessions) bypass scope enforcement:
+ * req.keyScope is undefined for JWT requests, which are always allowed through.
+ */
+import { Request, Response, NextFunction } from 'express';
+import { KeyScope } from '../pg-auth-store.js';
+/**
+ * Middleware factory that enforces API key scope.
+ * Pass the set of scopes that are permitted to access the guarded route.
+ *
+ * @example
+ * // Only full-access keys may manage billing:
+ * router.post('/v1/billing', requireScope('full'), handler);
+ *
+ * // Read and full keys may scrape:
+ * app.use('/v1/scrape', requireScope('full', 'read', 'restricted'), scrapeRouter);
+ */
+export declare function requireScope(...allowedScopes: KeyScope[]): (req: Request, res: Response, next: NextFunction) => void;

package/dist/server/middleware/scope-guard.js

@@ -0,0 +1,45 @@
+/**
+ * Scope enforcement middleware for API key permission scoping.
+ *
+ * Keys have one of three scopes:
+ *   'full'       — all endpoints (default)
+ *   'read'       — read/fetch operations only
+ *   'restricted' — /v1/scrape only (for limited sharing)
+ *
+ * JWT-authenticated requests (dashboard sessions) bypass scope enforcement:
+ * req.keyScope is undefined for JWT requests, which are always allowed through.
+ */
+/**
+ * Middleware factory that enforces API key scope.
+ * Pass the set of scopes that are permitted to access the guarded route.
+ *
+ * @example
+ * // Only full-access keys may manage billing:
+ * router.post('/v1/billing', requireScope('full'), handler);
+ *
+ * // Read and full keys may scrape:
+ * app.use('/v1/scrape', requireScope('full', 'read', 'restricted'), scrapeRouter);
+ */
+export function requireScope(...allowedScopes) {
+    return (req, res, next) => {
+        // JWT sessions (req.keyScope === undefined) always pass through.
+        // Scope enforcement only applies to API key requests.
+        if (req.keyScope === undefined) {
+            return next();
+        }
+        if (!allowedScopes.includes(req.keyScope)) {
+            res.status(403).json({
+                success: false,
+                error: {
+                    type: 'insufficient_scope',
+                    message: `This API key has '${req.keyScope}' scope. This endpoint requires: ${allowedScopes.join(' or ')}.`,
+                    docs: 'https://webpeel.dev/docs/authentication#scopes',
+                    hint: 'Create a new API key with the required scope in your dashboard.',
+                },
+                requestId: req.requestId,
+            });
+            return;
+        }
+        next();
+    };
+}

package/dist/server/middleware/url-validator.d.ts

@@ -0,0 +1,15 @@
+/**
+ * URL validation middleware to prevent SSRF attacks
+ * Validates URLs BEFORE any network request is made
+ */
+/**
+ * Validate URL to prevent SSRF attacks
+ * Blocks localhost, private IPs, link-local addresses, and non-HTTP(S) protocols
+ */
+export declare function validateUrlForSSRF(urlString: string): void;
+/**
+ * SSRF Error class
+ */
+export declare class SSRFError extends Error {
+    constructor(message: string);
+}

package/dist/server/middleware/url-validator.js

@@ -0,0 +1,201 @@
+/**
+ * URL validation middleware to prevent SSRF attacks
+ * Validates URLs BEFORE any network request is made
+ */
+/**
+ * Validate URL to prevent SSRF attacks
+ * Blocks localhost, private IPs, link-local addresses, and non-HTTP(S) protocols
+ */
+export function validateUrlForSSRF(urlString) {
+    // Parse URL
+    let url;
+    try {
+        url = new URL(urlString);
+    }
+    catch {
+        throw new Error('Invalid URL format');
+    }
+    // Only allow HTTP(S)
+    if (!['http:', 'https:'].includes(url.protocol)) {
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+    const hostname = url.hostname.toLowerCase();
+    // Block localhost patterns
+    const localhostPatterns = ['localhost', '0.0.0.0'];
+    if (localhostPatterns.some(pattern => hostname === pattern || hostname.endsWith('.' + pattern))) {
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+    // SECURITY: Block well-known cloud metadata service hostnames.
+    // These hostnames resolve to link-local IPs (169.254.x.x) which are blocked
+    // by IP, but hostname-level blocking provides defense-in-depth against DNS
+    // rebinding attacks where a domain transiently resolves to a valid IP during
+    // validation, then resolves to a private IP for the actual fetch.
+    const metadataHostnames = [
+        'metadata.google.internal', // GCP: resolves to 169.254.169.254
+        'metadata.goog', // GCP alternate
+        'metadata.internal', // Generic internal
+        'instance-data.ec2.internal', // AWS alternate
+        'computeMetadata', // Partial GCP hostname
+    ];
+    if (metadataHostnames.some(m => hostname === m || hostname.endsWith('.' + m))) {
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+    // Parse and validate IP addresses
+    const ipv4Info = parseIPv4(hostname);
+    if (ipv4Info) {
+        validateIPv4ForSSRF(ipv4Info);
+    }
+    // Validate IPv6
+    if (hostname.includes(':')) {
+        validateIPv6ForSSRF(hostname);
+    }
+}
+/**
+ * SSRF Error class
+ */
+export class SSRFError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'SSRFError';
+    }
+}
+/**
+ * Parse IPv4 address in any format (dotted, hex, octal, decimal)
+ */
+function parseIPv4(hostname) {
+    const cleaned = hostname.replace(/^\[|\]$/g, '');
+    // Standard dotted notation: 192.168.1.1
+    const dottedRegex = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/;
+    const dottedMatch = cleaned.match(dottedRegex);
+    if (dottedMatch) {
+        const octets = dottedMatch.slice(1).map(Number);
+        if (octets.every(o => o >= 0 && o <= 255)) {
+            return octets;
+        }
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+    // Hex notation: 0x7f000001
+    if (/^0x[0-9a-fA-F]+$/.test(cleaned)) {
+        const num = parseInt(cleaned, 16);
+        return [
+            (num >>> 24) & 0xff,
+            (num >>> 16) & 0xff,
+            (num >>> 8) & 0xff,
+            num & 0xff,
+        ];
+    }
+    // Octal notation
+    if (/^0[0-7]/.test(cleaned)) {
+        if (/^0[0-7]+$/.test(cleaned)) {
+            const num = parseInt(cleaned, 8);
+            if (num <= 0xffffffff) {
+                return [
+                    (num >>> 24) & 0xff,
+                    (num >>> 16) & 0xff,
+                    (num >>> 8) & 0xff,
+                    num & 0xff,
+                ];
+            }
+        }
+        const parts = cleaned.split('.');
+        if (parts.length === 4) {
+            const octets = parts.map(p => parseInt(p, /^0[0-7]/.test(p) ? 8 : 10));
+            if (octets.every(o => o >= 0 && o <= 255)) {
+                return octets;
+            }
+        }
+    }
+    // Decimal notation: 2130706433
+    if (/^\d+$/.test(cleaned)) {
+        const num = parseInt(cleaned, 10);
+        if (num <= 0xffffffff) {
+            return [
+                (num >>> 24) & 0xff,
+                (num >>> 16) & 0xff,
+                (num >>> 8) & 0xff,
+                num & 0xff,
+            ];
+        }
+    }
+    return null;
+}
+/**
+ * Validate IPv4 address against private/reserved ranges
+ */
+function validateIPv4ForSSRF(octets) {
+    const [a, b, c, d] = octets;
+    // Loopback: 127.0.0.0/8
+    if (a === 127) {
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+    // Private: 10.0.0.0/8
+    if (a === 10) {
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+    // Private: 172.16.0.0/12
+    if (a === 172 && b >= 16 && b <= 31) {
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+    // Private: 192.168.0.0/16
+    if (a === 192 && b === 168) {
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+    // Link-local: 169.254.0.0/16 (includes AWS metadata endpoint)
+    if (a === 169 && b === 254) {
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+    // Broadcast: 255.255.255.255
+    if (a === 255 && b === 255 && c === 255 && d === 255) {
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+    // This network: 0.0.0.0/8
+    if (a === 0) {
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+}
+/**
+ * Validate IPv6 address against private/reserved ranges
+ */
+function validateIPv6ForSSRF(hostname) {
+    const addr = hostname.replace(/^\[|\]$/g, '').toLowerCase();
+    // Loopback: ::1
+    if (addr === '::1' || addr === '0:0:0:0:0:0:0:1') {
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+    // IPv6 mapped IPv4: ::ffff:192.168.1.1
+    if (addr.startsWith('::ffff:')) {
+        const ipv4Part = addr.substring(7);
+        if (ipv4Part.includes('.')) {
+            const parts = ipv4Part.split('.');
+            if (parts.length === 4) {
+                const octets = parts.map(p => parseInt(p, 10));
+                if (octets.every(o => !isNaN(o) && o >= 0 && o <= 255)) {
+                    validateIPv4ForSSRF(octets);
+                }
+            }
+        }
+        else {
+            const hexStr = ipv4Part.replace(/:/g, '');
+            if (/^[0-9a-f]{1,8}$/.test(hexStr)) {
+                const num = parseInt(hexStr, 16);
+                const octets = [
+                    (num >>> 24) & 0xff,
+                    (num >>> 16) & 0xff,
+                    (num >>> 8) & 0xff,
+                    num & 0xff,
+                ];
+                validateIPv4ForSSRF(octets);
+            }
+        }
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+    // Unique local addresses: fc00::/7
+    if (addr.startsWith('fc') || addr.startsWith('fd')) {
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+    // Link-local: fe80::/10
+    if (addr.startsWith('fe8') || addr.startsWith('fe9') ||
+        addr.startsWith('fea') || addr.startsWith('feb')) {
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+}