@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/server/job-queue.js

@@ -0,0 +1,145 @@
+/**
+ * Job queue for async operations
+ *
+ * Factory creates PostgreSQL-backed queue in production or in-memory queue for local dev.
+ * Tracks crawl, batch scrape, and extraction jobs with progress updates.
+ */
+import { randomUUID } from 'crypto';
+import { PostgresJobQueue } from './pg-job-queue.js';
+import { createLogger } from './logger.js';
+const log = createLogger('job-queue');
+/**
+ * In-memory job queue for local development
+ */
+export class InMemoryJobQueue {
+    jobs = new Map();
+    cleanupInterval;
+    constructor() {
+        // Clean expired jobs every hour
+        this.cleanupInterval = setInterval(() => {
+            this.cleanExpired();
+        }, 60 * 60 * 1000);
+    }
+    /**
+     * Create a new job
+     */
+    createJob(type, webhook, ownerId) {
+        const now = new Date().toISOString();
+        const job = {
+            id: randomUUID(),
+            type,
+            status: 'queued',
+            progress: 0,
+            total: 0,
+            completed: 0,
+            creditsUsed: 0,
+            data: [],
+            webhook,
+            ownerId,
+            createdAt: now,
+            updatedAt: now,
+            expiresAt: new Date(Date.now() + 25 * 60 * 60 * 1000).toISOString(), // 25h from now (updated on completion)
+        };
+        this.jobs.set(job.id, job);
+        return job;
+    }
+    /**
+     * Get a job by ID
+     */
+    getJob(id) {
+        return this.jobs.get(id) || null;
+    }
+    /**
+     * Update a job
+     */
+    updateJob(id, update) {
+        const job = this.jobs.get(id);
+        if (!job)
+            return;
+        Object.assign(job, update, {
+            updatedAt: new Date().toISOString(),
+        });
+        // When job completes/fails, set expiration to 24h from now
+        if (update.status === 'completed' || update.status === 'failed') {
+            job.expiresAt = new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString();
+        }
+        // Update progress percentage
+        if (job.total > 0) {
+            job.progress = Math.round((job.completed / job.total) * 100);
+        }
+        this.jobs.set(id, job);
+    }
+    /**
+     * Cancel a job
+     */
+    cancelJob(id) {
+        const job = this.jobs.get(id);
+        if (!job)
+            return false;
+        // Can only cancel queued or processing jobs
+        if (job.status !== 'queued' && job.status !== 'processing') {
+            return false;
+        }
+        this.updateJob(id, {
+            status: 'cancelled',
+            expiresAt: new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString(),
+        });
+        return true;
+    }
+    /**
+     * List jobs with optional filters
+     */
+    listJobs(options) {
+        let jobs = Array.from(this.jobs.values());
+        if (options?.ownerId) {
+            jobs = jobs.filter(j => j.ownerId === options.ownerId);
+        }
+        if (options?.type) {
+            jobs = jobs.filter(j => j.type === options.type);
+        }
+        if (options?.status) {
+            jobs = jobs.filter(j => j.status === options.status);
+        }
+        // Sort by creation time (newest first)
+        jobs.sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime());
+        if (options?.limit) {
+            jobs = jobs.slice(0, options.limit);
+        }
+        return jobs;
+    }
+    /**
+     * Remove expired jobs
+     */
+    cleanExpired() {
+        const now = Date.now();
+        for (const [id, job] of this.jobs.entries()) {
+            if (new Date(job.expiresAt).getTime() < now) {
+                this.jobs.delete(id);
+            }
+        }
+    }
+    /**
+     * Clean up interval on shutdown
+     */
+    destroy() {
+        clearInterval(this.cleanupInterval);
+    }
+}
+/**
+ * Create job queue based on environment
+ * - Uses PostgreSQL if DATABASE_URL is set
+ * - Falls back to in-memory for local development
+ */
+export function createJobQueue() {
+    if (process.env.DATABASE_URL) {
+        log.info('Using PostgreSQL job queue');
+        return new PostgresJobQueue();
+    }
+    else {
+        log.info('Using in-memory job queue');
+        return new InMemoryJobQueue();
+    }
+}
+// Legacy global instance for backwards compatibility (deprecated)
+// @deprecated Use createJobQueue() instead
+export const jobQueue = new InMemoryJobQueue();

package/dist/server/logger.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Lightweight structured logger — no external dependencies.
+ * Respects LOG_LEVEL env var: debug | info | warn | error  (default: info)
+ */
+export declare function createLogger(component: string): {
+    debug(msg: string, data?: Record<string, unknown>): void;
+    info(msg: string, data?: Record<string, unknown>): void;
+    warn(msg: string, data?: Record<string, unknown>): void;
+    error(msg: string, data?: Record<string, unknown>): void;
+};

package/dist/server/logger.js

@@ -0,0 +1,37 @@
+/**
+ * Lightweight structured logger — no external dependencies.
+ * Respects LOG_LEVEL env var: debug | info | warn | error  (default: info)
+ */
+const LEVELS = { debug: 0, info: 1, warn: 2, error: 3 };
+const LOG_LEVEL = (process.env.LOG_LEVEL ?? 'info');
+function shouldLog(level) {
+    return LEVELS[level] >= (LEVELS[LOG_LEVEL] ?? LEVELS.info);
+}
+function formatLog(level, component, message, data) {
+    const ts = new Date().toISOString();
+    const base = `${ts} [${level.toUpperCase()}] [${component}] ${message}`;
+    if (data && Object.keys(data).length > 0) {
+        return `${base} ${JSON.stringify(data)}`;
+    }
+    return base;
+}
+export function createLogger(component) {
+    return {
+        debug(msg, data) {
+            if (shouldLog('debug'))
+                console.debug(formatLog('debug', component, msg, data));
+        },
+        info(msg, data) {
+            if (shouldLog('info'))
+                console.info(formatLog('info', component, msg, data));
+        },
+        warn(msg, data) {
+            if (shouldLog('warn'))
+                console.warn(formatLog('warn', component, msg, data));
+        },
+        error(msg, data) {
+            if (shouldLog('error'))
+                console.error(formatLog('error', component, msg, data));
+        },
+    };
+}

package/dist/server/middleware/audit-log.d.ts

@@ -0,0 +1,14 @@
+/**
+ * WebPeel Audit Logging Middleware
+ *
+ * Records who accessed which API endpoints and the outcome.
+ * Designed to be privacy-safe:
+ *  - Logs userId, keyId, method, path, status, duration, IP, user-agent
+ *  - Does NOT log: request bodies, auth headers, query params (may contain API keys)
+ *  - Only logs /v1/ endpoints (skips health checks, static files)
+ *
+ * When DATABASE_URL is set, also writes to usage_logs table (fire-and-forget).
+ */
+import { Request, Response, NextFunction } from 'express';
+import '../types.js';
+export declare function auditMiddleware(req: Request, res: Response, next: NextFunction): void;

package/dist/server/middleware/audit-log.js

@@ -0,0 +1,73 @@
+/**
+ * WebPeel Audit Logging Middleware
+ *
+ * Records who accessed which API endpoints and the outcome.
+ * Designed to be privacy-safe:
+ *  - Logs userId, keyId, method, path, status, duration, IP, user-agent
+ *  - Does NOT log: request bodies, auth headers, query params (may contain API keys)
+ *  - Only logs /v1/ endpoints (skips health checks, static files)
+ *
+ * When DATABASE_URL is set, also writes to usage_logs table (fire-and-forget).
+ */
+import '../types.js'; // Augments Express.Request with requestId
+import { createLogger } from '../logger.js';
+import pg from 'pg';
+const auditLog = createLogger('audit');
+// ── Singleton audit DB pool ───────────────────────────────────────────────────
+// One small pool shared by all requests — created lazily on first /v1/ request.
+let _auditPool = null;
+function getAuditPool() {
+    if (!_auditPool && process.env.DATABASE_URL) {
+        _auditPool = new pg.Pool({
+            connectionString: process.env.DATABASE_URL,
+            ssl: process.env.DATABASE_URL.includes('sslmode=require')
+                ? { rejectUnauthorized: process.env.PG_REJECT_UNAUTHORIZED !== 'false' }
+                : undefined,
+            max: 3,
+            idleTimeoutMillis: 30_000,
+        });
+    }
+    return _auditPool;
+}
+export function auditMiddleware(req, res, next) {
+    const startTime = Date.now();
+    res.on('finish', () => {
+        const duration = Date.now() - startTime;
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const auth = req.auth;
+        const userId = auth?.keyInfo?.accountId || 'anonymous';
+        // Use a truncated prefix of the key as a safe identifier (never log the full key)
+        const rawKey = auth?.keyInfo?.key;
+        const keyId = rawKey ? rawKey.slice(0, 8) + '...' : 'none';
+        const clientIp = req.headers['cf-connecting-ip'] ||
+            req.headers['x-forwarded-for'] ||
+            req.ip;
+        // Only log API endpoints (skip health checks, static files)
+        if (req.path.startsWith('/v1/')) {
+            auditLog.info(`${req.method} ${req.path}`, {
+                action: `${req.method} ${req.path}`,
+                userId,
+                keyId,
+                statusCode: res.statusCode,
+                duration,
+                ip: clientIp,
+                userAgent: req.headers['user-agent']?.slice(0, 100),
+                // DO NOT log: request body, auth headers, query params (may contain API keys or secrets)
+            });
+            // ── Fire-and-forget DB write ────────────────────────────────────────────
+            // Only runs when DATABASE_URL is set. Never slows down the response.
+            if (process.env.DATABASE_URL) {
+                const dbUserId = userId !== 'anonymous' ? userId : null;
+                const urlParam = typeof req.query?.url === 'string' ? req.query.url.slice(0, 2048) : null;
+                getAuditPool()
+                    .query(`INSERT INTO usage_logs
+              (user_id, url, endpoint, method, status_code, processing_time_ms, ip_address, created_at)
+             VALUES ($1, $2, $3, $4, $5, $6, $7, NOW())`, [dbUserId, urlParam, req.path, req.method, res.statusCode, duration, clientIp])
+                    .catch(() => {
+                    // Ignore audit DB failures — never propagate to the request
+                });
+            }
+        }
+    });
+    next();
+}

package/dist/server/middleware/auth.d.ts

@@ -0,0 +1,35 @@
+/**
+ * API key authentication middleware with SOFT LIMIT enforcement
+ *
+ * Philosophy: Never fully block users. When weekly limits are exceeded,
+ * degrade to HTTP-only mode instead of returning 429.
+ * BURST limits (hourly) are HARD limits and return 429.
+ *
+ * Dual auth: Accepts both API keys AND JWT session tokens.
+ * API keys are validated via the auth store; JWTs are verified with JWT_SECRET.
+ * Dashboard pages use JWT tokens; CLI/SDK users use API keys.
+ */
+import { Request, Response, NextFunction } from 'express';
+import { AuthStore, ApiKeyInfo } from '../auth-store.js';
+import { KeyScope } from '../pg-auth-store.js';
+import '../types.js';
+declare global {
+    namespace Express {
+        interface Request {
+            auth?: {
+                keyInfo: ApiKeyInfo | null;
+                tier: 'free' | 'starter' | 'pro' | 'enterprise' | 'max' | 'admin';
+                rateLimit: number;
+                softLimited: boolean;
+                extraUsageAvailable: boolean;
+            };
+            /**
+             * Permission scope of the authenticated API key.
+             * Undefined when authenticated via JWT (dashboard session) — JWT users bypass scope enforcement.
+             * Set to 'full' | 'read' | 'restricted' for API key requests.
+             */
+            keyScope?: KeyScope;
+        }
+    }
+}
+export declare function createAuthMiddleware(authStore: AuthStore): (req: Request, res: Response, next: NextFunction) => Promise<void>;

package/dist/server/middleware/auth.js

@@ -0,0 +1,225 @@
+/**
+ * API key authentication middleware with SOFT LIMIT enforcement
+ *
+ * Philosophy: Never fully block users. When weekly limits are exceeded,
+ * degrade to HTTP-only mode instead of returning 429.
+ * BURST limits (hourly) are HARD limits and return 429.
+ *
+ * Dual auth: Accepts both API keys AND JWT session tokens.
+ * API keys are validated via the auth store; JWTs are verified with JWT_SECRET.
+ * Dashboard pages use JWT tokens; CLI/SDK users use API keys.
+ */
+import jwt from 'jsonwebtoken';
+import { PostgresAuthStore } from '../pg-auth-store.js';
+import '../types.js'; // Augments Express.Request with requestId
+export function createAuthMiddleware(authStore) {
+    return async (req, res, next) => {
+        try {
+            // Extract API key from Authorization header or X-API-Key header
+            const authHeader = req.headers.authorization;
+            const apiKeyHeader = req.headers['x-api-key'];
+            // SECURITY: Skip API key auth for public/JWT-protected endpoints
+            // These routes either need no auth or use their own JWT middleware
+            const isPublicEndpoint = req.path === '/health' || req.path === '/ready' ||
+                req.path.startsWith('/v1/auth/') ||
+                req.path === '/v1/webhooks/stripe' ||
+                req.path === '/v1/me' ||
+                req.path.startsWith('/v1/keys') ||
+                req.path.startsWith('/v1/extra-usage');
+            if (isPublicEndpoint) {
+                req.auth = {
+                    keyInfo: null,
+                    tier: 'free',
+                    rateLimit: 10,
+                    softLimited: false,
+                    extraUsageAvailable: false,
+                };
+                return next();
+            }
+            // SECURITY: Ensure headers are strings before processing (guards against
+            // malformed multi-value headers that would cause a null reference crash → 502)
+            const rawAuth = typeof authHeader === 'string' ? authHeader : undefined;
+            const rawApiKey = typeof apiKeyHeader === 'string' ? apiKeyHeader : undefined;
+            let apiKey = null;
+            if (rawAuth?.startsWith('Bearer ')) {
+                apiKey = rawAuth.slice(7);
+            }
+            else if (rawApiKey) {
+                apiKey = rawApiKey;
+            }
+            if (!apiKey) {
+                // No credentials supplied — let the request through unauthenticated.
+                // Public routes (/health, /v1/auth/*) already returned above.
+                // Protected routes must check req.auth?.keyInfo and return 401 themselves.
+                req.auth = undefined;
+                return next();
+            }
+            // Validate API key if provided
+            let keyInfo = null;
+            let softLimited = false;
+            let extraUsageAvailable = false;
+            if (apiKey) {
+                keyInfo = await authStore.validateKey(apiKey);
+                if (!keyInfo) {
+                    // Check if key exists but is expired (before falling through to JWT)
+                    if (authStore instanceof PostgresAuthStore) {
+                        const expired = await authStore.isKeyExpired(apiKey);
+                        if (expired) {
+                            res.status(401).json({
+                                success: false,
+                                error: {
+                                    type: 'key_expired',
+                                    message: 'This API key has expired.',
+                                    hint: 'Generate a new key at https://app.webpeel.dev/keys',
+                                    docs: 'https://webpeel.dev/docs/errors#unauthorized',
+                                },
+                                metadata: { requestId: req.requestId },
+                            });
+                            return;
+                        }
+                    }
+                    // API key not found — try JWT session token as fallback.
+                    // Dashboard uses JWT tokens (from /v1/auth/login or /v1/auth/oauth),
+                    // while CLI/SDK users use API keys. Support both seamlessly.
+                    const jwtSecret = process.env.JWT_SECRET;
+                    if (jwtSecret) {
+                        try {
+                            const payload = jwt.verify(apiKey, jwtSecret);
+                            if (payload.userId) {
+                                // Valid JWT — treat as authenticated session user.
+                                // Set req.auth with null keyInfo (no API key) and attach
+                                // the JWT payload so routes can use req.user.userId.
+                                req.auth = {
+                                    keyInfo: null,
+                                    tier: payload.tier || 'free',
+                                    rateLimit: 25,
+                                    softLimited: false,
+                                    extraUsageAvailable: false,
+                                };
+                                req.user = payload;
+                                return next();
+                            }
+                        }
+                        catch (e) {
+                            if (process.env.DEBUG)
+                                console.debug('[webpeel]', 'jwt verify failed:', e instanceof Error ? e.message : e);
+                        }
+                    }
+                    // Firecrawl compat routes expect { error: "string" } not { error: { type, message } }
+                    const isCompatRoute = /^\/(v[12]\/(scrape|crawl|map|search))/.test(req.path);
+                    if (isCompatRoute) {
+                        res.status(401).json({
+                            success: false,
+                            error: 'Invalid or expired API key. Get one at https://app.webpeel.dev/keys',
+                        });
+                    }
+                    else {
+                        res.status(401).json({
+                            success: false,
+                            error: {
+                                type: 'invalid_key',
+                                message: 'Invalid or expired API key.',
+                                hint: 'Check your key at https://app.webpeel.dev/keys or generate a new one.',
+                                docs: 'https://webpeel.dev/docs/errors#unauthorized',
+                            },
+                            metadata: { requestId: req.requestId },
+                        });
+                    }
+                    return;
+                }
+                // Check limits (only for PostgresAuthStore, skip for admin tier)
+                if (authStore instanceof PostgresAuthStore && keyInfo?.tier !== 'admin') {
+                    // HARD LIMIT: Check burst limit first (per-hour cap)
+                    const { allowed: burstAllowed, burst } = await authStore.checkBurstLimit(apiKey);
+                    if (!burstAllowed) {
+                        // Burst limit exceeded - HARD 429 with Retry-After
+                        const retryAfterSeconds = 60 * parseInt(burst.resetsIn.match(/\d+/)?.[0] || '1', 10);
+                        res.setHeader('Retry-After', retryAfterSeconds.toString());
+                        res.setHeader('X-Burst-Limit', burst.limit.toString());
+                        res.setHeader('X-Burst-Used', burst.count.toString());
+                        const tier = keyInfo?.tier || 'free';
+                        const upgradeHint = tier === 'free'
+                            ? ' Upgrade to Pro ($9/mo) for 100/hr → https://webpeel.dev/pricing'
+                            : tier === 'pro'
+                                ? ' Upgrade to Max ($29/mo) for 500/hr → https://webpeel.dev/pricing'
+                                : '';
+                        res.status(429).json({
+                            success: false,
+                            error: {
+                                type: 'burst_limit_exceeded',
+                                message: `Hourly burst limit exceeded (${burst.count}/${burst.limit} on ${tier} plan). Resets in ${burst.resetsIn}.`,
+                                hint: `Retry after ${burst.resetsIn}.${upgradeHint}`,
+                                docs: 'https://webpeel.dev/docs/errors#rate-limited',
+                            },
+                            metadata: { requestId: req.requestId },
+                        });
+                        return;
+                    }
+                    // Add burst headers
+                    res.setHeader('X-Burst-Limit', burst.limit.toString());
+                    res.setHeader('X-Burst-Used', burst.count.toString());
+                    res.setHeader('X-Burst-Remaining', burst.remaining.toString());
+                    // SOFT LIMIT: Check weekly usage
+                    const { allowed, usage } = await authStore.checkLimit(apiKey);
+                    // Check if extra usage is available
+                    if (!allowed) {
+                        extraUsageAvailable = await authStore.canUseExtraUsage(apiKey);
+                        if (!extraUsageAvailable) {
+                            // Over weekly quota, no extra usage — SOFT LIMIT (degrade to HTTP-only)
+                            softLimited = true;
+                            res.setHeader('X-Soft-Limited', 'true');
+                            res.setHeader('X-Soft-Limit-Reason', 'Weekly quota exceeded. Requests degraded to HTTP-only mode.');
+                            res.setHeader('X-Upgrade-URL', 'https://webpeel.dev/pricing');
+                        }
+                    }
+                    // Add weekly usage headers
+                    if (usage) {
+                        res.setHeader('X-Weekly-Limit', usage.totalAvailable.toString());
+                        res.setHeader('X-Weekly-Used', usage.totalUsed.toString());
+                        res.setHeader('X-Weekly-Remaining', Math.max(0, usage.remaining).toString());
+                        res.setHeader('X-Weekly-Percent', usage.percentUsed.toString());
+                        res.setHeader('X-Weekly-Resets-At', usage.resetsAt);
+                        // Warn if over 80% usage and not using extra usage
+                        if (usage.percentUsed >= 80 && !softLimited && !extraUsageAvailable) {
+                            res.setHeader('X-Usage-Warning', `You've used ${usage.percentUsed}% of your weekly quota. Consider upgrading at https://webpeel.dev/pricing`);
+                        }
+                    }
+                    // Add extra usage headers if available
+                    const extraInfo = await authStore.getExtraUsageInfo(apiKey);
+                    if (extraInfo) {
+                        res.setHeader('X-Extra-Usage-Enabled', extraInfo.enabled ? 'true' : 'false');
+                        res.setHeader('X-Extra-Usage-Balance', extraInfo.balance.toFixed(2));
+                        if (extraInfo.enabled) {
+                            res.setHeader('X-Extra-Usage-Spent', extraInfo.spent.toFixed(2));
+                            res.setHeader('X-Extra-Usage-Limit', extraInfo.spendingLimit.toFixed(2));
+                        }
+                    }
+                }
+            }
+            // Set auth context on request
+            req.auth = {
+                keyInfo,
+                tier: keyInfo?.tier || 'free',
+                rateLimit: keyInfo?.rateLimit || 10,
+                softLimited,
+                extraUsageAvailable,
+            };
+            // Attach API key scope (only for API key auth; JWT users get undefined = bypass scope checks)
+            if (keyInfo) {
+                req.keyScope = keyInfo.scope || 'full';
+            }
+            next();
+        }
+        catch (_error) {
+            res.status(500).json({
+                success: false,
+                error: {
+                    type: 'internal_error',
+                    message: 'Authentication failed',
+                    docs: 'https://webpeel.dev/docs/errors#internal-error',
+                },
+                metadata: { requestId: req.requestId },
+            });
+        }
+    };
+}

package/dist/server/middleware/rate-limit.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Redis-backed rate limiting middleware.
+ *
+ * Uses `rate-limiter-flexible` for atomic Redis operations — works correctly
+ * across all 6 K8s API pods (shared state via Redis).
+ *
+ * Falls back to in-memory when REDIS_URL is not set (CLI/local dev).
+ *
+ * Features:
+ * - Per-key (API key or IP) rate limiting
+ * - Tier-based limits: free=50/hr, pro=100/hr, max=500/hr
+ * - Route-based cost weighting: crawl=5x, render=3x, batch/screenshot=2x
+ * - Per-IP rate limiting ON TOP of API key limits (abuse prevention)
+ * - Exempt paths for health/docs endpoints
+ */
+import { Request, Response, NextFunction } from 'express';
+import '../types.js';
+export declare class RateLimiter {
+    private keyLimiter;
+    private ipLimiter;
+    private windowMs;
+    private initialized;
+    private initPromise;
+    constructor(windowMs?: number);
+    private init;
+    waitForInit(): Promise<void>;
+    /**
+     * Check if request is allowed.
+     * Returns { allowed, remaining, retryAfter? }
+     */
+    checkLimit(identifier: string, limit: number, cost?: number): Promise<{
+        allowed: boolean;
+        remaining: number;
+        retryAfter?: number;
+    }>;
+    /**
+     * Check per-IP limit (global cap regardless of API key).
+     */
+    checkIpLimit(ip: string, cost?: number): Promise<{
+        allowed: boolean;
+        remaining: number;
+        retryAfter?: number;
+    }>;
+    /**
+     * Backward compat: cleanup is no-op for Redis (TTL handles it).
+     * Kept for in-memory fallback and for app.ts which calls it on an interval.
+     */
+    cleanup(): void;
+}
+export declare function createRateLimitMiddleware(limiter: RateLimiter): (req: Request, res: Response, next: NextFunction) => Promise<void>;