@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/server/routes/cache-warm.js

@@ -0,0 +1,212 @@
+/**
+ * Cache Pre-Warming Routes
+ *
+ * GET /internal/popular-urls  — Top N URLs fetched in the last 24h (for CF Worker)
+ * GET /internal/cache-status  — Current warmer state (warmed URLs, last run time)
+ *
+ * Both routes are mounted BEFORE auth middleware so they're accessible internally.
+ * /internal/popular-urls is protected by CACHE_WARM_SECRET bearer token when set.
+ *
+ * startCacheWarmer() — server-side self-warming (opt-in via ENABLE_CACHE_WARM=true)
+ */
+import { Router } from 'express';
+import { createLogger } from '../logger.js';
+const log = createLogger('cache-warm');
+// ─── Fallback URLs ────────────────────────────────────────────────────────────
+// Used when the DB has no usage data yet (new deployment / empty DB).
+const FALLBACK_URLS = [
+    'https://www.bbc.com/news',
+    'https://news.ycombinator.com',
+    'https://github.com',
+    'https://en.wikipedia.org/wiki/Main_Page',
+    'https://www.reuters.com',
+    'https://techcrunch.com',
+    'https://stripe.com/docs',
+    'https://developer.mozilla.org',
+    'https://react.dev',
+    'https://docs.python.org/3/',
+    'https://nodejs.org/en/docs',
+    'https://www.npmjs.com',
+    'https://vercel.com/docs',
+    'https://nextjs.org/docs',
+    'https://tailwindcss.com/docs',
+    'https://www.typescriptlang.org/docs/',
+    'https://docs.render.com',
+    'https://cloudflare.com/docs',
+    'https://aws.amazon.com/documentation/',
+    'https://docs.github.com',
+    'https://www.nytimes.com',
+    'https://www.theguardian.com',
+    'https://arstechnica.com',
+    'https://www.wired.com',
+    'https://www.bloomberg.com/technology',
+    'https://lobste.rs',
+    'https://www.producthunt.com',
+    'https://stackoverflow.com',
+    'https://css-tricks.com',
+    'https://web.dev',
+];
+// ─── In-memory warmer state ───────────────────────────────────────────────────
+const warmerState = {
+    warmedUrls: new Set(),
+    lastWarmTime: null,
+};
+// ─── Router ──────────────────────────────────────────────────────────────────
+export function createCacheWarmRouter(pool) {
+    const router = Router();
+    // GET /internal/popular-urls
+    router.get('/internal/popular-urls', async (req, res) => {
+        // Auth check — if CACHE_WARM_SECRET is set, require it
+        const secret = process.env.CACHE_WARM_SECRET;
+        if (secret) {
+            const authHeader = req.headers['authorization'] || '';
+            const token = authHeader.startsWith('Bearer ') ? authHeader.slice(7) : '';
+            if (token !== secret) {
+                res.status(401).json({
+                    success: false,
+                    error: { type: 'unauthorized', message: 'Invalid or missing bearer token.' },
+                });
+                return;
+            }
+        }
+        let urls = [];
+        // Query DB if available
+        if (pool) {
+            try {
+                const result = await pool.query(`
+          SELECT url, COUNT(*) as fetch_count
+          FROM usage_logs
+          WHERE created_at > NOW() - INTERVAL '24 hours'
+            AND url IS NOT NULL
+            AND status_code >= 200 AND status_code < 300
+            AND url NOT LIKE '%localhost%'
+            AND url NOT LIKE '%127.0.0.1%'
+            AND url NOT LIKE '%169.254%'
+          GROUP BY url
+          ORDER BY fetch_count DESC
+          LIMIT 50
+        `);
+                urls = result.rows.map((row) => ({
+                    url: row.url,
+                    count: parseInt(row.fetch_count, 10),
+                }));
+            }
+            catch (err) {
+                log.warn('Failed to query usage_logs, falling back to static list', {
+                    error: err?.message,
+                });
+            }
+        }
+        // Fall back to static list if no DB data
+        if (urls.length === 0) {
+            log.info('No usage data found, using fallback URL list');
+            urls = FALLBACK_URLS.map((url) => ({ url, count: 0 }));
+        }
+        res.json({
+            urls,
+            total: urls.length,
+            generatedAt: new Date().toISOString(),
+        });
+    });
+    // GET /internal/cache-status
+    router.get('/internal/cache-status', (_req, res) => {
+        res.json({
+            warmedUrls: Array.from(warmerState.warmedUrls),
+            urlCount: warmerState.warmedUrls.size,
+            lastWarmTime: warmerState.lastWarmTime?.toISOString() ?? null,
+        });
+    });
+    return router;
+}
+// ─── Self-Warming ─────────────────────────────────────────────────────────────
+/**
+ * startCacheWarmer — server-side self-warming (fallback when no CF Worker).
+ *
+ * Every `intervalMs` (default 2 min):
+ *  1. Queries /internal/popular-urls (via the DB, not HTTP)
+ *  2. Fetches each URL through /r/<url> with concurrency 5
+ *  3. Updates warmerState for /internal/cache-status
+ *
+ * Only started if ENABLE_CACHE_WARM=true.
+ */
+export function startCacheWarmer(pool, intervalMs = 900_000) {
+    log.info('Cache warmer started', { intervalMs });
+    const runWarm = async () => {
+        const t0 = Date.now();
+        log.info('Cache warm cycle starting');
+        // Determine base URL
+        // RENDER_EXTERNAL_URL is legacy (Render is retired). On K8s, use PUBLIC_URL or localhost.
+        const base = process.env.PUBLIC_URL?.replace(/\/$/, '') ||
+            process.env.API_BASE_URL?.replace(/\/$/, '') ||
+            `http://localhost:${process.env.PORT || 3000}`;
+        // Step 1: Fetch popular URLs (same logic as the endpoint)
+        let urls = [];
+        if (pool) {
+            try {
+                const result = await pool.query(`
+          SELECT url, COUNT(*) as fetch_count
+          FROM usage_logs
+          WHERE created_at > NOW() - INTERVAL '24 hours'
+            AND url IS NOT NULL
+            AND status_code >= 200 AND status_code < 300
+            AND url NOT LIKE '%localhost%'
+            AND url NOT LIKE '%127.0.0.1%'
+            AND url NOT LIKE '%169.254%'
+          GROUP BY url
+          ORDER BY fetch_count DESC
+          LIMIT 50
+        `);
+                urls = result.rows.map((row) => ({
+                    url: row.url,
+                    count: parseInt(row.fetch_count, 10),
+                }));
+            }
+            catch (err) {
+                log.warn('Warm cycle: DB query failed, using fallback', { error: err?.message });
+            }
+        }
+        if (urls.length === 0) {
+            urls = FALLBACK_URLS.map((u) => ({ url: u, count: 0 }));
+        }
+        // Step 2: Warm each URL with concurrency 5
+        const concurrency = 5;
+        let warmed = 0;
+        let failed = 0;
+        const newWarmedSet = new Set();
+        for (let i = 0; i < urls.length; i += concurrency) {
+            const batch = urls.slice(i, i + concurrency);
+            const results = await Promise.allSettled(batch.map(({ url }) => fetch(`${base}/r/${encodeURIComponent(url)}`, {
+                headers: { 'User-Agent': 'WebPeel-CacheWarmer/1.0' },
+                signal: AbortSignal.timeout(15_000),
+            }).then((r) => {
+                if (r.ok) {
+                    newWarmedSet.add(url);
+                    warmed++;
+                }
+                else {
+                    failed++;
+                }
+            })));
+            // Count settled rejections as failures
+            results.forEach((r) => {
+                if (r.status === 'rejected') {
+                    failed++;
+                }
+            });
+        }
+        // Step 3: Update state
+        warmerState.warmedUrls = newWarmedSet;
+        warmerState.lastWarmTime = new Date();
+        const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
+        log.info(`Warmed ${warmed}/${urls.length} URLs in ${elapsed}s`, { failed });
+    };
+    // Run once immediately, then on interval
+    void runWarm().catch((err) => {
+        log.error('Cache warm cycle error', { error: err?.message });
+    });
+    setInterval(() => {
+        void runWarm().catch((err) => {
+            log.error('Cache warm cycle error', { error: err?.message });
+        });
+    }, intervalMs);
+}

package/dist/server/routes/cli-usage.d.ts

@@ -0,0 +1,6 @@
+/**
+ * CLI Usage endpoint — works with API key auth (not JWT)
+ * Used by the `webpeel usage` command and pre-fetch usage checks
+ */
+import { Router } from 'express';
+export declare function createCLIUsageRouter(): Router;

package/dist/server/routes/cli-usage.js

@@ -0,0 +1,127 @@
+/**
+ * CLI Usage endpoint — works with API key auth (not JWT)
+ * Used by the `webpeel usage` command and pre-fetch usage checks
+ */
+import { Router } from 'express';
+import pg from 'pg';
+const { Pool } = pg;
+export function createCLIUsageRouter() {
+    const router = Router();
+    const dbUrl = process.env.DATABASE_URL;
+    if (!dbUrl) {
+        // If no DB, return a stub router
+        router.get('/v1/cli/usage', (req, res) => {
+            res.status(501).json({
+                success: false,
+                error: {
+                    type: 'not_configured',
+                    message: 'Usage tracking requires PostgreSQL backend',
+                    docs: 'https://webpeel.dev/docs/errors#not_configured',
+                },
+                requestId: req.requestId,
+            });
+        });
+        return router;
+    }
+    const pool = new Pool({
+        connectionString: dbUrl,
+        // TLS: enabled when DATABASE_URL contains sslmode=require.
+        // Secure by default (rejectUnauthorized: true); set PG_REJECT_UNAUTHORIZED=false
+        // only for managed DBs (Render/Neon/Supabase) that use self-signed certs.
+        ssl: process.env.DATABASE_URL?.includes('sslmode=require')
+            ? { rejectUnauthorized: process.env.PG_REJECT_UNAUTHORIZED !== 'false' }
+            : undefined,
+    });
+    /**
+     * GET /v1/cli/usage
+     * Returns usage info for the authenticated API key's owner
+     * Auth: API key via Authorization: Bearer <key> or X-API-Key header
+     */
+    router.get('/v1/cli/usage', async (req, res) => {
+        try {
+            // Require API key auth (set by global auth middleware)
+            if (!req.auth?.keyInfo?.accountId) {
+                res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'Valid API key required. Run `webpeel login` to authenticate.', docs: 'https://webpeel.dev/docs/authentication' }, requestId: req.requestId });
+                return;
+            }
+            const userId = req.auth.keyInfo.accountId;
+            // Get user plan info
+            const planResult = await pool.query('SELECT tier, weekly_limit, burst_limit FROM users WHERE id = $1', [userId]);
+            if (planResult.rows.length === 0) {
+                res.status(404).json({ success: false, error: { type: 'user_not_found', message: 'User not found', docs: 'https://webpeel.dev/docs/errors#user_not_found' }, requestId: req.requestId });
+                return;
+            }
+            const plan = planResult.rows[0];
+            // Current week (ISO format)
+            const now = new Date();
+            const year = now.getUTCFullYear();
+            const jan4 = new Date(Date.UTC(year, 0, 4));
+            const weekNum = Math.ceil(((now.getTime() - jan4.getTime()) / 86400000 + jan4.getUTCDay() + 1) / 7);
+            const currentWeek = `${year}-W${String(weekNum).padStart(2, '0')}`;
+            // Current hour bucket
+            const currentHour = now.toISOString().substring(0, 13);
+            // Get weekly usage
+            const weeklyResult = await pool.query(`SELECT 
+          COALESCE(SUM(wu.total_count), 0) as total_used,
+          COALESCE(SUM(wu.basic_count), 0) as basic_used,
+          COALESCE(SUM(wu.stealth_count), 0) as stealth_used,
+          COALESCE(SUM(wu.search_count), 0) as search_used
+        FROM api_keys ak
+        LEFT JOIN weekly_usage wu ON wu.api_key_id = ak.id AND wu.week = $2
+        WHERE ak.user_id = $1 AND ak.is_active = true`, [userId, currentWeek]);
+            const weekly = weeklyResult.rows[0];
+            const totalUsed = parseInt(weekly.total_used) || 0;
+            const weeklyLimit = plan.weekly_limit || 125;
+            const remaining = Math.max(0, weeklyLimit - totalUsed);
+            // Get burst usage
+            const burstResult = await pool.query(`SELECT COALESCE(SUM(bu.count), 0) as burst_used
+        FROM api_keys ak
+        LEFT JOIN burst_usage bu ON bu.api_key_id = ak.id AND bu.hour_bucket = $2
+        WHERE ak.user_id = $1 AND ak.is_active = true`, [userId, currentHour]);
+            const burstUsed = parseInt(burstResult.rows[0]?.burst_used) || 0;
+            const burstLimit = plan.burst_limit || 25;
+            const minutesRemaining = 59 - now.getUTCMinutes();
+            // Get next Monday reset time
+            const dayOfWeek = now.getUTCDay();
+            const daysUntilMonday = dayOfWeek === 0 ? 1 : 8 - dayOfWeek;
+            const nextMonday = new Date(now);
+            nextMonday.setUTCDate(now.getUTCDate() + daysUntilMonday);
+            nextMonday.setUTCHours(0, 0, 0, 0);
+            res.json({
+                plan: {
+                    tier: plan.tier,
+                    weeklyLimit,
+                    burstLimit,
+                },
+                weekly: {
+                    used: totalUsed,
+                    limit: weeklyLimit,
+                    remaining,
+                    resetsAt: nextMonday.toISOString(),
+                    percentUsed: Math.round((totalUsed / weeklyLimit) * 100),
+                },
+                burst: {
+                    used: burstUsed,
+                    limit: burstLimit,
+                    resetsIn: minutesRemaining <= 0 ? '< 1 min' : `${minutesRemaining}m`,
+                },
+                // Simple boolean flags for CLI to check quickly
+                canFetch: remaining > 0 && burstUsed < burstLimit,
+                upgradeUrl: 'https://webpeel.dev/pricing',
+            });
+        }
+        catch (error) {
+            console.error('CLI usage error:', error);
+            res.status(500).json({
+                success: false,
+                error: {
+                    type: 'internal_error',
+                    message: 'Failed to retrieve usage',
+                    docs: 'https://webpeel.dev/docs/errors#internal_error',
+                },
+                requestId: req.requestId,
+            });
+        }
+    });
+    return router;
+}

package/dist/server/routes/compat.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Firecrawl API Compatibility Layer
+ *
+ * Drop-in replacement for Firecrawl's API - users can switch by ONLY changing the base URL.
+ * This is our killer acquisition feature.
+ *
+ * NOTE: Error responses in this file intentionally use Firecrawl's format:
+ *   { success: false, error: "Human-readable message" }
+ * This is required for Firecrawl drop-in compatibility and differs from the
+ * standard WebPeel API error format: { error: "error_code", message: "description" }.
+ * Do NOT change this format — it would break Firecrawl-compatible integrations.
+ *
+ * Implements Firecrawl endpoints:
+ * - POST /v1/scrape
+ * - POST /v2/scrape  (v2 with formats: ["screenshot"] support)
+ * - POST /v1/crawl
+ * - GET /v1/crawl/:id
+ * - POST /v1/search
+ * - POST /v1/map
+ */
+import { Router } from 'express';
+import type { IJobQueue } from '../job-queue.js';
+export declare function createCompatRouter(jobQueue: IJobQueue): Router;