@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/server/routes/go.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Affiliate link cloaker — /go/:store/:path
+ *
+ * Redirects through our domain so users see webpeel.dev/go/amazon/...
+ * instead of amazon.com/dp/...?tag=wp0b7-20
+ *
+ * Examples:
+ *   /go/amazon/dp/B0D1XD1ZV3 → amazon.com/dp/B0D1XD1ZV3?tag=wp0b7-20
+ *   /go/walmart/ip/123456    → walmart.com/ip/123456?wmlspartner=...
+ *   /go/booking/hotel/hilton → booking.com/hotel/hilton?aid=...
+ *   /go?url=https://amazon.com/dp/B0... → adds affiliate tag + redirects
+ */
+import { Router } from 'express';
+export declare function createGoRouter(): Router;

package/dist/server/routes/go.js

@@ -0,0 +1,81 @@
+/**
+ * Affiliate link cloaker — /go/:store/:path
+ *
+ * Redirects through our domain so users see webpeel.dev/go/amazon/...
+ * instead of amazon.com/dp/...?tag=wp0b7-20
+ *
+ * Examples:
+ *   /go/amazon/dp/B0D1XD1ZV3 → amazon.com/dp/B0D1XD1ZV3?tag=wp0b7-20
+ *   /go/walmart/ip/123456    → walmart.com/ip/123456?wmlspartner=...
+ *   /go/booking/hotel/hilton → booking.com/hotel/hilton?aid=...
+ *   /go?url=https://amazon.com/dp/B0... → adds affiliate tag + redirects
+ */
+import { Router } from 'express';
+// Store configs: domain + affiliate param + env var
+const STORES = {
+    amazon: { domain: 'www.amazon.com', param: 'tag', envVar: 'AMAZON_AFFILIATE_TAG' },
+    walmart: { domain: 'www.walmart.com', param: 'wmlspartner', envVar: 'WALMART_AFFILIATE_ID' },
+    bestbuy: { domain: 'www.bestbuy.com', param: 'ref', envVar: 'BESTBUY_AFFILIATE_ID' },
+    target: { domain: 'www.target.com', param: 'afid', envVar: 'TARGET_AFFILIATE_ID' },
+    ebay: { domain: 'www.ebay.com', param: 'campid', envVar: 'EBAY_AFFILIATE_ID' },
+    etsy: { domain: 'www.etsy.com', param: 'ref', envVar: 'ETSY_AFFILIATE_ID' },
+    booking: { domain: 'www.booking.com', param: 'aid', envVar: 'BOOKING_AFFILIATE_ID' },
+    kayak: { domain: 'www.kayak.com', param: 'affid', envVar: 'KAYAK_AFFILIATE_ID' },
+    expedia: { domain: 'www.expedia.com', param: 'affcid', envVar: 'EXPEDIA_AFFILIATE_ID' },
+};
+export function createGoRouter() {
+    const router = Router();
+    // Route 1: /go?url=https://amazon.com/dp/... (raw URL redirect)
+    // Route 2: /go/:store/*path (clean path redirect)
+    router.get('/go', (req, res) => {
+        const rawUrl = req.query.url;
+        if (!rawUrl) {
+            res.status(400).json({ error: 'Missing ?url= parameter' });
+            return;
+        }
+        try {
+            const parsed = new URL(rawUrl);
+            const hostname = parsed.hostname.replace('www.', '');
+            // Find matching store and add affiliate tag
+            for (const [, config] of Object.entries(STORES)) {
+                const storeDomain = config.domain.replace('www.', '');
+                if (hostname === storeDomain || hostname.endsWith('.' + storeDomain)) {
+                    const tag = process.env[config.envVar];
+                    if (tag) {
+                        parsed.searchParams.set(config.param, tag);
+                    }
+                    break;
+                }
+            }
+            res.redirect(301, parsed.toString());
+        }
+        catch {
+            res.redirect(301, rawUrl);
+        }
+    });
+    router.get('/go/:store/*', (req, res) => {
+        const store = String(req.params.store || "").toLowerCase();
+        const path = req.params[0] || '';
+        const config = STORES[store];
+        if (!config) {
+            res.status(404).json({ error: `Unknown store: ${store}` });
+            return;
+        }
+        // Build the target URL
+        const targetUrl = new URL(`https://${config.domain}/${path}`);
+        // Copy query params from the request
+        for (const [key, value] of Object.entries(req.query)) {
+            if (typeof value === 'string') {
+                targetUrl.searchParams.set(key, value);
+            }
+        }
+        // Add affiliate tag
+        const tag = process.env[config.envVar];
+        if (tag) {
+            targetUrl.searchParams.set(config.param, tag);
+        }
+        // 301 redirect — browser goes directly to store
+        res.redirect(301, targetUrl.toString());
+    });
+    return router;
+}

package/dist/server/routes/health.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Health check endpoints
+ * NOTE: This route is mounted BEFORE auth/rate-limit middleware in app.ts
+ * so it's never blocked by rate limiting (Render hits it every ~30s).
+ *
+ * GET /health  — liveness probe: always returns 200 if process is alive
+ * GET /ready   — readiness probe: checks DB + job queue; returns 503 if not ready
+ */
+import { Router } from 'express';
+import type pg from 'pg';
+export declare function createHealthRouter(pool?: pg.Pool | null): Router;

package/dist/server/routes/health.js

@@ -0,0 +1,141 @@
+/**
+ * Health check endpoints
+ * NOTE: This route is mounted BEFORE auth/rate-limit middleware in app.ts
+ * so it's never blocked by rate limiting (Render hits it every ~30s).
+ *
+ * GET /health  — liveness probe: always returns 200 if process is alive
+ * GET /ready   — readiness probe: checks DB + job queue; returns 503 if not ready
+ */
+import { Router } from 'express';
+import { readFileSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { fetchCache, searchCache } from '../../core/fetch-cache.js';
+import { browserCircuitBreaker } from '../../core/circuit-breaker.js';
+import { getProxyState } from '../../core/proxy-config.js';
+// Memory threshold for readiness degradation (90% of 1GB pod limit)
+const MEMORY_READY_LIMIT_MB = 900;
+const startTime = Date.now();
+// Read version once at startup
+let version = 'unknown';
+try {
+    const pkgPath = join(dirname(fileURLToPath(import.meta.url)), '..', '..', '..', 'package.json');
+    version = JSON.parse(readFileSync(pkgPath, 'utf-8')).version;
+}
+catch {
+    // Fallback for bundled/Docker environments
+    try {
+        const altPath = join(process.cwd(), 'package.json');
+        version = JSON.parse(readFileSync(altPath, 'utf-8')).version;
+    }
+    catch { /* keep 'unknown' */ }
+}
+export function createHealthRouter(pool) {
+    const router = Router();
+    // ------------------------------------------------------------------
+    // GET /health — liveness probe
+    // K8s: if this fails, pod is restarted
+    // ------------------------------------------------------------------
+    router.get('/health', (_req, res) => {
+        const uptime = Math.floor((Date.now() - startTime) / 1000);
+        const fetchStats = fetchCache.stats();
+        const searchStats = searchCache.stats();
+        const mem = process.memoryUsage();
+        res.json({
+            status: 'healthy',
+            version,
+            uptime,
+            timestamp: new Date().toISOString(),
+            memory: {
+                rss: Math.round(mem.rss / 1024 / 1024),
+                heapUsed: Math.round(mem.heapUsed / 1024 / 1024),
+                heapTotal: Math.round(mem.heapTotal / 1024 / 1024),
+                external: Math.round(mem.external / 1024 / 1024),
+            },
+            browser: browserCircuitBreaker.getState(),
+            proxy: getProxyState(),
+            cache: {
+                fetch: {
+                    size: fetchStats.size,
+                    hitRate: fetchStats.hitRate,
+                },
+                search: {
+                    size: searchStats.size,
+                    hitRate: searchStats.hitRate,
+                },
+            },
+        });
+    });
+    // ------------------------------------------------------------------
+    // GET /ready — readiness probe
+    // K8s: if this fails, pod is removed from service endpoints (no traffic)
+    // Checks: database connectivity + queue (job table) reachability
+    // ------------------------------------------------------------------
+    router.get('/ready', async (_req, res) => {
+        const checks = {};
+        let allOk = true;
+        // --- Memory pressure check ---
+        const mem = process.memoryUsage();
+        const rssMB = Math.round(mem.rss / 1024 / 1024);
+        if (rssMB > MEMORY_READY_LIMIT_MB) {
+            checks.memory = { ok: false, rss: rssMB, error: `RSS ${rssMB}MB exceeds limit ${MEMORY_READY_LIMIT_MB}MB — pod under memory pressure` };
+            allOk = false;
+        }
+        else {
+            checks.memory = { ok: true, rss: rssMB };
+        }
+        // --- Database check ---
+        if (pool) {
+            const t0 = Date.now();
+            try {
+                await pool.query('SELECT 1');
+                checks.database = { ok: true, latencyMs: Date.now() - t0 };
+            }
+            catch (err) {
+                checks.database = { ok: false, latencyMs: Date.now() - t0, error: err?.message ?? 'unknown' };
+                allOk = false;
+            }
+        }
+        else {
+            // No pool configured (in-memory mode / local dev without DATABASE_URL)
+            checks.database = { ok: true, latencyMs: 0 };
+        }
+        // --- Job queue check (probe the jobs table via DATABASE_URL directly) ---
+        if (process.env.DATABASE_URL) {
+            const t0 = Date.now();
+            try {
+                // Reuse the same pool if available, or do a lightweight table existence check
+                if (pool) {
+                    await pool.query('SELECT COUNT(*) FROM jobs WHERE status = $1 LIMIT 1', ['queued']);
+                    checks.queue = { ok: true, latencyMs: Date.now() - t0 };
+                }
+                else {
+                    checks.queue = { ok: true, latencyMs: 0 };
+                }
+            }
+            catch (err) {
+                // Table may not exist in early boot; treat as non-fatal
+                const msg = err?.message ?? '';
+                if (msg.includes('relation "jobs" does not exist')) {
+                    checks.queue = { ok: true, latencyMs: Date.now() - t0 };
+                }
+                else {
+                    checks.queue = { ok: false, latencyMs: Date.now() - t0, error: msg };
+                    allOk = false;
+                }
+            }
+        }
+        else {
+            checks.queue = { ok: true, latencyMs: 0 };
+        }
+        const status = allOk ? 200 : 503;
+        res.status(status).json({
+            status: allOk ? 'ready' : 'not_ready',
+            version,
+            uptime: Math.floor((Date.now() - startTime) / 1000),
+            timestamp: new Date().toISOString(),
+            checks,
+        });
+    });
+    return router;
+}

package/dist/server/routes/jobs.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Async jobs API - crawl endpoints with SSE support
+ */
+import { Router } from 'express';
+import type { AuthStore } from '../auth-store.js';
+import type { IJobQueue } from '../job-queue.js';
+export declare function createJobsRouter(jobQueue: IJobQueue, authStore: AuthStore): Router;