@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/server/routes/fetch.js

@@ -0,0 +1,1274 @@
+/**
+ * Fetch endpoint with caching
+ */
+import { Router } from 'express';
+import '../types.js'; // Augments Express.Request with requestId
+import { peel } from '../../index.js';
+import { proxyContextStorage, getProxyUsage } from '../../core/proxy-config.js';
+import { normalizeActions } from '../../core/actions.js';
+import { extractInlineJson } from '../../core/extract-inline.js';
+import { LRUCache } from 'lru-cache';
+import { validateUrlForSSRF, SSRFError } from '../middleware/url-validator.js';
+import { wantsEnvelope, successResponse } from '../utils/response.js';
+import { getSchemaTemplate } from '../../core/schema-templates.js';
+import { quickAnswer } from '../../core/quick-answer.js';
+import { sendUsageAlertEmail, checkAndSendDualAlert } from '../email-service.js';
+import { extractLinks } from '../../core/links.js';
+// ── Method cost table — exposed in response headers + body ────────────────────
+const methodCosts = {
+    'simple': '0.0002', // $0.0002 per page (basic HTTP)
+    'domain-api': '0.0002', // same as simple (we use the site's API)
+    'stealth': '0.005', // $0.005 per page (headless browser)
+    'browser': '0.005', // same as stealth
+    'captcha': '0.02', // $0.02 per page (anti-bot bypass)
+};
+// ── Helper: classify an error thrown by peel() into a FetchErrorType ─────────
+function classifyFetchError(err) {
+    const code = err.code || err.name || '';
+    const msg = (err.message || '').toLowerCase();
+    if (code === 'TIMEOUT' || msg.includes('timeout') || msg.includes('timed out')) {
+        return 'timeout';
+    }
+    if (code === 'BLOCKED' || msg.includes('blocked') || msg.includes('cloudflare challenge') || msg.includes('captcha') || msg.includes('bot detection')) {
+        return 'blocked';
+    }
+    if (msg.includes('http 404') || msg.includes('not found') || msg.includes('dns resolution failed') || msg.includes('enotfound') || msg.includes('getaddrinfo')) {
+        return 'not_found';
+    }
+    if (msg.match(/http\s+5\d{2}/) || msg.includes('server error') || msg.includes('internal server')) {
+        return 'server_error';
+    }
+    if (code === 'NETWORK' || msg.includes('network') || msg.includes('econnrefused') || msg.includes('connection refused') || msg.includes('connection reset')) {
+        return 'network';
+    }
+    return 'unknown';
+}
+// ── Helper: build a clean, user-facing error message from a peel() error ─────
+function buildFetchErrorMessage(err) {
+    const type = classifyFetchError(err);
+    const hints = {
+        timeout: 'Try increasing timeout with ?timeout=20000, or use render=true for JS-heavy sites.',
+        blocked: 'Site blocked the request. Try adding render=true or stealth mode.',
+        not_found: 'Verify the URL is correct and the site is accessible.',
+        server_error: 'The target site returned a server error. Try again later.',
+        network: 'Could not connect to the target URL. Verify the URL is correct and the site is online.',
+        unknown: undefined,
+    };
+    const docsLinks = {
+        timeout: 'https://webpeel.dev/docs/errors#timeout',
+        blocked: 'https://webpeel.dev/docs/errors#blocked',
+        not_found: 'https://webpeel.dev/docs/errors#not-found',
+        server_error: 'https://webpeel.dev/docs/errors#server-error',
+        network: 'https://webpeel.dev/docs/errors#network',
+        unknown: 'https://webpeel.dev/docs/errors',
+    };
+    // Sanitize message: strip HTML chars, truncate
+    const safeMsg = (err.message || 'An unexpected error occurred while fetching the URL')
+        .replace(/[<>"']/g, '')
+        .trim();
+    const messages = {
+        timeout: `Request timed out. Try increasing the timeout parameter or use render=true for JavaScript-heavy sites.`,
+        blocked: `Site blocked the request. Try adding render=true or stealth mode to bypass bot protection.`,
+        not_found: `The URL could not be reached — the domain may not exist or the page was not found.`,
+        server_error: `The target website returned a server error while processing the request.`,
+        network: `Could not reach this website. The server may be down or the URL may be incorrect.`,
+        unknown: safeMsg,
+    };
+    return { type, message: messages[type] || safeMsg, hint: hints[type], docs: docsLinks[type] };
+}
+// ── Helper: extractive summarizer (TF-IDF-like sentence scoring) ─────────────
+function extractSummary(content, maxWords = 150) {
+    if (!content)
+        return '';
+    const sentences = content
+        .split(/(?<=[.!?])\s+/)
+        .map(s => s.trim())
+        .filter(s => s.length > 40 && s.length < 600);
+    if (sentences.length === 0) {
+        const words = content.split(/\s+/);
+        return words.slice(0, maxWords).join(' ') + (words.length > maxWords ? '\u2026' : '');
+    }
+    if (sentences.length <= 3)
+        return sentences.join(' ');
+    const allWords = content.toLowerCase().split(/\W+/).filter(w => w.length > 3);
+    const wordFreq = {};
+    for (const w of allWords)
+        wordFreq[w] = (wordFreq[w] || 0) + 1;
+    const maxFreq = Math.max(1, ...Object.values(wordFreq));
+    const scored = sentences.map((sentence, idx) => {
+        const words = sentence.toLowerCase().split(/\W+/).filter(w => w.length > 3);
+        const score = words.reduce((sum, w) => sum + (wordFreq[w] || 0) / maxFreq, 0) / Math.max(1, words.length);
+        const posBonus = idx === 0 ? 0.3 : idx === sentences.length - 1 ? 0.1 : 0;
+        return { sentence, score: score + posBonus, idx };
+    });
+    scored.sort((a, b) => b.score - a.score);
+    const selected = [];
+    let wc = 0;
+    for (const item of scored) {
+        const itemWc = item.sentence.split(/\s+/).length;
+        if (wc + itemWc > maxWords * 1.3)
+            break;
+        selected.push(item);
+        wc += itemWc;
+        if (selected.length >= 5)
+            break;
+    }
+    selected.sort((a, b) => a.idx - b.idx);
+    return selected.map(s => s.sentence).join(' ');
+}
+// ── Helper: check usage and determine if alert email should be sent ───────────
+async function checkAndTriggerAlert(pgStore, userId) {
+    const getCurrentWeek = () => {
+        const now = new Date();
+        const year = now.getUTCFullYear();
+        const jan4 = new Date(Date.UTC(year, 0, 4));
+        const weekNum = Math.ceil(((now.getTime() - jan4.getTime()) / 86400000 + jan4.getUTCDay() + 1) / 7);
+        return `${year}-W${String(weekNum).padStart(2, '0')}`;
+    };
+    const currentWeek = getCurrentWeek();
+    const result = await pgStore.pool.query(`SELECT u.email, u.name, u.tier, u.alert_threshold, u.alert_email, u.alert_sent_at,
+            u.weekly_limit,
+            COALESCE(SUM(wu.total_count), 0) AS total_used,
+            u.weekly_limit + COALESCE(MAX(wu.rollover_credits), 0) AS total_available
+     FROM users u
+     LEFT JOIN api_keys ak ON ak.user_id = u.id
+     LEFT JOIN weekly_usage wu ON wu.api_key_id = ak.id AND wu.week = $2
+     WHERE u.id = $1
+     GROUP BY u.id, u.email, u.name, u.tier, u.alert_threshold, u.alert_email, u.alert_sent_at, u.weekly_limit`, [userId, currentWeek]);
+    const row = result.rows[0];
+    if (!row || !row.alert_threshold)
+        return { shouldSendAlert: false };
+    const used = parseInt(row.total_used, 10) || 0;
+    const total = parseInt(row.total_available, 10) || row.weekly_limit || 999;
+    const usagePercent = total > 0 ? Math.round((used / total) * 100) : 0;
+    // Only alert if: crosses threshold AND haven't sent alert this week
+    const lastAlert = row.alert_sent_at ? new Date(row.alert_sent_at) : null;
+    const oneWeekAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000);
+    const alreadySentThisWeek = lastAlert !== null && lastAlert > oneWeekAgo;
+    return {
+        shouldSendAlert: usagePercent >= row.alert_threshold && !alreadySentThisWeek,
+        usagePercent,
+        used,
+        total,
+        userEmail: row.email,
+        userName: row.name || undefined,
+        userTier: row.tier,
+        alertEmail: row.alert_email || undefined,
+    };
+}
+const VALID_LLM_PROVIDERS = ['openai', 'anthropic', 'google'];
+export function createFetchRouter(authStore) {
+    const router = Router();
+    // LRU cache: 5 minute TTL, max 500 entries, 100MB total size
+    const cache = new LRUCache({
+        max: 500,
+        ttl: 5 * 60 * 1000, // 5 minutes default
+        maxSize: 100 * 1024 * 1024, // 100MB
+        sizeCalculation: (entry) => {
+            return JSON.stringify(entry).length;
+        },
+    });
+    router.get('/v1/fetch', async (req, res) => {
+        try {
+            // Require authentication — API key or JWT session
+            const userId = req.auth?.keyInfo?.accountId || req.user?.userId;
+            if (!userId) {
+                res.status(401).json({
+                    success: false,
+                    error: {
+                        type: 'unauthorized',
+                        message: 'API key required. Get one free at https://app.webpeel.dev/keys',
+                        hint: 'Get a free API key at https://app.webpeel.dev/keys',
+                        docs: 'https://webpeel.dev/docs/errors#unauthorized',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            const { url, render, wait, format, includeTags, excludeTags, images, location, languages, onlyMainContent, actions, maxAge, storeInCache, stream, noCache, cacheTtl, budget, question, summary, readable, stealth, screenshot, maxTokens, selector, exclude, fullPage, raw, noDomainApi, lite, timeout, schema, detail, captionImages, highlightQuery, highlightMaxChars, } = req.query;
+            const detailMode = detail || 'standard';
+            // Validate URL parameter
+            if (!url || typeof url !== 'string') {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Missing or invalid "url" parameter.',
+                        hint: 'Pass a URL as a query parameter: GET /v1/fetch?url=https://example.com',
+                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            // SECURITY: Validate URL format and length
+            if (url.length > 2048) {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_url',
+                        message: 'URL too long (max 2048 characters)',
+                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            try {
+                const parsed = new URL(url);
+                // Normalize URL for consistent caching
+                const normalizedUrl = parsed.href;
+                // Use normalized URL for cache key
+                if (normalizedUrl !== url) {
+                    // URL was normalized, update for caching
+                }
+            }
+            catch {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_url',
+                        message: 'Invalid URL format',
+                        hint: 'Ensure the URL includes a scheme (https://) and a valid hostname',
+                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            // SECURITY: Validate URL to prevent SSRF attacks
+            try {
+                validateUrlForSSRF(url);
+            }
+            catch (error) {
+                if (error instanceof SSRFError) {
+                    res.status(400).json({
+                        success: false,
+                        error: {
+                            type: 'forbidden_url',
+                            message: 'This URL is blocked for security. Localhost, private networks, and non-HTTP URLs are not allowed.',
+                            hint: 'See docs for allowed URL formats.',
+                            docs: 'https://webpeel.dev/docs/errors#forbidden-url',
+                        },
+                        requestId: req.requestId,
+                    });
+                    return;
+                }
+                throw error;
+            }
+            // Parse actions query param (JSON-encoded array)
+            let parsedActions;
+            if (actions && typeof actions === 'string') {
+                try {
+                    const raw = JSON.parse(actions);
+                    parsedActions = normalizeActions(raw);
+                }
+                catch (e) {
+                    res.status(400).json({
+                        success: false,
+                        error: {
+                            type: 'invalid_request',
+                            message: 'Invalid "actions" parameter: must be a valid JSON array',
+                            docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                        },
+                        requestId: req.requestId,
+                    });
+                    return;
+                }
+            }
+            // Build cache key (include new parameters)
+            const actionsKey = parsedActions ? JSON.stringify(parsedActions) : '';
+            const cacheKey = `fetch:${url}:${render}:${wait}:${format}:${includeTags}:${excludeTags}:${images}:${location}:${languages}:${onlyMainContent}:${stream}:${actionsKey}:${budget}:${question}:${summary}:${readable}:${stealth}:${screenshot}:${maxTokens}:${selector}:${exclude}:${fullPage}:${raw}:${highlightQuery || ''}:${highlightMaxChars || ''}`;
+            // Cache bypass: ?noCache=true or Cache-Control: no-cache header
+            const bypassCache = noCache === 'true' || req.headers['cache-control'] === 'no-cache';
+            // Per-request TTL (cacheTtl in seconds, default 300s = 5 min)
+            const cacheTtlMs = cacheTtl !== undefined
+                ? parseInt(cacheTtl, 10) * 1000
+                : 5 * 60 * 1000;
+            // Check cache (with maxAge support)
+            const maxAgeMs = maxAge !== undefined ? parseInt(maxAge, 10) : 172800000; // Default 2 days
+            if (!bypassCache) {
+                const cached = cache.get(cacheKey);
+                if (cached && maxAgeMs > 0) {
+                    const cacheAge = Date.now() - cached.timestamp;
+                    if (cacheAge < maxAgeMs && cacheAge < cacheTtlMs) {
+                        res.setHeader('X-Cache', 'HIT');
+                        res.setHeader('X-Cache-Status', 'HIT');
+                        res.setHeader('X-Cache-Age', Math.floor(cacheAge / 1000).toString());
+                        // Cache-Control: allow Cloudflare edge to cache successful GET responses
+                        res.setHeader('Cache-Control', 'public, s-maxage=60, stale-while-revalidate=300');
+                        if (wantsEnvelope(req)) {
+                            successResponse(res, cached.result, {
+                                requestId: req.requestId,
+                                cached: true,
+                            });
+                        }
+                        else {
+                            res.json(cached.result);
+                        }
+                        return;
+                    }
+                }
+            }
+            // Parse options
+            const isSoftLimited = req.auth?.softLimited === true;
+            const hasExtraUsage = req.auth?.extraUsageAvailable === true;
+            // Parse tag arrays from comma-separated strings
+            const includeTagsArray = includeTags
+                ? includeTags.split(',').map(t => t.trim()).filter(Boolean)
+                : undefined;
+            const excludeTagsArray = excludeTags
+                ? excludeTags.split(',').map(t => t.trim()).filter(Boolean)
+                : undefined;
+            const languagesArray = languages
+                ? languages.split(',').map(l => l.trim()).filter(Boolean)
+                : undefined;
+            // onlyMainContent is a shortcut for common include tags
+            const finalIncludeTags = onlyMainContent === 'true'
+                ? ['main', 'article', '.content', '#content']
+                : includeTagsArray;
+            // When actions are present, force browser mode (skip HTTP fast path)
+            const hasActions = parsedActions && parsedActions.length > 0;
+            const shouldRender = hasActions || render === 'true';
+            const options = {
+                // SOFT LIMIT: When over quota AND no extra usage, force HTTP-only
+                // If extra usage is available, allow full functionality
+                // Exception: actions always require render
+                render: (isSoftLimited && !hasExtraUsage && !hasActions) ? false : shouldRender,
+                wait: (isSoftLimited && !hasExtraUsage) ? 0 : (wait ? parseInt(wait, 10) : undefined),
+                format: format || 'markdown',
+                stream: stream === 'true',
+                includeTags: finalIncludeTags,
+                excludeTags: excludeTagsArray,
+                images: images === 'true',
+                actions: parsedActions,
+                location: location || languagesArray ? {
+                    country: location,
+                    languages: languagesArray,
+                } : undefined,
+                budget: budget ? parseInt(budget, 10) : undefined,
+                question: question,
+                readable: readable === 'true',
+                stealth: (isSoftLimited && !hasExtraUsage) ? false : stealth === 'true',
+                screenshot: (isSoftLimited && !hasExtraUsage) ? false : screenshot === 'true',
+                maxTokens: maxTokens ? parseInt(maxTokens, 10) : undefined,
+                selector: selector,
+                exclude: exclude ? exclude.split(',').map(s => s.trim()).filter(Boolean) : undefined,
+                fullPage: fullPage === 'true',
+                raw: raw === 'true',
+                noDomainApi: noDomainApi === 'true',
+                lite: lite === 'true',
+                timeout: timeout ? parseInt(timeout, 10) : undefined,
+                captionImages: captionImages === 'true',
+                highlightQuery: highlightQuery,
+                highlightMaxChars: highlightMaxChars ? parseInt(highlightMaxChars, 10) : undefined,
+                // Prevent auto-escalation to browser unless render=true is explicitly requested.
+                // On 512MB containers, surprise browser launches cause OOM kills.
+                // Domain extractors (GitHub, Wikipedia, npm etc.) use HTTP APIs, not the browser.
+                noEscalate: !shouldRender,
+            };
+            // Auto-budget: default to 4000 tokens for API requests when no budget specified
+            // Opt-out: budget=0 explicitly disables. Lite mode disables auto-budget.
+            if (options.budget === undefined && !options.lite) {
+                options.budget = 4000;
+                res.setHeader('X-Auto-Budget', '4000');
+            }
+            // Inform the user if their request was degraded
+            if (isSoftLimited && !hasExtraUsage && render === 'true' && !hasActions) {
+                res.setHeader('X-Degraded', 'render=true downgraded to HTTP-only (quota exceeded)');
+            }
+            if (isSoftLimited && !hasExtraUsage && stealth === 'true') {
+                res.setHeader('X-Degraded', 'stealth=true downgraded (quota exceeded)');
+            }
+            if (isSoftLimited && !hasExtraUsage && screenshot === 'true') {
+                res.setHeader('X-Degraded', 'screenshot=true downgraded (quota exceeded)');
+            }
+            // Validate wait parameter
+            if (options.wait !== undefined && (isNaN(options.wait) || options.wait < 0 || options.wait > 60000)) {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Invalid "wait" parameter: must be between 0 and 60000ms',
+                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            // Validate format parameter
+            if (!['markdown', 'text', 'html', 'clean'].includes(options.format || '')) {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Invalid "format" parameter: must be "markdown", "text", "html", or "clean"',
+                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            const shouldStream = options.stream === true;
+            if (shouldStream) {
+                res.setHeader('X-Stream', 'true');
+                if (typeof res.flushHeaders === 'function') {
+                    res.flushHeaders();
+                }
+            }
+            // Fetch content — wrap in proxy context so tier limits are enforced automatically
+            const startTime = Date.now();
+            const userTier = req.auth?.tier || req.auth?.keyInfo?.tier || 'free';
+            const result = await proxyContextStorage.run({ userId: userId ?? undefined, tier: userTier }, () => peel(url, options));
+            const elapsed = Date.now() - startTime;
+            // --- BM25 Schema Template Extraction (GET, no LLM needed) ---
+            if (schema && typeof schema === 'string' && result.content) {
+                const template = getSchemaTemplate(schema);
+                if (template) {
+                    const { quickAnswer } = await import('../../core/quick-answer.js');
+                    const { smartExtractSchemaFields } = await import('../../core/schema-postprocess.js');
+                    const extracted = smartExtractSchemaFields(result.content, template.fields, quickAnswer, {
+                        pageTitle: result.title,
+                        pageUrl: result.url,
+                        metadata: result.metadata,
+                    });
+                    result.extracted = extracted;
+                }
+                else {
+                    // --- Custom JSON Schema → BM25 extraction (no LLM needed) ---
+                    // Supports: {"price": "number", "title": "string"} or {"price": "What is the price?"}
+                    let customFields = null;
+                    try {
+                        const parsed = JSON.parse(schema);
+                        if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
+                            customFields = {};
+                            for (const [key, typeOrQuestion] of Object.entries(parsed)) {
+                                const val = String(typeOrQuestion);
+                                // If the value looks like a question (has "?" or is >20 chars), use it directly
+                                if (val.includes('?') || val.length > 20) {
+                                    customFields[key] = val;
+                                }
+                                else {
+                                    // Convert type name to a natural question
+                                    const typeStr = val.toLowerCase();
+                                    const humanKey = key.replace(/([A-Z])/g, ' $1').toLowerCase().replace(/_/g, ' ');
+                                    if (typeStr === 'number' || typeStr === 'float' || typeStr === 'integer') {
+                                        customFields[key] = `What is the ${humanKey}? (as a number)`;
+                                    }
+                                    else if (typeStr === 'boolean') {
+                                        customFields[key] = `Is the ${humanKey} true or false?`;
+                                    }
+                                    else if (typeStr.includes('[]') || typeStr === 'array') {
+                                        customFields[key] = `What are the ${humanKey}? List all of them.`;
+                                    }
+                                    else {
+                                        customFields[key] = `What is the ${humanKey}?`;
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    catch { /* not valid JSON, ignore */ }
+                    if (customFields && Object.keys(customFields).length > 0) {
+                        const { quickAnswer } = await import('../../core/quick-answer.js');
+                        const { smartExtractSchemaFields } = await import('../../core/schema-postprocess.js');
+                        const extracted = smartExtractSchemaFields(result.content, customFields, quickAnswer, {
+                            pageTitle: result.title,
+                            pageUrl: result.url,
+                            metadata: result.metadata,
+                        });
+                        result.extracted = extracted;
+                    }
+                }
+            }
+            // Determine fetch type from the result method
+            const fetchType = result.method === 'stealth' ? 'stealth' :
+                result.method === 'browser' ? 'stealth' : 'basic';
+            // Log request to database (PostgreSQL only)
+            const pgStore = authStore;
+            // Log usage for BOTH API key auth AND JWT session auth
+            const logUserId = req.auth?.keyInfo?.accountId || req.user?.userId;
+            if (logUserId && typeof pgStore.pool !== 'undefined') {
+                pgStore.pool.query(`INSERT INTO usage_logs 
+            (user_id, endpoint, url, method, processing_time_ms, status_code, ip_address, user_agent, tokens_used)
+          VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, [
+                    logUserId,
+                    'fetch',
+                    url,
+                    fetchType,
+                    elapsed,
+                    200,
+                    req.ip || req.socket.remoteAddress,
+                    req.get('user-agent'),
+                    result?.tokens || null,
+                ]).catch((err) => {
+                    console.error('Failed to log request to usage_logs:', err);
+                });
+            }
+            // Track usage (check for trackBurstUsage method to detect PostgresAuthStore)
+            if (req.auth?.keyInfo?.key && typeof pgStore.trackBurstUsage === 'function') {
+                // Track burst usage (always)
+                await pgStore.trackBurstUsage(req.auth.keyInfo.key);
+                // If soft-limited with extra usage available, charge to extra usage
+                if (isSoftLimited && hasExtraUsage) {
+                    const extraResult = await pgStore.trackExtraUsage(req.auth.keyInfo.key, fetchType, url, elapsed, 200 // PeelResult doesn't include statusCode, assume success
+                    );
+                    if (extraResult.success) {
+                        res.setHeader('X-Extra-Usage-Charged', `$${extraResult.cost.toFixed(4)}`);
+                        res.setHeader('X-Extra-Usage-New-Balance', extraResult.newBalance.toFixed(2));
+                    }
+                    else {
+                        // Extra usage failed - fall back to soft limit
+                        res.setHeader('X-Degraded', 'Extra usage insufficient, degraded to soft limit');
+                    }
+                }
+                else if (!isSoftLimited) {
+                    // Normal weekly usage tracking
+                    await pgStore.trackUsage(req.auth.keyInfo.key, fetchType);
+                }
+                // If soft-limited WITHOUT extra usage, don't track (already over quota)
+            }
+            // Check usage alert (fire-and-forget, never block the response)
+            if (req.auth?.keyInfo?.accountId && typeof pgStore.pool !== 'undefined') {
+                // Automatic dual-threshold alerts (80% and 90%)
+                checkAndSendDualAlert(pgStore.pool, req.auth.keyInfo.accountId).catch(() => { });
+                // Legacy: user-configured single-threshold alert
+                try {
+                    const alertResult = await checkAndTriggerAlert(pgStore, req.auth.keyInfo.accountId);
+                    if (alertResult.shouldSendAlert && alertResult.usagePercent !== undefined) {
+                        await sendUsageAlertEmail({
+                            toEmail: alertResult.alertEmail || alertResult.userEmail,
+                            userName: alertResult.userName,
+                            usagePercent: alertResult.usagePercent,
+                            used: alertResult.used,
+                            total: alertResult.total,
+                            tier: alertResult.userTier,
+                        });
+                        // Mark alert as sent so we don't spam (rate-limited to once/week)
+                        await pgStore.pool.query('UPDATE users SET alert_sent_at = NOW() WHERE id = $1', [req.auth.keyInfo.accountId]);
+                    }
+                }
+                catch (alertErr) {
+                    // Never let alert errors affect the main response
+                    console.warn('[alert] Failed to check/send alert:', alertErr);
+                }
+            }
+            // Cache result (unless storeInCache is explicitly false or cache bypass requested)
+            if (storeInCache !== 'false' && !bypassCache) {
+                cache.set(cacheKey, {
+                    result,
+                    timestamp: Date.now(),
+                }, { ttl: cacheTtlMs });
+            }
+            // Apply ?detail=brief mode: truncate content and prepend TL;DR
+            if (detailMode === 'brief' && result.content) {
+                const words = result.content.split(/\s+/);
+                const truncatedWords = words.slice(0, 500);
+                const truncated = truncatedWords.join(' ');
+                // Extract TL;DR from first non-empty paragraph
+                const firstPara = result.content
+                    .split(/\n{2,}/)
+                    .map((p) => p.replace(/^#+\s*/, '').trim())
+                    .find((p) => p.length > 40 && !p.startsWith('!') && !p.startsWith('['));
+                const tldr = firstPara
+                    ? firstPara.replace(/\s+/g, ' ').slice(0, 300) + (firstPara.length > 300 ? '...' : '')
+                    : truncated.slice(0, 200) + '...';
+                result.content = `**TL;DR:** ${tldr}\n\n---\n\n${truncated}${words.length > 500 ? '\n\n*[Content truncated — use ?detail=full for complete output]*' : ''}`;
+                const tokenEstimate = Math.round(truncatedWords.length * 0.75);
+                res.setHeader('X-Detail-Mode', 'brief');
+                res.setHeader('X-Token-Estimate', tokenEstimate.toString());
+            }
+            // --- question → answer field (GET) ---
+            // When ?question= is provided, run quickAnswer() on the fetched content
+            // and expose the result as an `answer` field in the response.
+            const getAnswerResult = (question && typeof question === 'string' && result.content)
+                ? quickAnswer({ question, content: result.content, url: result.url })
+                : undefined;
+            // --- summary field (GET) ---
+            // When ?summary=true, return a truncated 500-word summary in a `summary` field.
+            const getSummaryText = (summary === 'true' && result.content)
+                ? extractSummary(result.content)
+                : undefined;
+            // Add usage headers (kept for backward compat; also surfaced in envelope metadata)
+            res.setHeader('X-Cache', 'MISS');
+            res.setHeader('X-Cache-Status', 'MISS');
+            res.setHeader('X-Credits-Used', '1');
+            res.setHeader('X-Processing-Time', elapsed.toString());
+            res.setHeader('X-Fetch-Type', fetchType);
+            // Proxy bandwidth usage headers — let users track their proxy consumption
+            if (userId) {
+                const proxyStats = getProxyUsage(userId, userTier);
+                res.setHeader('X-Proxy-Bytes-Used', proxyStats.used.toString());
+                res.setHeader('X-Proxy-Bytes-Remaining', proxyStats.remaining === -1 ? 'unlimited' : proxyStats.remaining.toString());
+                res.setHeader('X-Proxy-Limit', proxyStats.limit === -1 ? 'unlimited' : proxyStats.limit.toString());
+            }
+            // Method + cost headers — customers can see what tier they're using
+            res.setHeader('X-Method', result.method || 'simple');
+            res.setHeader('X-Method-Cost', methodCosts[result.method || 'simple'] ?? '0.005');
+            res.setHeader('X-Tokens', String(result.tokens || 0));
+            // Cache-Control: allow Cloudflare edge to cache successful GET responses for 60s
+            res.setHeader('Cache-Control', 'public, s-maxage=60, stale-while-revalidate=300');
+            // Response timing headers — let customers see exactly where time is spent
+            const timingFetch = result.timing?.fetch ?? 0;
+            const timingParse = (result.timing?.convert ?? 0) + (result.timing?.metadata ?? 0) + (result.timing?.prune ?? 0);
+            res.setHeader('X-Response-Time', `${elapsed}ms`);
+            res.setHeader('X-Fetch-Time', `${timingFetch}ms`);
+            res.setHeader('X-Parse-Time', `${timingParse}ms`);
+            res.setHeader('Server-Timing', `fetch;dur=${timingFetch}, parse;dur=${timingParse}, total;dur=${elapsed}`);
+            // Build response — extend result with optional answer/summary fields
+            const getResponseBody = { ...result };
+            // Ensure method + cost are always present in body
+            if (!getResponseBody.method)
+                getResponseBody.method = result.method || 'simple';
+            getResponseBody.cost = methodCosts[result.method || 'simple'] ?? '0.005';
+            if (getAnswerResult !== undefined)
+                getResponseBody.answer = getAnswerResult.answer;
+            if (getSummaryText !== undefined)
+                getResponseBody.summary = getSummaryText;
+            if (wantsEnvelope(req)) {
+                successResponse(res, getResponseBody, {
+                    requestId: req.requestId,
+                    processingTimeMs: elapsed,
+                    creditsUsed: 1,
+                    cached: false,
+                    fetchType,
+                });
+            }
+            else {
+                res.json(getResponseBody);
+            }
+        }
+        catch (error) {
+            const err = error;
+            // Log error to database (PostgreSQL only)
+            const pgStore = authStore;
+            if (req.auth?.keyInfo?.accountId && typeof pgStore.pool !== 'undefined') {
+                const url = req.query.url;
+                const render = req.query.render === 'true';
+                const fetchType = render ? 'stealth' : 'basic';
+                pgStore.pool.query(`INSERT INTO usage_logs 
+            (user_id, endpoint, url, method, status_code, error, ip_address, user_agent, tokens_used)
+          VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, [
+                    req.auth.keyInfo.accountId,
+                    'fetch',
+                    url,
+                    fetchType,
+                    500,
+                    err.message || 'Unknown error',
+                    req.ip || req.socket.remoteAddress,
+                    req.get('user-agent'),
+                    null,
+                ]).catch((logErr) => {
+                    console.error('Failed to log error to usage_logs:', logErr);
+                });
+            }
+            // SECURITY: Sanitize error messages to prevent information disclosure
+            if (res.headersSent)
+                return; // Timeout middleware already responded
+            const requestUrl = req.query.url;
+            if (err.code || err.name === 'TimeoutError' || err.name === 'BlockedError' || err.name === 'NetworkError' || err.name === 'WebPeelError') {
+                // WebPeelError from core library - safe to expose with helpful context
+                const { type, message, hint, docs } = buildFetchErrorMessage(err);
+                const statusCode = type === 'timeout' ? 504
+                    : type === 'blocked' ? 403
+                        : type === 'not_found' ? 404
+                            : type === 'network' || type === 'server_error' ? 502
+                                : 500;
+                res.status(statusCode).json({
+                    success: false,
+                    error: {
+                        type,
+                        message,
+                        url: requestUrl,
+                        ...(hint ? { hint } : {}),
+                        docs: docs || 'https://webpeel.dev/docs/errors',
+                    },
+                    requestId: req.requestId,
+                });
+            }
+            else {
+                // Unexpected error - generic message only
+                console.error('Fetch error:', err); // Log full error server-side
+                res.status(500).json({
+                    success: false,
+                    error: {
+                        type: 'unknown',
+                        message: 'An unexpected error occurred while fetching the URL. If this persists, check https://webpeel.dev/status',
+                        url: requestUrl,
+                        docs: 'https://webpeel.dev/docs/errors',
+                    },
+                    requestId: req.requestId,
+                });
+            }
+        }
+    });
+    // -----------------------------------------------------------------------
+    // POST /v1/fetch — same as GET but accepts JSON body with extract param
+    // POST /v2/scrape — alias with identical behaviour
+    // -----------------------------------------------------------------------
+    async function handlePostFetch(req, res) {
+        try {
+            // Require authentication — API key or JWT session
+            const postUserId = req.auth?.keyInfo?.accountId || req.user?.userId;
+            if (!postUserId) {
+                res.status(401).json({
+                    success: false,
+                    error: {
+                        type: 'unauthorized',
+                        message: 'API key required. Get one free at https://app.webpeel.dev/keys',
+                        hint: 'Get a free API key at https://app.webpeel.dev/keys',
+                        docs: 'https://webpeel.dev/docs/errors#unauthorized',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            const { url, render, wait, format, includeTags, excludeTags, images, location, languages, onlyMainContent, actions: rawActions, storeInCache: storeFlag, 
+            // Cache control
+            noCache: noCacheBody, cacheTtl: cacheTtlBody, 
+            // Inline extraction (BYOK)
+            extract, llmProvider, llmApiKey, llmModel, 
+            // Firecrawl-compatible formats array
+            formats, stream, 
+            // Extended peel options
+            budget, question, summary: summaryParam, readable, stealth, screenshot, maxTokens, selector, exclude, fullPage, raw, noDomainApi, lite, timeout, proxies, chunk, device, viewportWidth, viewportHeight, deviceScaleFactor, waitUntil, waitSelector, blockResources, cloaked, schema: bodySchema, highlightQuery: bodyHighlightQuery, highlightMaxChars: bodyHighlightMaxChars, } = req.body;
+            // --- Validate URL -------------------------------------------------------
+            if (!url || typeof url !== 'string') {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Missing or invalid "url" in request body.',
+                        hint: 'Send JSON: { "url": "https://example.com" }',
+                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            if (url.length > 2048) {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_url',
+                        message: 'URL too long (max 2048 characters)',
+                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            try {
+                new URL(url);
+            }
+            catch {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_url',
+                        message: 'Invalid URL format',
+                        hint: 'Ensure the URL includes a scheme (https://) and a valid hostname',
+                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            try {
+                validateUrlForSSRF(url);
+            }
+            catch (error) {
+                if (error instanceof SSRFError) {
+                    res.status(400).json({
+                        success: false,
+                        error: {
+                            type: 'forbidden_url',
+                            message: 'This URL is blocked for security. Localhost, private networks, and non-HTTP URLs are not allowed.',
+                            hint: 'See docs for allowed URL formats.',
+                            docs: 'https://webpeel.dev/docs/errors#forbidden-url',
+                        },
+                        requestId: req.requestId,
+                    });
+                    return;
+                }
+                throw error;
+            }
+            // --- Parse and normalize actions -----------------------------------------
+            let postActions;
+            if (rawActions !== undefined) {
+                try {
+                    postActions = normalizeActions(rawActions);
+                }
+                catch (e) {
+                    res.status(400).json({
+                        success: false,
+                        error: {
+                            type: 'invalid_request',
+                            message: `Invalid "actions" parameter: ${e.message}`,
+                            docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                        },
+                        requestId: req.requestId,
+                    });
+                    return;
+                }
+            }
+            // --- Cache bypass and lookup -------------------------------------------
+            const postBypassCache = noCacheBody === true || req.headers['cache-control'] === 'no-cache';
+            const postCacheTtlMs = typeof cacheTtlBody === 'number' ? cacheTtlBody * 1000 : 5 * 60 * 1000;
+            const postActionsKey = postActions ? JSON.stringify(postActions) : '';
+            const postCacheKey = `fetch:${url}:${render}:${wait}:${format}:${JSON.stringify(includeTags)}:${JSON.stringify(excludeTags)}:${images}:${location}:${JSON.stringify(languages)}:${onlyMainContent}:${stream}:${postActionsKey}:${budget}:${question}:${summaryParam}:${readable}:${stealth}:${screenshot}:${maxTokens}:${selector}:${JSON.stringify(exclude)}:${fullPage}:${raw}`;
+            if (!postBypassCache && !extract) {
+                const cached = cache.get(postCacheKey);
+                if (cached) {
+                    const cacheAge = Date.now() - cached.timestamp;
+                    if (cacheAge < postCacheTtlMs) {
+                        res.setHeader('X-Cache', 'HIT');
+                        res.setHeader('X-Cache-Status', 'HIT');
+                        res.setHeader('X-Cache-Age', Math.floor(cacheAge / 1000).toString());
+                        if (wantsEnvelope(req)) {
+                            successResponse(res, cached.result, {
+                                requestId: req.requestId,
+                                cached: true,
+                            });
+                        }
+                        else {
+                            res.json(cached.result);
+                        }
+                        return;
+                    }
+                }
+            }
+            // --- Resolve inline extract from body or Firecrawl-compatible formats ---
+            let resolvedExtract = extract;
+            if (!resolvedExtract && Array.isArray(formats)) {
+                const jsonFormat = formats.find((f) => (typeof f === 'object' && f !== null && f.type === 'json') ||
+                    (typeof f === 'string' && f === 'json'));
+                if (jsonFormat && typeof jsonFormat === 'object' && (jsonFormat.schema || jsonFormat.prompt)) {
+                    resolvedExtract = {
+                        schema: jsonFormat.schema,
+                        prompt: jsonFormat.prompt,
+                    };
+                }
+            }
+            // Resolve schema template names (e.g. "product", "article") to field objects
+            if (resolvedExtract && typeof resolvedExtract.schema === 'string') {
+                const tmpl = getSchemaTemplate(resolvedExtract.schema);
+                if (tmpl) {
+                    resolvedExtract = { ...resolvedExtract, schema: tmpl.fields };
+                }
+                else {
+                    // Try parsing as JSON string
+                    try {
+                        resolvedExtract = { ...resolvedExtract, schema: JSON.parse(resolvedExtract.schema) };
+                    }
+                    catch { /* leave as-is */ }
+                }
+            }
+            // Validate LLM params if extraction is requested
+            if (resolvedExtract && (resolvedExtract.schema || resolvedExtract.prompt)) {
+                if (!llmProvider || !VALID_LLM_PROVIDERS.includes(llmProvider)) {
+                    res.status(400).json({
+                        success: false,
+                        error: {
+                            type: 'invalid_request',
+                            message: `"llmProvider" is required for inline extraction and must be one of: ${VALID_LLM_PROVIDERS.join(', ')}`,
+                            docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                        },
+                        requestId: req.requestId,
+                    });
+                    return;
+                }
+                if (!llmApiKey || typeof llmApiKey !== 'string' || llmApiKey.trim().length === 0) {
+                    res.status(400).json({
+                        success: false,
+                        error: {
+                            type: 'invalid_request',
+                            message: 'Missing or invalid "llmApiKey" (BYOK required for inline extraction)',
+                            hint: 'Pass your LLM provider API key in the "llmApiKey" field',
+                            docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                        },
+                        requestId: req.requestId,
+                    });
+                    return;
+                }
+            }
+            // --- Build PeelOptions ---------------------------------------------------
+            const isSoftLimited = req.auth?.softLimited === true;
+            const hasExtraUsage = req.auth?.extraUsageAvailable === true;
+            const includeTagsArray = Array.isArray(includeTags) ? includeTags : undefined;
+            const excludeTagsArray = Array.isArray(excludeTags) ? excludeTags : undefined;
+            const languagesArray = Array.isArray(languages) ? languages : undefined;
+            const finalIncludeTags = onlyMainContent === true
+                ? ['main', 'article', '.content', '#content']
+                : includeTagsArray;
+            const resolvedFormat = format || 'markdown';
+            if (!['markdown', 'text', 'html', 'clean'].includes(resolvedFormat)) {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Invalid "format" parameter: must be "markdown", "text", "html", or "clean"',
+                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            const resolvedWait = typeof wait === 'number' ? wait : undefined;
+            if (resolvedWait !== undefined && (isNaN(resolvedWait) || resolvedWait < 0 || resolvedWait > 60000)) {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Invalid "wait" parameter: must be between 0 and 60000ms',
+                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            // When actions are present, force browser mode
+            const postHasActions = postActions && postActions.length > 0;
+            const postShouldRender = postHasActions || render === true;
+            // Normalize exclude: accept string (comma-separated) or string array
+            const excludeArray = exclude
+                ? (Array.isArray(exclude) ? exclude : exclude.split(',').map(s => s.trim()).filter(Boolean))
+                : undefined;
+            const options = {
+                render: (isSoftLimited && !hasExtraUsage && !postHasActions) ? false : postShouldRender,
+                wait: (isSoftLimited && !hasExtraUsage) ? 0 : resolvedWait,
+                format: resolvedFormat,
+                stream: stream === true,
+                includeTags: finalIncludeTags,
+                excludeTags: excludeTagsArray,
+                images: images === true,
+                actions: postActions,
+                location: location || languagesArray ? {
+                    country: location,
+                    languages: languagesArray,
+                } : undefined,
+                budget: typeof budget === 'number' ? budget : undefined,
+                question: question,
+                readable: readable === true,
+                stealth: (isSoftLimited && !hasExtraUsage) ? false : stealth === true,
+                screenshot: (isSoftLimited && !hasExtraUsage) ? false : (screenshot === true || (Array.isArray(formats) && formats.some((f) => (typeof f === 'string' ? f : f?.type) === 'screenshot'))),
+                maxTokens: typeof maxTokens === 'number' ? maxTokens : undefined,
+                selector: selector,
+                exclude: excludeArray,
+                fullPage: fullPage === true,
+                raw: raw === true,
+                noDomainApi: noDomainApi === true,
+                lite: lite === true,
+                timeout: typeof timeout === 'number' ? timeout : undefined,
+                proxies: Array.isArray(proxies) ? proxies : undefined,
+                device: device,
+                viewportWidth: typeof viewportWidth === 'number' ? viewportWidth : undefined,
+                viewportHeight: typeof viewportHeight === 'number' ? viewportHeight : undefined,
+                deviceScaleFactor: typeof deviceScaleFactor === 'number' ? deviceScaleFactor : undefined,
+                waitUntil: waitUntil,
+                waitSelector: waitSelector,
+                blockResources: Array.isArray(blockResources) ? blockResources : undefined,
+            };
+            if (cloaked)
+                options.cloaked = cloaked;
+            if (bodyHighlightQuery)
+                options.highlightQuery = bodyHighlightQuery;
+            if (bodyHighlightMaxChars)
+                options.highlightMaxChars = bodyHighlightMaxChars;
+            if (chunk)
+                options.chunk = chunk === true ? true : chunk;
+            // Auto-budget: default to 4000 tokens for API requests when no budget specified
+            // Opt-out: budget=0 explicitly disables. Lite mode disables auto-budget.
+            if (options.budget === undefined && !options.lite) {
+                options.budget = 4000;
+                res.setHeader('X-Auto-Budget', '4000');
+            }
+            if (isSoftLimited && !hasExtraUsage && render === true && !postHasActions) {
+                res.setHeader('X-Degraded', 'render=true downgraded to HTTP-only (quota exceeded)');
+            }
+            if (isSoftLimited && !hasExtraUsage && stealth === true) {
+                res.setHeader('X-Degraded', 'stealth=true downgraded (quota exceeded)');
+            }
+            if (isSoftLimited && !hasExtraUsage && screenshot === true) {
+                res.setHeader('X-Degraded', 'screenshot=true downgraded (quota exceeded)');
+            }
+            const shouldStream = options.stream === true;
+            if (shouldStream) {
+                res.setHeader('X-Stream', 'true');
+                if (typeof res.flushHeaders === 'function') {
+                    res.flushHeaders();
+                }
+            }
+            // --- Fetch content — wrap in proxy context so tier limits are enforced automatically
+            const startTime = Date.now();
+            const postUserTier = req.auth?.tier || req.auth?.keyInfo?.tier || 'free';
+            const result = await proxyContextStorage.run({ userId: postUserId ?? undefined, tier: postUserTier }, () => peel(url, options));
+            const elapsed = Date.now() - startTime;
+            // --- BM25 Schema Template Extraction (POST, no LLM needed) ---
+            if (bodySchema && typeof bodySchema === 'string' && result.content) {
+                const template = getSchemaTemplate(bodySchema);
+                if (template) {
+                    const { quickAnswer } = await import('../../core/quick-answer.js');
+                    const { smartExtractSchemaFields } = await import('../../core/schema-postprocess.js');
+                    const extracted = smartExtractSchemaFields(result.content, template.fields, quickAnswer, {
+                        pageTitle: result.title,
+                        pageUrl: result.url,
+                        metadata: result.metadata,
+                    });
+                    result.extracted = extracted;
+                }
+                else {
+                    // --- Custom JSON Schema → BM25 extraction (no LLM needed) ---
+                    // Supports: {"price": "number", "title": "string"} or {"price": "What is the price?"}
+                    let customFields = null;
+                    try {
+                        const parsed = JSON.parse(bodySchema);
+                        if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
+                            customFields = {};
+                            for (const [key, typeOrQuestion] of Object.entries(parsed)) {
+                                const val = String(typeOrQuestion);
+                                // If the value looks like a question (has "?" or is >20 chars), use it directly
+                                if (val.includes('?') || val.length > 20) {
+                                    customFields[key] = val;
+                                }
+                                else {
+                                    // Convert type name to a natural question
+                                    const typeStr = val.toLowerCase();
+                                    const humanKey = key.replace(/([A-Z])/g, ' $1').toLowerCase().replace(/_/g, ' ');
+                                    if (typeStr === 'number' || typeStr === 'float' || typeStr === 'integer') {
+                                        customFields[key] = `What is the ${humanKey}? (as a number)`;
+                                    }
+                                    else if (typeStr === 'boolean') {
+                                        customFields[key] = `Is the ${humanKey} true or false?`;
+                                    }
+                                    else if (typeStr.includes('[]') || typeStr === 'array') {
+                                        customFields[key] = `What are the ${humanKey}? List all of them.`;
+                                    }
+                                    else {
+                                        customFields[key] = `What is the ${humanKey}?`;
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    catch { /* not valid JSON, ignore */ }
+                    if (customFields && Object.keys(customFields).length > 0) {
+                        const { quickAnswer } = await import('../../core/quick-answer.js');
+                        const { smartExtractSchemaFields } = await import('../../core/schema-postprocess.js');
+                        const extracted = smartExtractSchemaFields(result.content, customFields, quickAnswer, {
+                            pageTitle: result.title,
+                            pageUrl: result.url,
+                            metadata: result.metadata,
+                        });
+                        result.extracted = extracted;
+                    }
+                }
+            }
+            // --- Inline extraction (post-fetch) -------------------------------------
+            let jsonData;
+            let extractTokensUsed;
+            if (resolvedExtract && (resolvedExtract.schema || resolvedExtract.prompt) && llmApiKey) {
+                const extractResult = await extractInlineJson(result.content, {
+                    schema: resolvedExtract.schema,
+                    prompt: resolvedExtract.prompt,
+                    llmProvider: llmProvider,
+                    llmApiKey: llmApiKey.trim(),
+                    llmModel,
+                });
+                jsonData = extractResult.data;
+                extractTokensUsed = extractResult.tokensUsed;
+            }
+            // --- Usage tracking (same as GET) ----------------------------------------
+            const fetchType = result.method === 'stealth' ? 'stealth' :
+                result.method === 'browser' ? 'stealth' : 'basic';
+            const pgStore = authStore;
+            if (req.auth?.keyInfo?.accountId && typeof pgStore.pool !== 'undefined') {
+                pgStore.pool.query(`INSERT INTO usage_logs
+            (user_id, endpoint, url, method, processing_time_ms, status_code, ip_address, user_agent, tokens_used)
+          VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, [
+                    req.auth.keyInfo.accountId,
+                    'fetch',
+                    url,
+                    fetchType,
+                    elapsed,
+                    200,
+                    req.ip || req.socket.remoteAddress,
+                    req.get('user-agent'),
+                    result?.tokens || null,
+                ]).catch((err) => {
+                    console.error('Failed to log request to usage_logs:', err);
+                });
+            }
+            if (req.auth?.keyInfo?.key && typeof pgStore.trackBurstUsage === 'function') {
+                await pgStore.trackBurstUsage(req.auth.keyInfo.key);
+                if (isSoftLimited && hasExtraUsage) {
+                    const extraResult = await pgStore.trackExtraUsage(req.auth.keyInfo.key, fetchType, url, elapsed, 200);
+                    if (extraResult.success) {
+                        res.setHeader('X-Extra-Usage-Charged', `$${extraResult.cost.toFixed(4)}`);
+                        res.setHeader('X-Extra-Usage-New-Balance', extraResult.newBalance.toFixed(2));
+                    }
+                    else {
+                        res.setHeader('X-Degraded', 'Extra usage insufficient, degraded to soft limit');
+                    }
+                }
+                else if (!isSoftLimited) {
+                    await pgStore.trackUsage(req.auth.keyInfo.key, fetchType);
+                }
+                // Automatic dual-threshold alerts (80% and 90%) — POST route
+                if (req.auth?.keyInfo?.accountId) {
+                    checkAndSendDualAlert(pgStore.pool, req.auth.keyInfo.accountId).catch(() => { });
+                }
+            }
+            // Cache result (skip extraction results — they depend on user's LLM keys)
+            if (storeFlag !== false && !postBypassCache && !resolvedExtract) {
+                cache.set(postCacheKey, { result, timestamp: Date.now() }, { ttl: postCacheTtlMs });
+            }
+            // --- question → answer field (POST) ---
+            // When question is provided, run quickAnswer() on the fetched content
+            // and expose the result as an `answer` field in the response.
+            const postAnswerResult = (question && typeof question === 'string' && result.content)
+                ? quickAnswer({ question, content: result.content, url: result.url })
+                : undefined;
+            // --- summary field (POST) ---
+            // When summary: true, return a truncated 500-word summary in a `summary` field.
+            const postSummaryText = (summaryParam === true && result.content)
+                ? extractSummary(result.content)
+                : undefined;
+            // --- Build response ------------------------------------------------------
+            // Headers kept for backward compat; also surfaced in envelope metadata.
+            res.setHeader('X-Cache', 'MISS');
+            res.setHeader('X-Cache-Status', 'MISS');
+            res.setHeader('X-Credits-Used', '1');
+            res.setHeader('X-Processing-Time', elapsed.toString());
+            res.setHeader('X-Fetch-Type', fetchType);
+            // Proxy bandwidth usage headers — let users track their proxy consumption
+            if (postUserId) {
+                const postProxyStats = getProxyUsage(postUserId, postUserTier);
+                res.setHeader('X-Proxy-Bytes-Used', postProxyStats.used.toString());
+                res.setHeader('X-Proxy-Bytes-Remaining', postProxyStats.remaining === -1 ? 'unlimited' : postProxyStats.remaining.toString());
+                res.setHeader('X-Proxy-Limit', postProxyStats.limit === -1 ? 'unlimited' : postProxyStats.limit.toString());
+            }
+            // Method + cost headers — customers can see what tier they're using
+            res.setHeader('X-Method', result.method || 'simple');
+            res.setHeader('X-Method-Cost', methodCosts[result.method || 'simple'] ?? '0.005');
+            res.setHeader('X-Tokens', String(result.tokens || 0));
+            // Response timing headers — let customers see exactly where time is spent
+            const postTimingFetch = result.timing?.fetch ?? 0;
+            const postTimingParse = (result.timing?.convert ?? 0) + (result.timing?.metadata ?? 0) + (result.timing?.prune ?? 0);
+            res.setHeader('X-Response-Time', `${elapsed}ms`);
+            res.setHeader('X-Fetch-Time', `${postTimingFetch}ms`);
+            res.setHeader('X-Parse-Time', `${postTimingParse}ms`);
+            res.setHeader('Server-Timing', `fetch;dur=${postTimingFetch}, parse;dur=${postTimingParse}, total;dur=${elapsed}`);
+            const responseBody = { ...result };
+            // Ensure method + cost are always present in body
+            if (!responseBody.method)
+                responseBody.method = result.method || 'simple';
+            responseBody.cost = methodCosts[result.method || 'simple'] ?? '0.005';
+            if (jsonData !== undefined) {
+                responseBody.json = jsonData;
+            }
+            if (extractTokensUsed) {
+                responseBody.extractTokensUsed = extractTokensUsed;
+            }
+            if (postAnswerResult !== undefined) {
+                responseBody.answer = postAnswerResult.answer;
+            }
+            if (postSummaryText !== undefined) {
+                responseBody.summary = postSummaryText;
+            }
+            // --- Multi-format response (formats array) --------------------------------
+            // When 'formats' is provided as an array, populate each requested format
+            // as a top-level field in the response. Backward-compatible: single-format
+            // responses continue to work unchanged via the 'format' param.
+            if (Array.isArray(formats) && formats.length > 0) {
+                for (const fmt of formats) {
+                    const fmtStr = typeof fmt === 'string' ? fmt : fmt?.type;
+                    switch (fmtStr) {
+                        case 'markdown':
+                            responseBody.markdown = result.content;
+                            break;
+                        case 'html':
+                            // If the user requested html format, result.content is already html.
+                            // Otherwise fall back to empty string (html is not exposed by peel()).
+                            responseBody.html = (resolvedFormat === 'html') ? result.content : '';
+                            break;
+                        case 'rawHtml':
+                            // rawHtml is not surfaced by PeelResult — return empty string.
+                            responseBody.rawHtml = '';
+                            break;
+                        case 'screenshot':
+                            responseBody.screenshot = result.screenshot || null;
+                            break;
+                        case 'links': {
+                            // PeelResult.links is already a deduplicated string[] — convert to
+                            // {url, text} objects. Fall back to cheerio extraction if links is empty.
+                            if (result.links && result.links.length > 0) {
+                                responseBody.links = result.links.map((url) => ({ url, text: '' }));
+                            }
+                            else {
+                                responseBody.links = extractLinks(result.content || '', result.url);
+                            }
+                            break;
+                        }
+                        case 'json':
+                            // Already handled above via resolvedExtract / jsonData
+                            break;
+                    }
+                }
+            }
+            if (wantsEnvelope(req)) {
+                successResponse(res, responseBody, {
+                    requestId: req.requestId,
+                    processingTimeMs: elapsed,
+                    creditsUsed: 1,
+                    cached: false,
+                    fetchType,
+                });
+            }
+            else {
+                res.json(responseBody);
+            }
+        }
+        catch (error) {
+            const err = error;
+            console.error('POST fetch/scrape error:', err);
+            if (res.headersSent)
+                return; // Timeout middleware already responded
+            const postUrl = req.body?.url;
+            if (err.code || err.name === 'TimeoutError' || err.name === 'BlockedError' || err.name === 'NetworkError' || err.name === 'WebPeelError') {
+                const { type, message, hint, docs } = buildFetchErrorMessage(err);
+                const statusCode = type === 'timeout' ? 504
+                    : type === 'blocked' ? 403
+                        : type === 'not_found' ? 404
+                            : type === 'network' || type === 'server_error' ? 502
+                                : 500;
+                res.status(statusCode).json({
+                    success: false,
+                    error: {
+                        type,
+                        message,
+                        url: postUrl,
+                        ...(hint ? { hint } : {}),
+                        docs: docs || 'https://webpeel.dev/docs/errors',
+                    },
+                    requestId: req.requestId,
+                });
+            }
+            else {
+                res.status(500).json({
+                    success: false,
+                    error: {
+                        type: 'unknown',
+                        message: 'An unexpected error occurred. If this persists, check https://webpeel.dev/status',
+                        url: postUrl,
+                        docs: 'https://webpeel.dev/docs/errors',
+                    },
+                    requestId: req.requestId,
+                });
+            }
+        }
+    }
+    router.post('/v1/fetch', handlePostFetch);
+    router.post('/v2/scrape', handlePostFetch);
+    return router;
+}