@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/server/routes/batch.js

@@ -0,0 +1,493 @@
+/**
+ * Batch scrape API - process multiple URLs concurrently
+ */
+import { Router } from 'express';
+import crypto from 'crypto';
+import { peel } from '../../index.js';
+import { sendWebhook, normalizeWebhook } from './webhooks.js';
+import { initSSE, sendSSE, endSSE, wantsSSE } from '../utils/sse.js';
+export function createBatchRouter(jobQueue) {
+    const router = Router();
+    /**
+     * POST /v1/batch/scrape - Submit batch of URLs
+     */
+    router.post('/v1/batch/scrape', async (req, res) => {
+        try {
+            const { urls, formats, extract, maxTokens, webhook, concurrency } = req.body;
+            // Validate required parameters
+            if (!urls || !Array.isArray(urls) || urls.length === 0) {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Missing or invalid "urls" parameter (must be non-empty array)',
+                        hint: 'Pass a non-empty array of URL strings in the "urls" field.',
+                        docs: 'https://webpeel.dev/docs/errors#invalid_request',
+                    },
+                    requestId: crypto.randomUUID(),
+                });
+                return;
+            }
+            // Limit batch size
+            if (urls.length > 100) {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Batch size too large (max 100 URLs)',
+                        hint: 'Split your request into batches of 100 URLs or fewer.',
+                        docs: 'https://webpeel.dev/docs/errors#invalid_request',
+                    },
+                    requestId: crypto.randomUUID(),
+                });
+                return;
+            }
+            // Validate URLs
+            for (const url of urls) {
+                if (typeof url !== 'string') {
+                    res.status(400).json({
+                        success: false,
+                        error: {
+                            type: 'invalid_request',
+                            message: 'All URLs must be strings',
+                            hint: 'Each element in the "urls" array must be a string.',
+                            docs: 'https://webpeel.dev/docs/errors#invalid_request',
+                        },
+                        requestId: crypto.randomUUID(),
+                    });
+                    return;
+                }
+                try {
+                    new URL(url);
+                }
+                catch {
+                    res.status(400).json({
+                        success: false,
+                        error: {
+                            type: 'invalid_url',
+                            message: `Invalid URL format: ${url}`,
+                            hint: 'Ensure each URL includes a scheme (https://) and a valid hostname.',
+                            docs: 'https://webpeel.dev/docs/errors#invalid_url',
+                        },
+                        requestId: crypto.randomUUID(),
+                    });
+                    return;
+                }
+            }
+            const ownerId = req.auth?.keyInfo?.accountId;
+            // ── SSE streaming path ────────────────────────────────────────────────
+            if (wantsSSE(req)) {
+                const normalizedWebhook = webhook ? normalizeWebhook(webhook) : undefined;
+                const job = await jobQueue.createJob('batch', normalizedWebhook, ownerId);
+                await jobQueue.updateJob(job.id, { total: urls.length });
+                // Set SSE headers (X-Request-Id already set by global middleware)
+                initSSE(res);
+                sendSSE(res, 'started', {
+                    batchId: job.id,
+                    totalUrls: urls.length,
+                });
+                let closed = false;
+                const heartbeat = setInterval(() => {
+                    if (!closed)
+                        res.write('event: ping\ndata: {}\n\n');
+                }, 15_000);
+                req.on('close', () => {
+                    closed = true;
+                    clearInterval(heartbeat);
+                });
+                const startTime = Date.now();
+                let completedCount = 0;
+                let failedCount = 0;
+                try {
+                    jobQueue.updateJob(job.id, { status: 'processing' });
+                    const peelOptions = {
+                        format: formats?.[0] || 'markdown',
+                        extract,
+                        maxTokens,
+                    };
+                    // Process URLs with bounded concurrency (caller-specified, max 10)
+                    const results = new Array(urls.length);
+                    const maxConcurrent = Math.min(Math.max(parseInt(concurrency) || 5, 1), 10);
+                    let activeCount = 0;
+                    let urlIndex = 0;
+                    const runBatch = async () => {
+                        while (urlIndex < urls.length) {
+                            // Check cancellation
+                            const currentJob = await jobQueue.getJob(job.id);
+                            if (currentJob?.status === 'cancelled')
+                                break;
+                            while (activeCount >= maxConcurrent) {
+                                await new Promise(resolve => setTimeout(resolve, 50));
+                            }
+                            const url = urls[urlIndex];
+                            const index = urlIndex;
+                            urlIndex++;
+                            activeCount++;
+                            (async () => {
+                                try {
+                                    const result = await peel(url, peelOptions);
+                                    results[index] = result;
+                                    completedCount++;
+                                    jobQueue.updateJob(job.id, {
+                                        completed: completedCount + failedCount,
+                                        creditsUsed: completedCount + failedCount,
+                                    });
+                                    if (!closed) {
+                                        sendSSE(res, 'result', {
+                                            url,
+                                            content: result.content,
+                                            metadata: result.metadata,
+                                            index,
+                                        });
+                                    }
+                                }
+                                catch (err) {
+                                    failedCount++;
+                                    results[index] = { url, error: err.message || 'Unknown error' };
+                                    jobQueue.updateJob(job.id, {
+                                        completed: completedCount + failedCount,
+                                        creditsUsed: completedCount + failedCount,
+                                    });
+                                    if (!closed) {
+                                        sendSSE(res, 'error', {
+                                            url,
+                                            error: 'FETCH_ERROR',
+                                            message: err.message || 'Unknown error',
+                                            index,
+                                        });
+                                    }
+                                }
+                                finally {
+                                    activeCount--;
+                                }
+                            })();
+                        }
+                        // Wait for all in-flight requests
+                        while (activeCount > 0) {
+                            await new Promise(resolve => setTimeout(resolve, 50));
+                        }
+                    };
+                    await runBatch();
+                    jobQueue.updateJob(job.id, {
+                        status: 'completed',
+                        data: results,
+                    });
+                    if (!closed) {
+                        sendSSE(res, 'done', {
+                            batchId: job.id,
+                            completed: completedCount,
+                            failed: failedCount,
+                            duration: Date.now() - startTime,
+                        });
+                    }
+                }
+                catch (error) {
+                    jobQueue.updateJob(job.id, {
+                        status: 'failed',
+                        error: error.message || 'Unknown error',
+                    });
+                    if (!closed) {
+                        sendSSE(res, 'error', {
+                            error: 'BATCH_FAILED',
+                            message: error.message || 'Unknown error',
+                        });
+                    }
+                }
+                finally {
+                    clearInterval(heartbeat);
+                    if (!closed)
+                        endSSE(res);
+                }
+                return;
+            }
+            // ── Regular async job path (backward compat) ─────────────────────────
+            const normalizedWebhook2 = webhook ? normalizeWebhook(webhook) : undefined;
+            const job = await jobQueue.createJob('batch', normalizedWebhook2, ownerId);
+            await jobQueue.updateJob(job.id, {
+                total: urls.length,
+            });
+            // Start batch processing in background
+            setImmediate(async () => {
+                try {
+                    // Update job to processing
+                    jobQueue.updateJob(job.id, { status: 'processing' });
+                    // Send started webhook
+                    if (normalizedWebhook2) {
+                        await sendWebhook(normalizedWebhook2, 'started', {
+                            jobId: job.id,
+                            total: urls.length,
+                        });
+                    }
+                    // Build peel options
+                    const peelOptions = {
+                        format: formats?.[0] || 'markdown',
+                        extract,
+                        maxTokens,
+                    };
+                    // Process URLs with semaphore (caller-specified, max 10 concurrent)
+                    const results = [];
+                    const maxConcurrent = Math.min(Math.max(parseInt(concurrency) || 5, 1), 10);
+                    let activeCount = 0;
+                    let urlIndex = 0;
+                    const processBatch = async () => {
+                        while (urlIndex < urls.length) {
+                            // Check if job was cancelled
+                            const currentJob = await jobQueue.getJob(job.id);
+                            if (currentJob?.status === 'cancelled') {
+                                return;
+                            }
+                            // Wait for available slot
+                            while (activeCount >= maxConcurrent) {
+                                await new Promise(resolve => setTimeout(resolve, 100));
+                            }
+                            const url = urls[urlIndex];
+                            const index = urlIndex;
+                            urlIndex++;
+                            activeCount++;
+                            // Process URL
+                            (async () => {
+                                try {
+                                    const result = await peel(url, peelOptions);
+                                    results[index] = result;
+                                    // Update progress
+                                    const completed = results.filter(r => r !== undefined).length;
+                                    jobQueue.updateJob(job.id, {
+                                        completed,
+                                        creditsUsed: completed,
+                                    });
+                                    // Send page webhook
+                                    if (normalizedWebhook2) {
+                                        sendWebhook(normalizedWebhook2, 'page', {
+                                            jobId: job.id,
+                                            url,
+                                            completed,
+                                            total: urls.length,
+                                        }).catch(() => { }); // Fire and forget
+                                    }
+                                }
+                                catch (error) {
+                                    // Store error as result
+                                    results[index] = {
+                                        url,
+                                        error: error.message || 'Unknown error',
+                                    };
+                                    // Update progress
+                                    const completed = results.filter(r => r !== undefined).length;
+                                    jobQueue.updateJob(job.id, {
+                                        completed,
+                                        creditsUsed: completed,
+                                    });
+                                }
+                                finally {
+                                    activeCount--;
+                                }
+                            })();
+                        }
+                        // Wait for all tasks to complete
+                        while (activeCount > 0) {
+                            await new Promise(resolve => setTimeout(resolve, 100));
+                        }
+                    };
+                    await processBatch();
+                    // Update job with results
+                    jobQueue.updateJob(job.id, {
+                        status: 'completed',
+                        data: results,
+                    });
+                    // Send completed webhook and store delivery result
+                    if (normalizedWebhook2) {
+                        const delivery = await sendWebhook(normalizedWebhook2, 'completed', {
+                            jobId: job.id,
+                            total: urls.length,
+                            completed: results.length,
+                        });
+                        if (delivery) {
+                            jobQueue.updateJob(job.id, { webhookDelivery: delivery });
+                        }
+                    }
+                }
+                catch (error) {
+                    // Update job with error
+                    jobQueue.updateJob(job.id, {
+                        status: 'failed',
+                        error: error.message || 'Unknown error',
+                    });
+                    // Send failed webhook
+                    if (normalizedWebhook2) {
+                        await sendWebhook(normalizedWebhook2, 'failed', {
+                            jobId: job.id,
+                            error: error.message || 'Unknown error',
+                        });
+                    }
+                }
+            });
+            // Return job ID immediately
+            res.status(202).json({
+                success: true,
+                id: job.id,
+                url: `/v1/batch/scrape/${job.id}`,
+            });
+        }
+        catch (error) {
+            console.error('Batch scrape creation error:', error);
+            res.status(500).json({
+                success: false,
+                error: {
+                    type: 'internal_error',
+                    message: 'Failed to create batch scrape job',
+                    docs: 'https://webpeel.dev/docs/errors#internal_error',
+                },
+                requestId: crypto.randomUUID(),
+            });
+        }
+    });
+    /**
+     * GET /v1/batch/scrape/:id - Get batch scrape status + results
+     */
+    router.get('/v1/batch/scrape/:id', async (req, res) => {
+        try {
+            const id = req.params.id;
+            const job = await jobQueue.getJob(id);
+            if (!job) {
+                res.status(404).json({
+                    success: false,
+                    error: {
+                        type: 'not_found',
+                        message: 'Job not found',
+                        docs: 'https://webpeel.dev/docs/errors#not_found',
+                    },
+                    requestId: crypto.randomUUID(),
+                });
+                return;
+            }
+            // SECURITY: Verify the requester owns this job
+            const requestOwnerId = req.auth?.keyInfo?.accountId;
+            if (job.ownerId && requestOwnerId && job.ownerId !== requestOwnerId) {
+                res.status(404).json({
+                    success: false,
+                    error: {
+                        type: 'not_found',
+                        message: 'Job not found',
+                        docs: 'https://webpeel.dev/docs/errors#not_found',
+                    },
+                    requestId: crypto.randomUUID(),
+                });
+                return;
+            }
+            if (job.type !== 'batch') {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Job is not a batch scrape job',
+                        docs: 'https://webpeel.dev/docs/errors#invalid_request',
+                    },
+                    requestId: crypto.randomUUID(),
+                });
+                return;
+            }
+            res.json({
+                success: true,
+                status: job.status,
+                progress: job.progress,
+                total: job.total,
+                completed: job.completed,
+                creditsUsed: job.creditsUsed,
+                data: job.data,
+                error: job.error,
+                expiresAt: job.expiresAt,
+                ...(job.webhookDelivery ? { webhook: job.webhookDelivery } : {}),
+            });
+        }
+        catch (error) {
+            console.error('Get batch scrape error:', error);
+            res.status(500).json({
+                success: false,
+                error: {
+                    type: 'internal_error',
+                    message: 'Failed to retrieve job',
+                    docs: 'https://webpeel.dev/docs/errors#internal_error',
+                },
+                requestId: crypto.randomUUID(),
+            });
+        }
+    });
+    /**
+     * DELETE /v1/batch/scrape/:id - Cancel batch scrape job
+     */
+    router.delete('/v1/batch/scrape/:id', async (req, res) => {
+        try {
+            const id = req.params.id;
+            const job = await jobQueue.getJob(id);
+            if (!job) {
+                res.status(404).json({
+                    success: false,
+                    error: {
+                        type: 'not_found',
+                        message: 'Job not found',
+                        docs: 'https://webpeel.dev/docs/errors#not_found',
+                    },
+                    requestId: crypto.randomUUID(),
+                });
+                return;
+            }
+            // SECURITY: Verify the requester owns this job
+            const requestOwnerId = req.auth?.keyInfo?.accountId;
+            if (job.ownerId && requestOwnerId && job.ownerId !== requestOwnerId) {
+                res.status(404).json({
+                    success: false,
+                    error: {
+                        type: 'not_found',
+                        message: 'Job not found',
+                        docs: 'https://webpeel.dev/docs/errors#not_found',
+                    },
+                    requestId: crypto.randomUUID(),
+                });
+                return;
+            }
+            if (job.type !== 'batch') {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Job is not a batch scrape job',
+                        docs: 'https://webpeel.dev/docs/errors#invalid_request',
+                    },
+                    requestId: crypto.randomUUID(),
+                });
+                return;
+            }
+            const cancelled = await jobQueue.cancelJob(id);
+            if (!cancelled) {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Job cannot be cancelled (already completed or failed)',
+                        hint: 'Only pending or processing jobs can be cancelled.',
+                        docs: 'https://webpeel.dev/docs/errors#invalid_request',
+                    },
+                    requestId: crypto.randomUUID(),
+                });
+                return;
+            }
+            res.json({
+                success: true,
+                message: 'Job cancelled',
+            });
+        }
+        catch (error) {
+            console.error('Cancel batch scrape error:', error);
+            res.status(500).json({
+                success: false,
+                error: {
+                    type: 'internal_error',
+                    message: 'Failed to cancel job',
+                    docs: 'https://webpeel.dev/docs/errors#internal_error',
+                },
+                requestId: crypto.randomUUID(),
+            });
+        }
+    });
+    return router;
+}

package/dist/server/routes/cache-warm.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Cache Pre-Warming Routes
+ *
+ * GET /internal/popular-urls  — Top N URLs fetched in the last 24h (for CF Worker)
+ * GET /internal/cache-status  — Current warmer state (warmed URLs, last run time)
+ *
+ * Both routes are mounted BEFORE auth middleware so they're accessible internally.
+ * /internal/popular-urls is protected by CACHE_WARM_SECRET bearer token when set.
+ *
+ * startCacheWarmer() — server-side self-warming (opt-in via ENABLE_CACHE_WARM=true)
+ */
+import { Router } from 'express';
+import pg from 'pg';
+export declare function createCacheWarmRouter(pool: pg.Pool | null): Router;
+/**
+ * startCacheWarmer — server-side self-warming (fallback when no CF Worker).
+ *
+ * Every `intervalMs` (default 2 min):
+ *  1. Queries /internal/popular-urls (via the DB, not HTTP)
+ *  2. Fetches each URL through /r/<url> with concurrency 5
+ *  3. Updates warmerState for /internal/cache-status
+ *
+ * Only started if ENABLE_CACHE_WARM=true.
+ */
+export declare function startCacheWarmer(pool: pg.Pool | null, intervalMs?: number): void;