@doppelgangerdev/doppelganger 0.5.7 → 0.5.8

package/scrape.js

@@ -1,9 +1,10 @@
 const { chromium } = require('playwright');
-const { JSDOM } = require('jsdom');
 const fs = require('fs');
 const path = require('path');
-const { getProxySelection } = require('./proxy-rotation');
-const { selectUserAgent } = require('./user-agent-settings');
+const { spawn } = require('child_process');
+const { getProxySelection } = require('./proxy-rotation');
+const { selectUserAgent } = require('./user-agent-settings');
+const { formatHTML } = require('./html-utils');
 
 const STORAGE_STATE_PATH = path.join(__dirname, 'storage_state.json');
 const STORAGE_STATE_FILE = (() => {
@@ -26,8 +27,8 @@ const csvEscape = (value) => {
     return text;
 };
 
-const toCsvString = (raw) => {
-    if (raw === undefined || raw === null) return '';
+const toCsvString = (raw) => {
+    if (raw === undefined || raw === null) return '';
     if (typeof raw === 'string') {
         const trimmed = raw.trim();
         if (trimmed.startsWith('{') || trimmed.startsWith('[')) {
@@ -64,39 +65,39 @@ const toCsvString = (raw) => {
         const obj = row && typeof row === 'object' ? row : {};
         return allKeys.map((key) => csvEscape(obj[key])).join(',');
     });
-    return [headerLine, ...lines].join('\n');
-};
-
-const parseBooleanFlag = (value) => {
-    if (typeof value === 'boolean') return value;
-    if (value === undefined || value === null) return false;
-    const normalized = String(value).toLowerCase();
-    return normalized === 'true' || normalized === '1';
-};
-
-async function handleScrape(req, res) {
+    return [headerLine, ...lines].join('\n');
+};
+
+const parseBooleanFlag = (value) => {
+    if (typeof value === 'boolean') return value;
+    if (value === undefined || value === null) return false;
+    const normalized = String(value).toLowerCase();
+    return normalized === 'true' || normalized === '1';
+};
+
+async function handleScrape(req, res) {
     const url = req.body.url || req.query.url;
     const customHeaders = req.body.headers || {};
     const userSelector = req.body.selector || req.query.selector;
     const waitInput = req.body.wait || req.query.wait;
     const waitTime = waitInput ? parseFloat(waitInput) * 1000 : 2000;
-    const rotateUserAgents = req.body.rotateUserAgents || req.query.rotateUserAgents || false;
-    const rotateViewportRaw = req.body.rotateViewport ?? req.query.rotateViewport;
-    const rotateViewport = String(rotateViewportRaw).toLowerCase() === 'true' || rotateViewportRaw === true;
-    const runId = req.body.runId || req.query.runId || null;
-    const captureRunId = runId ? String(runId) : `run_${Date.now()}_unknown`;
+    const rotateUserAgents = req.body.rotateUserAgents || req.query.rotateUserAgents || false;
+    const rotateViewportRaw = req.body.rotateViewport ?? req.query.rotateViewport;
+    const rotateViewport = String(rotateViewportRaw).toLowerCase() === 'true' || rotateViewportRaw === true;
+    const runId = req.body.runId || req.query.runId || null;
+    const captureRunId = runId ? String(runId) : `run_${Date.now()}_unknown`;
     const rotateProxiesRaw = req.body.rotateProxies ?? req.query.rotateProxies;
     const rotateProxies = String(rotateProxiesRaw).toLowerCase() === 'true' || rotateProxiesRaw === true;
-    const includeShadowDomRaw = req.body.includeShadowDom ?? req.query.includeShadowDom;
-    const includeShadowDom = includeShadowDomRaw === undefined
-        ? true
-        : !(String(includeShadowDomRaw).toLowerCase() === 'false' || includeShadowDomRaw === false);
-    const disableRecordingRaw = req.body.disableRecording ?? req.query.disableRecording;
-    const disableRecording = parseBooleanFlag(disableRecordingRaw);
-    const statelessExecutionRaw = req.body.statelessExecution ?? req.query.statelessExecution;
-    const statelessExecution = parseBooleanFlag(statelessExecutionRaw);
-    const extractionScript = req.body.extractionScript || req.query.extractionScript;
-    const extractionFormat = (req.body.extractionFormat || req.query.extractionFormat) === 'csv' ? 'csv' : 'json';
+    const includeShadowDomRaw = req.body.includeShadowDom ?? req.query.includeShadowDom;
+    const includeShadowDom = includeShadowDomRaw === undefined
+        ? true
+        : !(String(includeShadowDomRaw).toLowerCase() === 'false' || includeShadowDomRaw === false);
+    const disableRecordingRaw = req.body.disableRecording ?? req.query.disableRecording;
+    const disableRecording = parseBooleanFlag(disableRecordingRaw);
+    const statelessExecutionRaw = req.body.statelessExecution ?? req.query.statelessExecution;
+    const statelessExecution = parseBooleanFlag(statelessExecutionRaw);
+    const extractionScript = req.body.extractionScript || req.query.extractionScript;
+    const extractionFormat = (req.body.extractionFormat || req.query.extractionFormat) === 'csv' ? 'csv' : 'json';
 
     if (!url) {
         return res.status(400).json({ error: 'URL is required.' });
@@ -104,24 +105,24 @@ async function handleScrape(req, res) {
 
     console.log(`Scraping: ${url}`);
 
-    const selectedUA = selectUserAgent(rotateUserAgents);
-
-    let browser;
-    let context;
-    let page;
-    try {
+    const selectedUA = await selectUserAgent(rotateUserAgents);
+
+    let browser;
+    let context;
+    let page;
+    try {
         // Use 'chrome' channel to use a real installed browser instead of default Chromium
         const launchOptions = {
             headless: true,
             channel: 'chrome',
-            args: [
-                '--no-sandbox',
-                '--disable-setuid-sandbox',
-                '--disable-dev-shm-usage',
-                '--disable-blink-features=AutomationControlled',
-                '--hide-scrollbars',
-                '--mute-audio'
-            ]
+            args: [
+                '--no-sandbox',
+                '--disable-setuid-sandbox',
+                '--disable-dev-shm-usage',
+                '--disable-blink-features=AutomationControlled',
+                '--hide-scrollbars',
+                '--mute-audio'
+            ]
         };
         const selection = getProxySelection(rotateProxies);
         if (selection.proxy) {
@@ -130,102 +131,102 @@ async function handleScrape(req, res) {
         console.log(`[PROXY] Mode: ${selection.mode}; Target: ${selection.proxy ? selection.proxy.server : 'host_ip'}`);
         browser = await chromium.launch(launchOptions);
 
-        const recordingsDir = path.join(__dirname, 'data', 'recordings');
-        if (!fs.existsSync(recordingsDir)) {
-            fs.mkdirSync(recordingsDir, { recursive: true });
-        }
-
-        const viewport = rotateViewport
-            ? { width: 1280 + Math.floor(Math.random() * 640), height: 720 + Math.floor(Math.random() * 360) }
-            : { width: 1366, height: 768 };
-
-        const contextOptions = {
-            userAgent: selectedUA,
-            extraHTTPHeaders: customHeaders,
-            viewport,
-            deviceScaleFactor: 1,
-            locale: 'en-US',
-            timezoneId: 'America/New_York',
-            colorScheme: 'dark',
-            permissions: ['geolocation']
-        };
-
-        const shouldUseStorageState = !statelessExecution && fs.existsSync(STORAGE_STATE_FILE);
-        if (shouldUseStorageState) {
-            contextOptions.storageState = STORAGE_STATE_FILE;
-        }
-
-        if (!disableRecording) {
-            contextOptions.recordVideo = { dir: recordingsDir, size: viewport };
-        }
-
-        context = await browser.newContext(contextOptions);
-
-        // Manual WebDriver Patch
-        await context.addInitScript(() => {
-            Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
-        });
-        await context.addInitScript(() => {
-            const cursorId = 'dg-cursor-overlay';
-            const dotId = 'dg-click-dot';
-            if (document.getElementById(cursorId)) return;
-            const cursor = document.createElement('div');
-            cursor.id = cursorId;
-            cursor.style.cssText = [
-                'position:fixed',
-                'top:0',
-                'left:0',
-                'width:18px',
-                'height:18px',
-                'margin-left:-9px',
-                'margin-top:-9px',
-                'border:2px solid rgba(56,189,248,0.7)',
-                'background:rgba(56,189,248,0.25)',
-                'border-radius:50%',
-                'box-shadow:0 0 10px rgba(56,189,248,0.6)',
-                'pointer-events:none',
-                'z-index:2147483647',
-                'transform:translate3d(0,0,0)',
-                'transition:transform 60ms ease-out'
-            ].join(';');
-            const dot = document.createElement('div');
-            dot.id = dotId;
-            dot.style.cssText = [
-                'position:fixed',
-                'top:0',
-                'left:0',
-                'width:10px',
-                'height:10px',
-                'margin-left:-5px',
-                'margin-top:-5px',
-                'background:rgba(239,68,68,0.9)',
-                'border-radius:50%',
-                'box-shadow:0 0 12px rgba(239,68,68,0.8)',
-                'pointer-events:none',
-                'z-index:2147483647',
-                'opacity:0',
-                'transform:translate3d(0,0,0) scale(0.6)',
-                'transition:opacity 120ms ease, transform 120ms ease'
-            ].join(';');
-            document.documentElement.appendChild(cursor);
-            document.documentElement.appendChild(dot);
-            const move = (x, y) => {
-                cursor.style.transform = `translate3d(${x}px, ${y}px, 0)`;
-            };
-            window.addEventListener('mousemove', (e) => move(e.clientX, e.clientY), { passive: true });
-            window.addEventListener('click', (e) => {
-                dot.style.left = `${e.clientX}px`;
-                dot.style.top = `${e.clientY}px`;
-                dot.style.opacity = '1';
-                dot.style.transform = 'translate3d(0,0,0) scale(1)';
-                cursor.style.transform = `translate3d(${e.clientX}px, ${e.clientY}px, 0) scale(0.65)`;
-                setTimeout(() => {
-                    dot.style.opacity = '0';
-                    dot.style.transform = 'translate3d(0,0,0) scale(0.6)';
-                    cursor.style.transform = `translate3d(${e.clientX}px, ${e.clientY}px, 0) scale(1)`;
-                }, 180);
-            }, true);
-        });
+        const recordingsDir = path.join(__dirname, 'data', 'recordings');
+        if (!fs.existsSync(recordingsDir)) {
+            fs.mkdirSync(recordingsDir, { recursive: true });
+        }
+
+        const viewport = rotateViewport
+            ? { width: 1280 + Math.floor(Math.random() * 640), height: 720 + Math.floor(Math.random() * 360) }
+            : { width: 1366, height: 768 };
+
+        const contextOptions = {
+            userAgent: selectedUA,
+            extraHTTPHeaders: customHeaders,
+            viewport,
+            deviceScaleFactor: 1,
+            locale: 'en-US',
+            timezoneId: 'America/New_York',
+            colorScheme: 'dark',
+            permissions: ['geolocation']
+        };
+
+        const shouldUseStorageState = !statelessExecution && fs.existsSync(STORAGE_STATE_FILE);
+        if (shouldUseStorageState) {
+            contextOptions.storageState = STORAGE_STATE_FILE;
+        }
+
+        if (!disableRecording) {
+            contextOptions.recordVideo = { dir: recordingsDir, size: viewport };
+        }
+
+        context = await browser.newContext(contextOptions);
+
+        // Manual WebDriver Patch
+        await context.addInitScript(() => {
+            Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
+        });
+        await context.addInitScript(() => {
+            const cursorId = 'dg-cursor-overlay';
+            const dotId = 'dg-click-dot';
+            if (document.getElementById(cursorId)) return;
+            const cursor = document.createElement('div');
+            cursor.id = cursorId;
+            cursor.style.cssText = [
+                'position:fixed',
+                'top:0',
+                'left:0',
+                'width:18px',
+                'height:18px',
+                'margin-left:-9px',
+                'margin-top:-9px',
+                'border:2px solid rgba(56,189,248,0.7)',
+                'background:rgba(56,189,248,0.25)',
+                'border-radius:50%',
+                'box-shadow:0 0 10px rgba(56,189,248,0.6)',
+                'pointer-events:none',
+                'z-index:2147483647',
+                'transform:translate3d(0,0,0)',
+                'transition:transform 60ms ease-out'
+            ].join(';');
+            const dot = document.createElement('div');
+            dot.id = dotId;
+            dot.style.cssText = [
+                'position:fixed',
+                'top:0',
+                'left:0',
+                'width:10px',
+                'height:10px',
+                'margin-left:-5px',
+                'margin-top:-5px',
+                'background:rgba(239,68,68,0.9)',
+                'border-radius:50%',
+                'box-shadow:0 0 12px rgba(239,68,68,0.8)',
+                'pointer-events:none',
+                'z-index:2147483647',
+                'opacity:0',
+                'transform:translate3d(0,0,0) scale(0.6)',
+                'transition:opacity 120ms ease, transform 120ms ease'
+            ].join(';');
+            document.documentElement.appendChild(cursor);
+            document.documentElement.appendChild(dot);
+            const move = (x, y) => {
+                cursor.style.transform = `translate3d(${x}px, ${y}px, 0)`;
+            };
+            window.addEventListener('mousemove', (e) => move(e.clientX, e.clientY), { passive: true });
+            window.addEventListener('click', (e) => {
+                dot.style.left = `${e.clientX}px`;
+                dot.style.top = `${e.clientY}px`;
+                dot.style.opacity = '1';
+                dot.style.transform = 'translate3d(0,0,0) scale(1)';
+                cursor.style.transform = `translate3d(${e.clientX}px, ${e.clientY}px, 0) scale(0.65)`;
+                setTimeout(() => {
+                    dot.style.opacity = '0';
+                    dot.style.transform = 'translate3d(0,0,0) scale(0.6)';
+                    cursor.style.transform = `translate3d(${e.clientX}px, ${e.clientY}px, 0) scale(1)`;
+                }, 180);
+            }, true);
+        });
         if (includeShadowDom) {
             await context.addInitScript(() => {
                 if (!Element.prototype.attachShadow) return;
@@ -237,7 +238,7 @@ async function handleScrape(req, res) {
             });
         }
 
-        page = await context.newPage();
+        page = await context.newPage();
 
         await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 });
 
@@ -345,112 +346,76 @@ async function handleScrape(req, res) {
 
         const runExtractionScript = async (script, html, pageUrl) => {
             if (!script || typeof script !== 'string') return { result: undefined, logs: [] };
-            try {
-                const dom = new JSDOM(html || '');
-                const { window } = dom;
-                const logBuffer = [];
-                const consoleProxy = {
-                    log: (...args) => logBuffer.push(args.join(' ')),
-                    warn: (...args) => logBuffer.push(args.join(' ')),
-                    error: (...args) => logBuffer.push(args.join(' '))
-                };
-                const shadowHelpers = (() => {
-                    const shadowQueryAll = (selector, root = window.document) => {
-                        const results = [];
-                        const walk = (node) => {
-                            if (!node) return;
-                            if (node.nodeType === 1) {
-                                const el = node;
-                                if (selector && el.matches && el.matches(selector)) results.push(el);
-                                if (el.tagName === 'TEMPLATE' && el.hasAttribute('data-shadowroot')) {
-                                    walk(el.content);
-                                }
-                            } else if (node.nodeType === 11) {
-                                // DocumentFragment
-                            }
-                            if (node.childNodes) {
-                                node.childNodes.forEach((child) => walk(child));
-                            }
-                        };
-                        walk(root);
-                        return results;
-                    };
 
-                    const shadowText = (root = window.document) => {
-                        const texts = [];
-                        const walk = (node) => {
-                            if (!node) return;
-                            if (node.nodeType === 3) {
-                                const text = node.nodeValue ? node.nodeValue.trim() : '';
-                                if (text) texts.push(text);
-                                return;
-                            }
-                            if (node.nodeType === 1) {
-                                const el = node;
-                                if (el.tagName === 'TEMPLATE' && el.hasAttribute('data-shadowroot')) {
-                                    walk(el.content);
-                                }
-                            }
-                            if (node.childNodes) {
-                                node.childNodes.forEach((child) => walk(child));
-                            }
-                        };
-                        walk(root);
-                        return texts;
-                    };
+            return new Promise((resolve) => {
+                const worker = spawn('node', [path.join(__dirname, 'extraction-worker.js')], {
+                    stdio: ['pipe', 'pipe', 'pipe'],
+                    env: { ...process.env, NODE_ENV: 'production' } // Minimal env
+                });
 
-                    return { shadowQueryAll, shadowText };
-                })();
-
-                const executor = new Function(
-                    '$$data',
-                    'window',
-                    'document',
-                    'DOMParser',
-                    'console',
-                    `"use strict"; return (async () => { ${script}\n})();`
-                );
-                const $$data = {
-                    html: () => html || '',
-                    url: () => pageUrl || '',
-                    window,
-                    document: window.document,
-                    shadowQueryAll: includeShadowDom ? shadowHelpers.shadowQueryAll : undefined,
-                    shadowText: includeShadowDom ? shadowHelpers.shadowText : undefined
-                };
-                const result = await executor($$data, window, window.document, window.DOMParser, consoleProxy);
-                return { result, logs: logBuffer };
-            } catch (e) {
-                return { result: `Extraction script error: ${e.message}`, logs: [] };
-            }
+                let stdout = '';
+                let stderr = '';
+
+                const workerTimeout = 5000;
+                const timer = setTimeout(() => {
+                    worker.kill();
+                    resolve({ result: 'Worker timed out', logs: [] });
+                }, workerTimeout);
+
+                worker.stdout.on('data', (data) => {
+                    stdout += data.toString();
+                });
+
+                worker.stderr.on('data', (data) => {
+                    stderr += data.toString();
+                });
+
+                worker.on('close', (code) => {
+                    clearTimeout(timer);
+                    if (code !== 0) {
+                        resolve({ result: `Worker exited with code ${code}: ${stderr}`, logs: [] });
+                        return;
+                    }
+                    try {
+                        const output = JSON.parse(stdout);
+                        resolve(output);
+                    } catch (e) {
+                        resolve({ result: `Worker output parse error: ${e.message}. Stdout: ${stdout}`, logs: [] });
+                    }
+                });
+
+                worker.on('error', (err) => {
+                     clearTimeout(timer);
+                     resolve({ result: `Worker spawn error: ${err.message}`, logs: [] });
+                });
+
+                const input = JSON.stringify({
+                    script,
+                    html,
+                    url: pageUrl,
+                    includeShadowDom
+                });
+
+                worker.stdin.write(input);
+                worker.stdin.end();
+            });
         };
 
         const extraction = await runExtractionScript(extractionScript, productHtml, page.url());
 
         // Ensure the public/screenshots directory exists
-        const capturesDir = path.join(__dirname, 'public', 'captures');
-        if (!fs.existsSync(capturesDir)) {
-            fs.mkdirSync(capturesDir, { recursive: true });
-        }
-
-        const screenshotName = `${captureRunId}_scrape_${Date.now()}.png`;
-        const screenshotPath = path.join(capturesDir, screenshotName);
-        try {
-            await page.screenshot({ path: screenshotPath, fullPage: false });
-        } catch (e) {
-            console.error('Screenshot failed:', e.message);
-        }
-
-        // Simple HTML Formatter
-        const formatHTML = (html) => {
-            let indent = 0;
-            return html.replace(/<(\/?)([a-z0-9]+)([^>]*?)(\/?)>/gi, (match, slash, tag, attrs, selfClose) => {
-                if (slash) indent--;
-                const result = '  '.repeat(Math.max(0, indent)) + match;
-                if (!slash && !selfClose && !['img', 'br', 'hr', 'input', 'link', 'meta'].includes(tag.toLowerCase())) indent++;
-                return '\n' + result;
-            }).trim();
-        };
+        const capturesDir = path.join(__dirname, 'public', 'captures');
+        if (!fs.existsSync(capturesDir)) {
+            fs.mkdirSync(capturesDir, { recursive: true });
+        }
+
+        const screenshotName = `${captureRunId}_scrape_${Date.now()}.png`;
+        const screenshotPath = path.join(capturesDir, screenshotName);
+        try {
+            await page.screenshot({ path: screenshotPath, fullPage: false });
+        } catch (e) {
+            console.error('Screenshot failed:', e.message);
+        }
 
         const rawExtraction = extraction.result !== undefined ? extraction.result : (extraction.logs.length ? extraction.logs.join('\n') : undefined);
         const formattedExtraction = extractionFormat === 'csv' ? toCsvString(rawExtraction) : rawExtraction;
@@ -465,48 +430,48 @@ async function handleScrape(req, res) {
             links: await page.$$eval('a[href]', elements => {
                 return elements.map(el => el.href).filter(href => href && href.startsWith('http'));
             }),
-            screenshot_url: `/captures/${screenshotName}`
-        };
-
-        // Save session state
-        if (!statelessExecution) {
-            await context.storageState({ path: STORAGE_STATE_FILE });
-        }
-
-        const video = page.video();
-        await context.close();
-        if (video) {
-            try {
-                const videoPath = await video.path();
-                if (videoPath && fs.existsSync(videoPath)) {
-                    const recordingName = `${captureRunId}_scrape_${Date.now()}.webm`;
-                    const recordingPath = path.join(capturesDir, recordingName);
-                    try {
-                        fs.renameSync(videoPath, recordingPath);
-                    } catch (err) {
-                        if (err && err.code === 'EXDEV') {
-                            fs.copyFileSync(videoPath, recordingPath);
-                            fs.unlinkSync(videoPath);
-                        } else {
-                            throw err;
-                        }
-                    }
-                }
-            } catch (e) {
-                console.error('Recording save failed:', e.message);
-            }
-        }
-
-        await browser.close();
-        res.json(data);
-    } catch (error) {
-        console.error('Scrape Error:', error);
-        try {
-            if (context) await context.close();
-        } catch {}
-        if (browser) await browser.close();
-        res.status(500).json({ error: 'Failed to scrape', details: error.message });
-    }
-}
+            screenshot_url: `/captures/${screenshotName}`
+        };
+
+        // Save session state
+        if (!statelessExecution) {
+            await context.storageState({ path: STORAGE_STATE_FILE });
+        }
+
+        const video = page.video();
+        await context.close();
+        if (video) {
+            try {
+                const videoPath = await video.path();
+                if (videoPath && fs.existsSync(videoPath)) {
+                    const recordingName = `${captureRunId}_scrape_${Date.now()}.webm`;
+                    const recordingPath = path.join(capturesDir, recordingName);
+                    try {
+                        await fs.promises.rename(videoPath, recordingPath);
+                    } catch (err) {
+                        if (err && err.code === 'EXDEV') {
+                            await fs.promises.copyFile(videoPath, recordingPath);
+                            await fs.promises.unlink(videoPath);
+                        } else {
+                            throw err;
+                        }
+                    }
+                }
+            } catch (e) {
+                console.error('Recording save failed:', e.message);
+            }
+        }
+
+        await browser.close();
+        res.json(data);
+    } catch (error) {
+        console.error('Scrape Error:', error);
+        try {
+            if (context) await context.close();
+        } catch {}
+        if (browser) await browser.close();
+        res.status(500).json({ error: 'Failed to scrape', details: error.message });
+    }
+}
 
 module.exports = { handleScrape };