@doppelgangerdev/doppelganger 0.5.6 → 0.5.8

package/agent.js

@@ -2,8 +2,10 @@ const { chromium } = require('playwright');
 const { JSDOM } = require('jsdom');
 const fs = require('fs');
 const path = require('path');
+const vm = require('vm');
 const { getProxySelection } = require('./proxy-rotation');
 const { selectUserAgent } = require('./user-agent-settings');
+const { formatHTML, safeFormatHTML } = require('./html-utils');
 
 const STORAGE_STATE_PATH = path.join(__dirname, 'storage_state.json');
 const STORAGE_STATE_FILE = (() => {
@@ -20,10 +22,10 @@ const STORAGE_STATE_FILE = (() => {
 
 const API_KEY_FILE = path.join(__dirname, 'data', 'api_key.json');
 
-const loadApiKey = () => {
-    if (!fs.existsSync(API_KEY_FILE)) return null;
+const loadApiKey = async () => {
     try {
-        const data = JSON.parse(fs.readFileSync(API_KEY_FILE, 'utf8'));
+        const raw = await fs.promises.readFile(API_KEY_FILE, 'utf8');
+        const data = JSON.parse(raw);
         return data && data.apiKey ? data.apiKey : null;
     } catch {
         return null;
@@ -113,6 +115,12 @@ async function overshootScroll(page, targetY) {
 const punctuationPause = /[.,!?;:]/;
 
 const randomBetween = (min, max) => min + Math.random() * (max - min);
+const parseBooleanFlag = (value) => {
+    if (typeof value === 'boolean') return value;
+    if (value === undefined || value === null) return false;
+    const normalized = String(value).toLowerCase();
+    return normalized === 'true' || normalized === '1';
+};
 
 async function humanType(page, selector, text, options = {}) {
     const { allowTypos = false, naturalTyping = false, fatigue = false } = options;
@@ -161,6 +169,93 @@ async function humanType(page, selector, text, options = {}) {
     }
 }
 
+const REAL_TARGET = Symbol('REAL_TARGET');
+
+function createSafeProxy(target) {
+    if (target === null || (typeof target !== 'object' && typeof target !== 'function')) {
+        return target;
+    }
+
+    let shadowTarget = target;
+    if (typeof target === 'function') {
+        shadowTarget = function (...args) { };
+        try { Object.defineProperty(shadowTarget, 'name', { value: target.name, configurable: true }); } catch {}
+        try { Object.defineProperty(shadowTarget, 'length', { value: target.length, configurable: true }); } catch {}
+        shadowTarget[REAL_TARGET] = target;
+    }
+
+    return new Proxy(shadowTarget, {
+        get(target, prop, receiver) {
+            const realTarget = target[REAL_TARGET] || target;
+            if (prop === 'constructor' || prop === '__proto__') {
+                return undefined;
+            }
+            if (prop === REAL_TARGET) return realTarget;
+
+            const value = Reflect.get(realTarget, prop, realTarget);
+
+            if (typeof value === 'function') {
+                return function (...args) {
+                    const realArgs = args.map(arg => {
+                        return (arg && arg[REAL_TARGET]) ? arg[REAL_TARGET] : arg;
+                    });
+                    const wrappedArgs = realArgs.map(arg => {
+                        if (typeof arg === 'function') {
+                            return function (...cbArgs) {
+                                const wrappedCbArgs = cbArgs.map(a => createSafeProxy(a));
+                                return arg.apply(this, wrappedCbArgs);
+                            }
+                        }
+                        return arg;
+                    });
+                    try {
+                        const result = value.apply(realTarget, wrappedArgs);
+                        return createSafeProxy(result);
+                    } catch (e) {
+                        throw e;
+                    }
+                };
+            }
+            return createSafeProxy(value);
+        },
+        apply(target, thisArg, argList) {
+             const realTarget = target[REAL_TARGET] || target;
+             const realThis = (thisArg && thisArg[REAL_TARGET]) ? thisArg[REAL_TARGET] : thisArg;
+             const realArgs = argList.map(arg => {
+                return (arg && arg[REAL_TARGET]) ? arg[REAL_TARGET] : arg;
+             });
+             const wrappedArgs = realArgs.map(arg => {
+                 if (typeof arg === 'function') {
+                      return function (...cbArgs) {
+                           const wrappedCbArgs = cbArgs.map(a => createSafeProxy(a));
+                           return arg.apply(this, wrappedCbArgs);
+                      }
+                 }
+                 return arg;
+             });
+
+             try {
+                 const result = Reflect.apply(realTarget, realThis, wrappedArgs);
+                 return createSafeProxy(result);
+             } catch (e) {
+                 throw e;
+             }
+        },
+        construct(target, argumentsList, newTarget) {
+            const realTarget = target[REAL_TARGET] || target;
+            const realArgs = argumentsList.map(arg => {
+                return (arg && arg[REAL_TARGET]) ? arg[REAL_TARGET] : arg;
+            });
+            try {
+                const result = Reflect.construct(realTarget, realArgs, realTarget);
+                return createSafeProxy(result);
+            } catch (e) {
+                throw e;
+            }
+        }
+    });
+}
+
 async function handleAgent(req, res) {
     const data = (req.method === 'POST') ? req.body : req.query;
     let { url, actions, wait: globalWait, rotateUserAgents, rotateProxies, humanTyping, stealth = {} } = data;
@@ -170,6 +265,10 @@ async function handleAgent(req, res) {
     const includeShadowDom = includeShadowDomRaw === undefined
         ? true
         : !(String(includeShadowDomRaw).toLowerCase() === 'false' || includeShadowDomRaw === false);
+    const disableRecordingRaw = data.disableRecording ?? req.query.disableRecording;
+    const disableRecording = parseBooleanFlag(disableRecordingRaw);
+    const statelessExecutionRaw = data.statelessExecution ?? req.query.statelessExecution;
+    const statelessExecution = parseBooleanFlag(statelessExecutionRaw);
     const {
         allowTypos = false,
         idleMovements = false,
@@ -194,10 +293,10 @@ async function handleAgent(req, res) {
         });
     }
 
-    const localPort = req.socket && req.socket.localPort;
-    const configuredPort = process.env.PORT || process.env.VITE_BACKEND_PORT;
-    const basePort = localPort || configuredPort || '11345';
-    const baseUrl = `${req.protocol || 'http'}://127.0.0.1:${basePort}`;
+    const localPort = req.socket && req.socket.localPort;
+    const configuredPort = process.env.PORT || process.env.VITE_BACKEND_PORT;
+    const basePort = localPort || configuredPort || '11345';
+    const baseUrl = `${req.protocol || 'http'}://127.0.0.1:${basePort}`;
     const runtimeVars = { ...(data.taskVariables || data.variables || {}) };
     let lastBlockOutput = null;
     runtimeVars['block.output'] = lastBlockOutput;
@@ -258,39 +357,58 @@ async function handleAgent(req, res) {
 
     const parseCsv = (input) => {
         const text = typeof input === 'string' ? input : String(input || '');
+        const len = text.length;
         const rows = [];
         let row = [];
         let current = '';
         let inQuotes = false;
+        const specialChar = /[",\n\r]/g;
 
-        for (let i = 0; i < text.length; i += 1) {
-            const char = text[i];
+        let i = 0;
+        while (i < len) {
             if (inQuotes) {
-                if (char === '"') {
-                    if (text[i + 1] === '"') {
-                        current += '"';
-                        i += 1;
-                    } else {
-                        inQuotes = false;
-                    }
+                const nextQuote = text.indexOf('"', i);
+                if (nextQuote === -1) {
+                    current += text.slice(i);
+                    i = len;
+                    break;
+                }
+                current += text.slice(i, nextQuote);
+                i = nextQuote;
+                if (i + 1 < len && text[i + 1] === '"') {
+                    current += '"';
+                    i += 2;
                 } else {
-                    current += char;
+                    inQuotes = false;
+                    i += 1;
                 }
             } else {
+                specialChar.lastIndex = i;
+                const match = specialChar.exec(text);
+                if (!match) {
+                    current += text.slice(i);
+                    i = len;
+                    break;
+                }
+                const idx = match.index;
+                const char = match[0];
+                current += text.slice(i, idx);
+                i = idx;
                 if (char === '"') {
                     inQuotes = true;
+                    i += 1;
                 } else if (char === ',') {
                     row.push(current);
                     current = '';
+                    i += 1;
                 } else if (char === '\n') {
                     row.push(current);
                     rows.push(row);
                     row = [];
                     current = '';
+                    i += 1;
                 } else if (char === '\r') {
-                    // ignore CR (handle CRLF)
-                } else {
-                    current += char;
+                    i += 1;
                 }
             }
         }
@@ -398,7 +516,7 @@ async function handleAgent(req, res) {
         return { startToEnd, startToElse, elseToEnd, endToStart };
     };
 
-    const selectedUA = selectUserAgent(rotateUserAgents);
+    const selectedUA = await selectUserAgent(rotateUserAgents);
 
     let browser;
     let context;
@@ -425,9 +543,7 @@ async function handleAgent(req, res) {
         browser = await chromium.launch(launchOptions);
 
         const recordingsDir = path.join(__dirname, 'data', 'recordings');
-        if (!fs.existsSync(recordingsDir)) {
-            fs.mkdirSync(recordingsDir, { recursive: true });
-        }
+        await fs.promises.mkdir(recordingsDir, { recursive: true });
 
         const rotateViewport = String(data.rotateViewport).toLowerCase() === 'true' || data.rotateViewport === true;
         const viewport = rotateViewport
@@ -442,13 +558,16 @@ async function handleAgent(req, res) {
             timezoneId: 'America/New_York',
             colorScheme: 'dark',
             permissions: ['geolocation'],
-            recordVideo: { dir: recordingsDir, size: viewport }
         };
 
-        if (fs.existsSync(STORAGE_STATE_FILE)) {
+        const shouldUseStorageState = !statelessExecution && fs.existsSync(STORAGE_STATE_FILE);
+        if (shouldUseStorageState) {
             contextOptions.storageState = STORAGE_STATE_FILE;
         }
 
+        if (!disableRecording) {
+            contextOptions.recordVideo = { dir: recordingsDir, size: viewport };
+        }
         context = await browser.newContext(contextOptions);
 
         await context.addInitScript(() => {
@@ -1009,7 +1128,7 @@ async function handleAgent(req, res) {
                     case 'start': {
                         const taskId = resolveMaybe(act.value);
                         if (!taskId) throw new Error('Missing task id.');
-                        const apiKey = loadApiKey() || data.apiKey || data.key;
+                        const apiKey = (await loadApiKey()) || data.apiKey || data.key;
                         if (!apiKey) {
                             logs.push('No API key available; attempting internal start.');
                         }
@@ -1350,16 +1469,6 @@ async function handleAgent(req, res) {
                     return { shadowQueryAll, shadowText };
                 })();
 
-                // CodeQL alerts on dynamic eval, but extraction scripts intentionally run inside the browser sandbox,
-                // so we expose only the helpers needed (window, document, DOMParser, console) and keep the evaluation confined there.
-                const executor = new Function(
-                    '$$data',
-                    'window',
-                    'document',
-                    'DOMParser',
-                    'console',
-                    `"use strict"; return (async () => { ${script}\n})();`
-                );
                 const $$data = {
                     html: () => html || '',
                     url: () => pageUrl || '',
@@ -1368,7 +1477,33 @@ async function handleAgent(req, res) {
                     shadowQueryAll: includeShadowDom ? shadowHelpers.shadowQueryAll : undefined,
                     shadowText: includeShadowDom ? shadowHelpers.shadowText : undefined
                 };
-                const result = await executor($$data, window, window.document, window.DOMParser, consoleProxy);
+
+                // Use vm for sandboxed execution
+                const sandbox = Object.create(null);
+                sandbox.window = createSafeProxy(window);
+                sandbox.document = createSafeProxy(window.document);
+                sandbox.DOMParser = createSafeProxy(window.DOMParser);
+                sandbox.console = createSafeProxy(consoleProxy);
+                sandbox.$$data = createSafeProxy($$data);
+
+                // Pass the script as a variable to avoid string interpolation (CodeQL: Code Injection)
+                sandbox.$$userScript = script;
+
+                const context = vm.createContext(sandbox);
+
+                // We use a static wrapper to execute the user script.
+                // This ensures that the code passed to vm.runInContext is constant and safe.
+                // The user script is retrieved from the sandbox environment and executed as an AsyncFunction.
+                const scriptCode = `
+                    "use strict";
+                    (async () => {
+                        const AsyncFunction = Object.getPrototypeOf(async function(){}).constructor;
+                        const fn = new AsyncFunction('$$data', 'window', 'document', 'DOMParser', 'console', $$userScript);
+                        return fn($$data, window, document, DOMParser, console);
+                    })();
+                `;
+
+                const result = await vm.runInContext(scriptCode, context);
                 return { result, logs: logBuffer };
             } catch (e) {
                 return { result: `Extraction script error: ${e.message}`, logs: [] };
@@ -1381,29 +1516,6 @@ async function handleAgent(req, res) {
         const extractionScript = extractionScriptRaw ? resolveTemplate(extractionScriptRaw) : undefined;
         const extraction = await runExtractionScript(extractionScript, cleanedHtml, page.url());
 
-        // Simple HTML Formatter (fallback to raw if formatting collapses content)
-        const formatHTML = (html) => {
-            let indent = 0;
-            return html.replace(/<(\/?)([a-z0-9]+)([^>]*?)(\/?)>/gi, (match, slash, tag, attrs, selfClose) => {
-                if (slash) indent--;
-                const result = '  '.repeat(Math.max(0, indent)) + match;
-                if (!slash && !selfClose && !['img', 'br', 'hr', 'input', 'link', 'meta'].includes(tag.toLowerCase())) indent++;
-                return '\n' + result;
-            }).trim();
-        };
-
-        const safeFormatHTML = (html) => {
-            if (typeof html !== 'string') return '';
-            try {
-                const formatted = formatHTML(html);
-                if (!formatted) return html;
-                if (formatted.length < Math.max(200, Math.floor(html.length * 0.5))) return html;
-                return formatted;
-            } catch {
-                return html;
-            }
-        };
-
         // Ensure the public/screenshots directory exists
         const capturesDir = path.join(__dirname, 'public', 'captures');
         if (!fs.existsSync(capturesDir)) {
@@ -1434,7 +1546,9 @@ async function handleAgent(req, res) {
         };
 
         const video = page.video();
-        try { await context.storageState({ path: STORAGE_STATE_FILE }); } catch {}
+        if (!statelessExecution) {
+            try { await context.storageState({ path: STORAGE_STATE_FILE }); } catch {}
+        }
         try { await context.close(); } catch {}
         if (video) {
             try {