@doppelgangerdev/doppelganger 0.5.7 → 0.5.8

package/LICENSE

@@ -1,6 +1,6 @@
-                                   Notice & Attribution License v1.0
-                                       Version 1.0, January 2026
+Notice & Attribution License v1.0
 
+© 2026 Mnemosyne
 
 1. Definitions
 1.1. "Software" means all source code, binaries, scripts, libraries, components, build configurations, and other software artifacts provided under this license.

package/README.md

@@ -1,31 +1,10 @@
-![Doppelganger Banner](banner.png)
+![Doppelganger Banner](https://raw.githubusercontent.com/mnemosyne-artificial-intelligence/doppelganger/main/banner.png)
 
 # Doppelganger — Browser Automation for Everyone
 
-<div align="center">
-  <a href="https://doppelgangerdev.com">
-    <img src="https://img.shields.io/badge/Website-doppelgangerdev.com-0056ff?style=for-the-badge&logo=googlechrome&rounded=true" alt="Website" />
-  </a>
-  <a href="https://doppelgangerdev.com/docs">
-    <img src="https://img.shields.io/badge/Docs-doppelgangerdev.com%2Fdocs-00c2ff?style=for-the-badge&logo=readthedocs&rounded=true" alt="Docs" />
-  </a>
-  <a href="https://forum.doppelgangerdev.com">
-    <img src="https://img.shields.io/badge/Forum-forum.doppelgangerdev.com-ff9900?style=for-the-badge&logo=discourse&rounded=true" alt="Forum" />
-  </a>
-  <a href="https://opensource.org/">
-    <img src="https://img.shields.io/badge/Open_Source-Yes-0056ff?style=for-the-badge&logo=opensourceinitiative&logoColor=white" alt="Open Source" />
-  </a>
-  <a href="https://www.npmjs.com/package/@doppelgangerdev/doppelganger">
-    <img src="https://img.shields.io/badge/Version-0.5.5-6a8cff?style=for-the-badge&logo=npm&rounded=true" alt="Version" />
-  </a>
-  <a href="https://hub.docker.com/r/mnemosyneai/doppelganger">
-    <img src="https://img.shields.io/badge/Docker-mnemosyneai%2Fdoppelganger-0db7ed?style=for-the-badge&logo=docker&rounded=true" alt="Docker" />
-  </a>
-</div>
-
 Doppelganger is a self‑hosted, block-first automation control plane built for teams that want predictable, auditable browser workflows without pushing sensitive data to third‑party SaaS. It bundles a React/Vite frontend, an Express/Playwright backend, helper scripts, and optional CLI tooling so you can sketch blocks, inject JavaScript, rotate proxies, and run everything locally.
 
-![Demo run](demo-run.gif)
+![Demo run](https://raw.githubusercontent.com/mnemosyne-artificial-intelligence/doppelganger/main/demo-run.gif)
 
 # What You Get
 
@@ -236,7 +215,7 @@ Authentication enforces sessions (`/api/auth/login`, `/api/auth/logout`, `/api/a
 
 # Maintenance
 
-- The project is governed by the **[Sustainable Use License (SUL 1.0)](https://github.com/mnemosyne-artificial-intelligence/doppelganger/blob/main/LICENSE)**; hosting it as a competing service is prohibited.
+- The project is governed by the **[Notice & Attribution License v1.0](https://github.com/mnemosyne-artificial-intelligence/doppelganger/blob/main/LICENSE)**, which grants royalty-free internal/private rights while requiring notice, attribution, and source delivery when the software is deployed for external end users; hosting it as a competing service is prohibited.
 - Keep `data/` and `storage_state.json` backed up if you rely on historical cookies or proxies.
 - Release updates by pulling `mnemosyneai/doppelganger` (Docker) or `npm i @doppelgangerdev/doppelganger` (npm). The Settings view always displays the current package version.
 - Contributions: follow `.github/` templates, respect `CONTRIBUTING.md`, and run available lint/test scripts if you touch critical areas.
@@ -250,15 +229,16 @@ Authentication enforces sessions (`/api/auth/login`, `/api/auth/logout`, `/api/a
 - [x] **Task proxy rotation toggle** — the “Rotate Proxies” option in each task ties into the Settings rotation controls, enabling rotation per execution.
 - [ ] **Action key combos** — add modifier shortcuts (e.g., Ctrl+Click, Shift+Scroll) so tasks can more closely mirror real user interactions.
 - [ ] **Click-and-drag block** — add an action that does drag gestures (selecting text, moving items) so tasks can simulate click-and-drag flows.
-- [x] **Recording controls** — Task editor now exposes a “Disable automated recording” switch in the general settings panel so workflows can skip video capture on a per-task basis.
+- [x] **Recording controls** — Task editor now exposes a “Disable automated recording” switch in the general settings panel so workflows can skip video capture on a per-task basis.
 - [ ] **File downloads** — add explicit support for agent tasks to download files (PDFs, CSVs, etc.) directly from target pages, then surface those downloads in the UI so users can preview or export them without sifting through captures.
-- [x] **Stateless mode** — Tasks now have a “Stateless execution” toggle alongside the recording controls so each run can skip `storage_state.json`, ensuring no cookies or local storage persist between executions for that workflow.
-- [ ] **Adblocking filters** — add controls so execution contexts can enable built-in ad/malware filtering (e.g., via hosts file overrides or request blocking) to reduce noise on sensitive sites.
-- [ ] **Extraction response mode** — add a Settings switch so users can choose whether the UI returns HTML+data (for debugging) or data-only payloads when extraction scripts run.
+- [x] **Stateless mode** — Tasks now have a “Stateless execution” toggle alongside the recording controls so each run can skip `storage_state.json`, ensuring no cookies or local storage persist between executions for that workflow.
+- [ ] **Adblocking filters** — add controls so execution contexts can enable built-in ad/malware filtering (e.g., via hosts file overrides or request blocking) to reduce noise on sensitive sites.
+- [ ] **Extraction response mode** — add a Settings switch so users can choose whether the UI returns HTML+data (for debugging) or data-only payloads when extraction scripts run.
 - [ ] **Folder organization** — group tasks, assets, and captures into named folders so operators can browse, filter, and download collections per workflow.
 - [ ] **Stable capture retention** — add filtering, pinning, and archiving in captures tab so teams can keep compliance records.
 - [ ] **Workspace templates** — allow saving and sharing workspace presets (layout + default proxies/agents) so new team members can onboard with pre-configured setups.
 - [ ] **Geo-targeted exits** — allow choosing proxy regions for tasks so you can pin the apparent location before running a job.
+- [ ] **Complete anti-detection coverage** — follow browserscan.net's anti-detection checklist (fingerprints, headers, fonts, WebRTC, etc.) so automated runs mimic real browsers across task executions.
 - [ ] **Session recording redaction** — add toggles to redact sensitive fields (passwords, credit cards) from recordings/logs before storing them.
 - [ ] **Two-factor authentication** — add optional TOTP/second-factor support to Settings/Auth so operators can lock down the UI with 2FA.
 - [ ] **AI-assisted fixing** — add an “AI auto-fix” helper that suggests layout, selector, and proxy tweaks after failed runs, letting teams approve or discard the proposed changes without switching contexts.
@@ -276,4 +256,4 @@ Authentication enforces sessions (`/api/auth/login`, `/api/auth/logout`, `/api/a
 
 - Report issues or request features via the GitHub repo issue tracker.
 - Follow the authors on `https://github.com/mnemosyne-artificial-intelligence` for releases.
-- Share automation recipes with other self-hosted users in your org, but respect the license for sharing infrastructure.
+- Share automation recipes with other self-hosted users in your org, but respect the license for sharing infrastructure.

package/agent.js

@@ -2,8 +2,10 @@ const { chromium } = require('playwright');
 const { JSDOM } = require('jsdom');
 const fs = require('fs');
 const path = require('path');
+const vm = require('vm');
 const { getProxySelection } = require('./proxy-rotation');
 const { selectUserAgent } = require('./user-agent-settings');
+const { formatHTML, safeFormatHTML } = require('./html-utils');
 
 const STORAGE_STATE_PATH = path.join(__dirname, 'storage_state.json');
 const STORAGE_STATE_FILE = (() => {
@@ -20,10 +22,10 @@ const STORAGE_STATE_FILE = (() => {
 
 const API_KEY_FILE = path.join(__dirname, 'data', 'api_key.json');
 
-const loadApiKey = () => {
-    if (!fs.existsSync(API_KEY_FILE)) return null;
+const loadApiKey = async () => {
     try {
-        const data = JSON.parse(fs.readFileSync(API_KEY_FILE, 'utf8'));
+        const raw = await fs.promises.readFile(API_KEY_FILE, 'utf8');
+        const data = JSON.parse(raw);
         return data && data.apiKey ? data.apiKey : null;
     } catch {
         return null;
@@ -110,17 +112,17 @@ async function overshootScroll(page, targetY) {
     }
 }
 
-const punctuationPause = /[.,!?;:]/;
-
-const randomBetween = (min, max) => min + Math.random() * (max - min);
-const parseBooleanFlag = (value) => {
-    if (typeof value === 'boolean') return value;
-    if (value === undefined || value === null) return false;
-    const normalized = String(value).toLowerCase();
-    return normalized === 'true' || normalized === '1';
-};
-
-async function humanType(page, selector, text, options = {}) {
+const punctuationPause = /[.,!?;:]/;
+
+const randomBetween = (min, max) => min + Math.random() * (max - min);
+const parseBooleanFlag = (value) => {
+    if (typeof value === 'boolean') return value;
+    if (value === undefined || value === null) return false;
+    const normalized = String(value).toLowerCase();
+    return normalized === 'true' || normalized === '1';
+};
+
+async function humanType(page, selector, text, options = {}) {
     const { allowTypos = false, naturalTyping = false, fatigue = false } = options;
     if (selector) await page.focus(selector);
     const chars = text.split('');
@@ -167,21 +169,108 @@ async function humanType(page, selector, text, options = {}) {
     }
 }
 
+const REAL_TARGET = Symbol('REAL_TARGET');
+
+function createSafeProxy(target) {
+    if (target === null || (typeof target !== 'object' && typeof target !== 'function')) {
+        return target;
+    }
+
+    let shadowTarget = target;
+    if (typeof target === 'function') {
+        shadowTarget = function (...args) { };
+        try { Object.defineProperty(shadowTarget, 'name', { value: target.name, configurable: true }); } catch {}
+        try { Object.defineProperty(shadowTarget, 'length', { value: target.length, configurable: true }); } catch {}
+        shadowTarget[REAL_TARGET] = target;
+    }
+
+    return new Proxy(shadowTarget, {
+        get(target, prop, receiver) {
+            const realTarget = target[REAL_TARGET] || target;
+            if (prop === 'constructor' || prop === '__proto__') {
+                return undefined;
+            }
+            if (prop === REAL_TARGET) return realTarget;
+
+            const value = Reflect.get(realTarget, prop, realTarget);
+
+            if (typeof value === 'function') {
+                return function (...args) {
+                    const realArgs = args.map(arg => {
+                        return (arg && arg[REAL_TARGET]) ? arg[REAL_TARGET] : arg;
+                    });
+                    const wrappedArgs = realArgs.map(arg => {
+                        if (typeof arg === 'function') {
+                            return function (...cbArgs) {
+                                const wrappedCbArgs = cbArgs.map(a => createSafeProxy(a));
+                                return arg.apply(this, wrappedCbArgs);
+                            }
+                        }
+                        return arg;
+                    });
+                    try {
+                        const result = value.apply(realTarget, wrappedArgs);
+                        return createSafeProxy(result);
+                    } catch (e) {
+                        throw e;
+                    }
+                };
+            }
+            return createSafeProxy(value);
+        },
+        apply(target, thisArg, argList) {
+             const realTarget = target[REAL_TARGET] || target;
+             const realThis = (thisArg && thisArg[REAL_TARGET]) ? thisArg[REAL_TARGET] : thisArg;
+             const realArgs = argList.map(arg => {
+                return (arg && arg[REAL_TARGET]) ? arg[REAL_TARGET] : arg;
+             });
+             const wrappedArgs = realArgs.map(arg => {
+                 if (typeof arg === 'function') {
+                      return function (...cbArgs) {
+                           const wrappedCbArgs = cbArgs.map(a => createSafeProxy(a));
+                           return arg.apply(this, wrappedCbArgs);
+                      }
+                 }
+                 return arg;
+             });
+
+             try {
+                 const result = Reflect.apply(realTarget, realThis, wrappedArgs);
+                 return createSafeProxy(result);
+             } catch (e) {
+                 throw e;
+             }
+        },
+        construct(target, argumentsList, newTarget) {
+            const realTarget = target[REAL_TARGET] || target;
+            const realArgs = argumentsList.map(arg => {
+                return (arg && arg[REAL_TARGET]) ? arg[REAL_TARGET] : arg;
+            });
+            try {
+                const result = Reflect.construct(realTarget, realArgs, realTarget);
+                return createSafeProxy(result);
+            } catch (e) {
+                throw e;
+            }
+        }
+    });
+}
+
 async function handleAgent(req, res) {
     const data = (req.method === 'POST') ? req.body : req.query;
     let { url, actions, wait: globalWait, rotateUserAgents, rotateProxies, humanTyping, stealth = {} } = data;
     const runId = data.runId ? String(data.runId) : null;
     const captureRunId = runId || `run_${Date.now()}_unknown`;
-    const includeShadowDomRaw = data.includeShadowDom ?? req.query.includeShadowDom;
-    const includeShadowDom = includeShadowDomRaw === undefined
-        ? true
-        : !(String(includeShadowDomRaw).toLowerCase() === 'false' || includeShadowDomRaw === false);
-    const disableRecordingRaw = data.disableRecording ?? req.query.disableRecording;
-    const disableRecording = parseBooleanFlag(disableRecordingRaw);
-    const statelessExecutionRaw = data.statelessExecution ?? req.query.statelessExecution;
-    const statelessExecution = parseBooleanFlag(statelessExecutionRaw);
-    const {
-        allowTypos = false,
+    const includeShadowDomRaw = data.includeShadowDom ?? req.query.includeShadowDom;
+    const includeShadowDom = includeShadowDomRaw === undefined
+        ? true
+        : !(String(includeShadowDomRaw).toLowerCase() === 'false' || includeShadowDomRaw === false);
+    const disableRecordingRaw = data.disableRecording ?? req.query.disableRecording;
+    const disableRecording = parseBooleanFlag(disableRecordingRaw);
+    const statelessExecutionRaw = data.statelessExecution ?? req.query.statelessExecution;
+    const statelessExecution = parseBooleanFlag(statelessExecutionRaw);
+    const {
+        allowTypos = false,
         idleMovements = false,
         overscroll = false,
         deadClicks = false,
@@ -204,10 +293,10 @@ async function handleAgent(req, res) {
         });
     }
 
-    const localPort = req.socket && req.socket.localPort;
-    const configuredPort = process.env.PORT || process.env.VITE_BACKEND_PORT;
-    const basePort = localPort || configuredPort || '11345';
-    const baseUrl = `${req.protocol || 'http'}://127.0.0.1:${basePort}`;
+    const localPort = req.socket && req.socket.localPort;
+    const configuredPort = process.env.PORT || process.env.VITE_BACKEND_PORT;
+    const basePort = localPort || configuredPort || '11345';
+    const baseUrl = `${req.protocol || 'http'}://127.0.0.1:${basePort}`;
     const runtimeVars = { ...(data.taskVariables || data.variables || {}) };
     let lastBlockOutput = null;
     runtimeVars['block.output'] = lastBlockOutput;
@@ -268,39 +357,58 @@ async function handleAgent(req, res) {
 
     const parseCsv = (input) => {
         const text = typeof input === 'string' ? input : String(input || '');
+        const len = text.length;
         const rows = [];
         let row = [];
         let current = '';
         let inQuotes = false;
+        const specialChar = /[",\n\r]/g;
 
-        for (let i = 0; i < text.length; i += 1) {
-            const char = text[i];
+        let i = 0;
+        while (i < len) {
             if (inQuotes) {
-                if (char === '"') {
-                    if (text[i + 1] === '"') {
-                        current += '"';
-                        i += 1;
-                    } else {
-                        inQuotes = false;
-                    }
+                const nextQuote = text.indexOf('"', i);
+                if (nextQuote === -1) {
+                    current += text.slice(i);
+                    i = len;
+                    break;
+                }
+                current += text.slice(i, nextQuote);
+                i = nextQuote;
+                if (i + 1 < len && text[i + 1] === '"') {
+                    current += '"';
+                    i += 2;
                 } else {
-                    current += char;
+                    inQuotes = false;
+                    i += 1;
                 }
             } else {
+                specialChar.lastIndex = i;
+                const match = specialChar.exec(text);
+                if (!match) {
+                    current += text.slice(i);
+                    i = len;
+                    break;
+                }
+                const idx = match.index;
+                const char = match[0];
+                current += text.slice(i, idx);
+                i = idx;
                 if (char === '"') {
                     inQuotes = true;
+                    i += 1;
                 } else if (char === ',') {
                     row.push(current);
                     current = '';
+                    i += 1;
                 } else if (char === '\n') {
                     row.push(current);
                     rows.push(row);
                     row = [];
                     current = '';
+                    i += 1;
                 } else if (char === '\r') {
-                    // ignore CR (handle CRLF)
-                } else {
-                    current += char;
+                    i += 1;
                 }
             }
         }
@@ -408,7 +516,7 @@ async function handleAgent(req, res) {
         return { startToEnd, startToElse, elseToEnd, endToStart };
     };
 
-    const selectedUA = selectUserAgent(rotateUserAgents);
+    const selectedUA = await selectUserAgent(rotateUserAgents);
 
     let browser;
     let context;
@@ -435,33 +543,31 @@ async function handleAgent(req, res) {
         browser = await chromium.launch(launchOptions);
 
         const recordingsDir = path.join(__dirname, 'data', 'recordings');
-        if (!fs.existsSync(recordingsDir)) {
-            fs.mkdirSync(recordingsDir, { recursive: true });
-        }
+        await fs.promises.mkdir(recordingsDir, { recursive: true });
 
         const rotateViewport = String(data.rotateViewport).toLowerCase() === 'true' || data.rotateViewport === true;
         const viewport = rotateViewport
             ? { width: 1280 + Math.floor(Math.random() * 640), height: 720 + Math.floor(Math.random() * 360) }
             : { width: 1366, height: 768 };
 
-        const contextOptions = {
-            userAgent: selectedUA,
-            viewport,
-            deviceScaleFactor: 1,
-            locale: 'en-US',
-            timezoneId: 'America/New_York',
-            colorScheme: 'dark',
-            permissions: ['geolocation'],
-        };
-
-        const shouldUseStorageState = !statelessExecution && fs.existsSync(STORAGE_STATE_FILE);
-        if (shouldUseStorageState) {
-            contextOptions.storageState = STORAGE_STATE_FILE;
-        }
-
-        if (!disableRecording) {
-            contextOptions.recordVideo = { dir: recordingsDir, size: viewport };
-        }
+        const contextOptions = {
+            userAgent: selectedUA,
+            viewport,
+            deviceScaleFactor: 1,
+            locale: 'en-US',
+            timezoneId: 'America/New_York',
+            colorScheme: 'dark',
+            permissions: ['geolocation'],
+        };
+
+        const shouldUseStorageState = !statelessExecution && fs.existsSync(STORAGE_STATE_FILE);
+        if (shouldUseStorageState) {
+            contextOptions.storageState = STORAGE_STATE_FILE;
+        }
+
+        if (!disableRecording) {
+            contextOptions.recordVideo = { dir: recordingsDir, size: viewport };
+        }
         context = await browser.newContext(contextOptions);
 
         await context.addInitScript(() => {
@@ -1022,7 +1128,7 @@ async function handleAgent(req, res) {
                     case 'start': {
                         const taskId = resolveMaybe(act.value);
                         if (!taskId) throw new Error('Missing task id.');
-                        const apiKey = loadApiKey() || data.apiKey || data.key;
+                        const apiKey = (await loadApiKey()) || data.apiKey || data.key;
                         if (!apiKey) {
                             logs.push('No API key available; attempting internal start.');
                         }
@@ -1363,16 +1469,6 @@ async function handleAgent(req, res) {
                     return { shadowQueryAll, shadowText };
                 })();
 
-                // CodeQL alerts on dynamic eval, but extraction scripts intentionally run inside the browser sandbox,
-                // so we expose only the helpers needed (window, document, DOMParser, console) and keep the evaluation confined there.
-                const executor = new Function(
-                    '$$data',
-                    'window',
-                    'document',
-                    'DOMParser',
-                    'console',
-                    `"use strict"; return (async () => { ${script}\n})();`
-                );
                 const $$data = {
                     html: () => html || '',
                     url: () => pageUrl || '',
@@ -1381,7 +1477,33 @@ async function handleAgent(req, res) {
                     shadowQueryAll: includeShadowDom ? shadowHelpers.shadowQueryAll : undefined,
                     shadowText: includeShadowDom ? shadowHelpers.shadowText : undefined
                 };
-                const result = await executor($$data, window, window.document, window.DOMParser, consoleProxy);
+
+                // Use vm for sandboxed execution
+                const sandbox = Object.create(null);
+                sandbox.window = createSafeProxy(window);
+                sandbox.document = createSafeProxy(window.document);
+                sandbox.DOMParser = createSafeProxy(window.DOMParser);
+                sandbox.console = createSafeProxy(consoleProxy);
+                sandbox.$$data = createSafeProxy($$data);
+
+                // Pass the script as a variable to avoid string interpolation (CodeQL: Code Injection)
+                sandbox.$$userScript = script;
+
+                const context = vm.createContext(sandbox);
+
+                // We use a static wrapper to execute the user script.
+                // This ensures that the code passed to vm.runInContext is constant and safe.
+                // The user script is retrieved from the sandbox environment and executed as an AsyncFunction.
+                const scriptCode = `
+                    "use strict";
+                    (async () => {
+                        const AsyncFunction = Object.getPrototypeOf(async function(){}).constructor;
+                        const fn = new AsyncFunction('$$data', 'window', 'document', 'DOMParser', 'console', $$userScript);
+                        return fn($$data, window, document, DOMParser, console);
+                    })();
+                `;
+
+                const result = await vm.runInContext(scriptCode, context);
                 return { result, logs: logBuffer };
             } catch (e) {
                 return { result: `Extraction script error: ${e.message}`, logs: [] };
@@ -1394,29 +1516,6 @@ async function handleAgent(req, res) {
         const extractionScript = extractionScriptRaw ? resolveTemplate(extractionScriptRaw) : undefined;
         const extraction = await runExtractionScript(extractionScript, cleanedHtml, page.url());
 
-        // Simple HTML Formatter (fallback to raw if formatting collapses content)
-        const formatHTML = (html) => {
-            let indent = 0;
-            return html.replace(/<(\/?)([a-z0-9]+)([^>]*?)(\/?)>/gi, (match, slash, tag, attrs, selfClose) => {
-                if (slash) indent--;
-                const result = '  '.repeat(Math.max(0, indent)) + match;
-                if (!slash && !selfClose && !['img', 'br', 'hr', 'input', 'link', 'meta'].includes(tag.toLowerCase())) indent++;
-                return '\n' + result;
-            }).trim();
-        };
-
-        const safeFormatHTML = (html) => {
-            if (typeof html !== 'string') return '';
-            try {
-                const formatted = formatHTML(html);
-                if (!formatted) return html;
-                if (formatted.length < Math.max(200, Math.floor(html.length * 0.5))) return html;
-                return formatted;
-            } catch {
-                return html;
-            }
-        };
-
         // Ensure the public/screenshots directory exists
         const capturesDir = path.join(__dirname, 'public', 'captures');
         if (!fs.existsSync(capturesDir)) {
@@ -1447,9 +1546,9 @@ async function handleAgent(req, res) {
         };
 
         const video = page.video();
-        if (!statelessExecution) {
-            try { await context.storageState({ path: STORAGE_STATE_FILE }); } catch {}
-        }
+        if (!statelessExecution) {
+            try { await context.storageState({ path: STORAGE_STATE_FILE }); } catch {}
+        }
         try { await context.close(); } catch {}
         if (video) {
             try {