@duckduckgo/autoconsent 16.58.0 → 16.60.0

package/playwright/isolated-world.mjs

@@ -0,0 +1,217 @@
+/**
+ * Runs the autoconsent content script in an isolated world of every frame of a Chromium page (via CDP),
+ * the way the browser extension does, and bridges its messages to Node. Shared by the Playwright E2E
+ * runner and the regional-testing harness used by agent skills.
+ */
+
+/**
+ * @typedef {import('playwright').Page} Page
+ */
+
+/**
+ * Primitives the message handler uses to talk to a specific frame's content script.
+ * `frameRef` is the isolated world's execution-context uniqueId the message arrived from;
+ * the transport resolves it to the owning CDP session and the matching page-world context.
+ * @typedef {Object} MessageTransport
+ * @property {(frameRef: string, message: object) => Promise<void>} sendToContentScript
+ * @property {(frameRef: string, code: string) => Promise<any>} evalInMainWorld
+ * @property {(frameRef: string) => boolean} isMainFrame - Whether the frame is the page's top frame.
+ */
+
+/**
+ * @typedef {(transport: MessageTransport) => (msg: any, frameRef: string) => Promise<void>} MessageHandlerFactory
+ */
+
+/**
+ * Isolated-world injection via CDP (Chromium only). For each frame we create a dedicated isolated
+ * world via `Page.createIsolatedWorld`, then inject the content script there and bridge messages
+ * over a per-context CDP binding.
+ *
+ * @param {Page} page
+ * @param {MessageHandlerFactory} createMessageHandler
+ * @param {string} contentScript - The autoconsent content script bundle.
+ * @param {string} [extraScript] - Runs in each isolated world after the content script.
+ */
+export async function injectIntoIsolatedWorld(page, createMessageHandler, contentScript, extraScript = '') {
+    // Isolated world name: `<prefix><pageWorldUniqueId><separator><frameId>`. Encoding the page-world
+    // uniqueId lets us later run eval snippets in that frame's main world.
+    const WORLD_PREFIX = 'autoconsent_iw_';
+    const WORLD_SEPARATOR = '_frame_';
+    const BINDING_PREFIX = 'autoconsentSendMessage_';
+
+    /** @type {Map<string, string>} isolated-world uniqueId -> page (main) world uniqueId */
+    const isolated2pageWorld = new Map();
+    /** @type {Map<string, any>} isolated-world uniqueId -> CDP session that owns it */
+    const sessionByContext = new Map();
+    /** @type {Map<string, string>} binding name -> isolated-world uniqueId */
+    const contextByBinding = new Map();
+    /** @type {Map<string, string>} isolated-world uniqueId -> binding name */
+    const bindingByContext = new Map();
+    /** @type {Map<string, string>} isolated-world uniqueId -> CDP frame id */
+    const frameByContext = new Map();
+    /** @type {Map<string, string>} CDP frame id -> uniqueId of the frame's current page (main) world */
+    const pageWorldByFrame = new Map();
+    /** @type {any} */
+    let pageSession = null;
+    /** @type {string|null} */
+    let mainFrameId = null;
+
+    const handle = createMessageHandler({
+        sendToContentScript: async (isolatedUniqueId, message) => {
+            const client = sessionByContext.get(isolatedUniqueId);
+            if (!client) return;
+            try {
+                await client.send('Runtime.evaluate', {
+                    expression: `autoconsentReceiveMessage(${JSON.stringify(message)})`,
+                    uniqueContextId: isolatedUniqueId,
+                    awaitPromise: true,
+                    // Some pages' CSP would otherwise block evaluating in the isolated world.
+                    allowUnsafeEvalBlockedByCSP: true,
+                });
+            } catch {
+                // The context may be gone if the frame navigated or detached.
+            }
+        },
+        evalInMainWorld: async (isolatedUniqueId, code) => {
+            const client = sessionByContext.get(isolatedUniqueId);
+            const pageWorldUniqueId = isolated2pageWorld.get(isolatedUniqueId);
+            if (!client || !pageWorldUniqueId) return false;
+            const { result, exceptionDetails } = await client.send('Runtime.evaluate', {
+                expression: code,
+                uniqueContextId: pageWorldUniqueId,
+                returnByValue: true,
+                awaitPromise: true,
+                // Eval snippets must run even when the page's CSP disallows eval.
+                allowUnsafeEvalBlockedByCSP: true,
+            });
+            return exceptionDetails ? false : (result?.value ?? false);
+        },
+        isMainFrame: (isolatedUniqueId) =>
+            sessionByContext.get(isolatedUniqueId) === pageSession && frameByContext.get(isolatedUniqueId) === mainFrameId,
+    });
+
+    async function attachToSession(/** @type {any} */ client) {
+        client.on('Runtime.executionContextCreated', async (/** @type {any} */ event) => {
+            const { context } = event;
+            const frameId = context.auxData?.frameId;
+
+            // Our isolated world finished initializing: wire up its binding and content script.
+            if (context.auxData?.type === 'isolated' && typeof context.name === 'string' && context.name.startsWith(WORLD_PREFIX)) {
+                const separatorIndex = context.name.indexOf(WORLD_SEPARATOR);
+                const pageWorldUniqueId = context.name.slice(WORLD_PREFIX.length, separatorIndex);
+                const intendedFrameId = context.name.slice(separatorIndex + WORLD_SEPARATOR.length);
+                // Chromium may create the named world in other frames too, and recreates it in the frame's
+                // next document; keep only the one we asked for in the current document.
+                if (intendedFrameId !== frameId || pageWorldByFrame.get(frameId) !== pageWorldUniqueId) return;
+
+                isolated2pageWorld.set(context.uniqueId, pageWorldUniqueId);
+                sessionByContext.set(context.uniqueId, client);
+                frameByContext.set(context.uniqueId, frameId);
+
+                const bindingName = `${BINDING_PREFIX}${context.uniqueId.replace(/\W/g, '_')}`;
+                contextByBinding.set(bindingName, context.uniqueId);
+                bindingByContext.set(context.uniqueId, bindingName);
+                try {
+                    await client.send('Runtime.addBinding', { name: bindingName, executionContextName: context.name });
+                    // CDP bindings take a single string arg, so wrap it in the shape the content script expects.
+                    await client.send('Runtime.evaluate', {
+                        expression: `window.autoconsentSendMessage = (m) => { window.${bindingName}(JSON.stringify(m)); return Promise.resolve(); };\n${contentScript}\n${extraScript}`,
+                        uniqueContextId: context.uniqueId,
+                        allowUnsafeEvalBlockedByCSP: true,
+                    });
+                } catch {
+                    // The frame may have navigated or detached before we finished wiring it up.
+                }
+                return;
+            }
+
+            // A regular page (main) world: request an isolated world for it, tagged with its id.
+            if (!frameId || context.auxData?.type !== 'default') return;
+            pageWorldByFrame.set(frameId, context.uniqueId);
+            if (!context.origin || context.origin === '://') return;
+            try {
+                await client.send('Page.createIsolatedWorld', {
+                    frameId,
+                    worldName: `${WORLD_PREFIX}${context.uniqueId}${WORLD_SEPARATOR}${frameId}`,
+                });
+            } catch {
+                // The frame may have navigated or detached.
+            }
+        });
+
+        client.on('Runtime.executionContextDestroyed', (/** @type {any} */ event) => {
+            const uniqueId = event.executionContextUniqueId;
+            if (!uniqueId) return;
+            isolated2pageWorld.delete(uniqueId);
+            sessionByContext.delete(uniqueId);
+            frameByContext.delete(uniqueId);
+            const bindingName = bindingByContext.get(uniqueId);
+            if (bindingName) {
+                bindingByContext.delete(uniqueId);
+                contextByBinding.delete(bindingName);
+                // Each document gets its own binding, so drop the old one with its context.
+                client.send('Runtime.removeBinding', { name: bindingName }).catch(() => {});
+            }
+        });
+
+        client.on('Runtime.bindingCalled', (/** @type {any} */ event) => {
+            const isolatedUniqueId = contextByBinding.get(event.name);
+            if (!isolatedUniqueId) return;
+            let msg;
+            try {
+                msg = JSON.parse(event.payload);
+            } catch {
+                return;
+            }
+            handle(msg, isolatedUniqueId);
+        });
+
+        // Page must be enabled before createIsolatedWorld; Runtime.enable replays existing contexts.
+        await client.send('Page.enable');
+        await client.send('Runtime.enable');
+    }
+
+    // The page session covers the main frame and all same-process (in-process) iframes.
+    pageSession = await page.context().newCDPSession(page);
+    const { frameTree } = await pageSession.send('Page.getFrameTree');
+    mainFrameId = frameTree.frame.id;
+    // Follow the top frame if Chromium replaces it (e.g. on prerender activation).
+    pageSession.on('Page.frameNavigated', (/** @type {any} */ event) => {
+        if (!event.frame.parentId) mainFrameId = event.frame.id;
+    });
+    await attachToSession(pageSession);
+
+    // Out-of-process iframes (OOPIFs) are separate CDP targets, so each needs its own session.
+    // newCDPSession throws for in-process frames, which are already handled above.
+    const attachedFrames = new WeakSet();
+    async function attachToOopif(/** @type {import('playwright').Frame} */ frame) {
+        if (!frame.parentFrame() || attachedFrames.has(frame)) return;
+        // Mark synchronously (before any await) so concurrent frameattached/framenavigated events
+        // can't open duplicate sessions for the same frame. Unmark on failure so a later event retries.
+        attachedFrames.add(frame);
+        let client;
+        try {
+            client = await page.context().newCDPSession(frame);
+        } catch {
+            // In-process frame (already covered by the page session) or the frame detached.
+            attachedFrames.delete(frame);
+            return;
+        }
+        try {
+            await attachToSession(client);
+        } catch {
+            // Wiring up the session failed (e.g. the OOPIF navigated/detached). Detach the
+            // half-initialized session before unmarking, otherwise a retry would open a second
+            // session for the same frame, leaving duplicate listeners and possible double injection.
+            try {
+                await client.detach();
+            } catch {
+                // The session may already be gone.
+            }
+            attachedFrames.delete(frame);
+        }
+    }
+    page.on('frameattached', attachToOopif);
+    page.on('framenavigated', attachToOopif);
+    await Promise.all(page.frames().map(attachToOopif));
+}

package/playwright/oxylabs.mjs

@@ -0,0 +1,196 @@
+/**
+ * Oxylabs Agent Browser connection: region targeting, the CDP endpoint, and credential redaction.
+ * Docs: https://developers.oxylabs.io/products/agent-browser
+ * Shared by the Playwright E2E runner and the `oxylabs-testing` skill.
+ *
+ * Requires env vars: OXYLABS_USER, OXYLABS_PASSWORD.
+ * Optional: OXYLABS_HOST (defaults to hb.oxylabs.io).
+ */
+
+/**
+ * @typedef {import('@playwright/test').Browser} Browser
+ */
+
+/**
+ * @typedef {'desktop'|'mobile'} OxylabsDevice
+ */
+
+/**
+ * @typedef {Object} OxylabsBrowserOptions
+ * @property {OxylabsDevice} [device] - Oxylabs ?p_device= value. Oxylabs defaults to 'desktop'.
+ * @property {boolean} [solveCaptcha=true] - Send ?solve_captcha=true: let Oxylabs solve captchas on the page.
+ */
+
+import { chromium } from '@playwright/test';
+
+/**
+ * US state- and city-targeted region keys, in addition to any two-letter country code. Query params
+ * follow https://developers.oxylabs.io/products/agent-browser/geolocation-and-proxy-selection
+ * (`p_state` is US-only; `p_city` needs `p_cc` or `p_state`).
+ * @type {Record<string, Record<string, string>>}
+ */
+export const OXYLABS_LOCAL_REGIONS = {
+    'us-california': { p_state: 'california' },
+    'us-newyork': { p_cc: 'US', p_city: 'new_york' },
+    'us-losangeles': { p_cc: 'US', p_city: 'los_angeles' },
+};
+
+/**
+ * @param {string} regionKey - A two-letter country code (e.g. 'de', 'pt') or a key of OXYLABS_LOCAL_REGIONS.
+ * @returns {Record<string, string>}
+ */
+export function oxylabsRegionParams(regionKey) {
+    if (Object.prototype.hasOwnProperty.call(OXYLABS_LOCAL_REGIONS, regionKey)) {
+        return OXYLABS_LOCAL_REGIONS[regionKey];
+    }
+    if (/^[a-z]{2}$/i.test(regionKey)) {
+        return { p_cc: regionKey.toUpperCase() };
+    }
+    throw new Error(
+        `Unknown Oxylabs region "${regionKey}". Use a two-letter country code or one of: ${Object.keys(OXYLABS_LOCAL_REGIONS).join(', ')}`,
+    );
+}
+
+/**
+ * @param {NodeJS.ProcessEnv} [env]
+ */
+export function isOxylabsConfigured(env = process.env) {
+    return Boolean(env.OXYLABS_USER && env.OXYLABS_PASSWORD);
+}
+
+/**
+ * Build the Oxylabs CDP endpoint for a region. The result contains credentials: never log it.
+ * @param {string} regionKey
+ * @param {OxylabsBrowserOptions} [opts]
+ * @param {NodeJS.ProcessEnv} [env]
+ */
+export function buildOxylabsEndpoint(regionKey, opts = {}, env = process.env) {
+    const user = env.OXYLABS_USER;
+    const password = env.OXYLABS_PASSWORD;
+    const host = env.OXYLABS_HOST || 'hb.oxylabs.io';
+    if (!user || !password) {
+        throw new Error('Missing Oxylabs credentials. Set OXYLABS_USER and OXYLABS_PASSWORD.');
+    }
+    const params = new URLSearchParams(oxylabsRegionParams(regionKey));
+    if (opts.device) {
+        params.set('p_device', opts.device);
+    }
+    // Accounts without captcha solving get HTTP 400 for this param, so only send it when asked.
+    if (opts.solveCaptcha ?? true) {
+        params.set('solve_captcha', 'true');
+    }
+    // user/password are interpolated raw — Oxylabs expects literal characters in
+    // userinfo (e.g. '+' must NOT be percent-encoded), matching their Python/JS
+    // SDK examples. If your credentials contain ':' or '@', escape upstream.
+    return `wss://${user}:${password}@${host}/?${params.toString()}`;
+}
+
+/**
+ * Playwright puts the endpoint URL in connection error messages; strip the credentials.
+ * @param {string} message
+ * @param {NodeJS.ProcessEnv} [env]
+ */
+export function redactCredentials(message, env = process.env) {
+    let out = message;
+    for (const secret of [env.OXYLABS_PASSWORD, env.OXYLABS_USER]) {
+        if (secret) out = out.split(secret).join('***');
+    }
+    return out;
+}
+
+/**
+ * Connect to an Oxylabs remote browser for a specific region.
+ * @param {string} regionKey
+ * @param {OxylabsBrowserOptions} [opts]
+ * @returns {Promise<Browser>}
+ */
+export async function connectOxylabs(regionKey, opts = {}) {
+    try {
+        return await chromium.connectOverCDP(buildOxylabsEndpoint(regionKey, opts), { timeout: 30000 });
+    } catch (e) {
+        // The first line has the cause; the call log after it repeats the endpoint URL.
+        const message = redactCredentials((e instanceof Error ? e.message : String(e)).split('\n')[0]);
+        const hint =
+            (opts.solveCaptcha ?? true) && message.includes('400') ? ' (is captcha solving enabled for this Oxylabs account?)' : '';
+        // eslint-disable-next-line preserve-caught-error -- the original error carries the credentials
+        throw new Error(message + hint);
+    }
+}
+
+// Captcha events arrive as window messages from the Oxylabs runtime. A listener in autoconsent's
+// isolated world forwards them over its CDP binding: Agent Browser drops main-world bindings, so
+// page.exposeBinding can't deliver them. See https://developers.oxylabs.io/products/agent-browser/captcha-handling
+const CAPTCHA_MESSAGE = 'oxylabsCaptcha';
+// Agent Browser sends `oxylabs-captcha-solve-start`; the docs list `-start`, `-end`, `-solve-end` and `-error`.
+const CAPTCHA_START_EVENTS = ['oxylabs-captcha-start', 'oxylabs-captcha-solve-start'];
+const CAPTCHA_END_EVENTS = ['oxylabs-captcha-end', 'oxylabs-captcha-solve-end'];
+const CAPTCHA_ERROR_EVENTS = ['oxylabs-captcha-error', 'oxylabs-captcha-solve-error'];
+// Longest a test pauses for solving, counted from the first start event: the wait in Oxylabs' example.
+const CAPTCHA_SOLVE_TIMEOUT_MS = 60000;
+const captchaListenerScript = `
+window.addEventListener("message", (e) => {
+    if (e?.data?.source === "oxylabs-runtime" && typeof e.data.type === "string") {
+        window.autoconsentSendMessage({ type: "${CAPTCHA_MESSAGE}", event: e.data.type });
+    }
+});
+`;
+
+/**
+ * Tracks the Oxylabs captcha solver for one page. Run `script` in each isolated world next to the
+ * content script and pass every message from it to `handleMessage`.
+ * @typedef {Object} CaptchaTracker
+ * @property {{ detected: boolean, solved: boolean|null }} state - The solver state.
+ * @property {Promise<void>} done - Resolves when solving ends (success or error).
+ * @property {() => boolean} isSolving - Whether waits should pause for the solver.
+ * @property {string} script
+ * @property {(msg: any) => boolean} handleMessage - Returns true for a captcha event.
+ */
+
+/**
+ * @returns {CaptchaTracker}
+ */
+export function createCaptchaTracker() {
+    /** @type {{ detected: boolean, solved: boolean|null }} */
+    const state = { detected: false, solved: null };
+    /** @type {() => void} */
+    let resolveDone = () => {};
+    /** @type {Promise<void>} */
+    const done = new Promise((resolve) => {
+        resolveDone = resolve;
+    });
+
+    let solving = false;
+    /** @type {number|null} */
+    let firstStartAt = null;
+
+    function handleCaptchaEvent(/** @type {string} */ type) {
+        if (CAPTCHA_START_EVENTS.includes(type)) {
+            state.detected = true;
+            solving = true;
+            firstStartAt ??= Date.now();
+        } else if (CAPTCHA_END_EVENTS.includes(type)) {
+            state.detected = true;
+            state.solved = true;
+            solving = false;
+            resolveDone();
+        } else if (CAPTCHA_ERROR_EVENTS.includes(type)) {
+            state.detected = true;
+            state.solved = false;
+            solving = false;
+            resolveDone();
+        }
+    }
+
+    return {
+        state,
+        done,
+        // Oxylabs gives no deadline for a start event, so waits pause whenever one arrives.
+        isSolving: () => solving && firstStartAt !== null && Date.now() - firstStartAt < CAPTCHA_SOLVE_TIMEOUT_MS,
+        script: captchaListenerScript,
+        handleMessage: (msg) => {
+            if (msg?.type !== CAPTCHA_MESSAGE) return false;
+            handleCaptchaEvent(msg.event);
+            return true;
+        },
+    };
+}