@duckduckgo/autoconsent 16.59.0 → 16.61.0

package/.agents/lib/regional-testing/harness.mjs

@@ -13,16 +13,8 @@
 /** @typedef {import('../../../lib/types.js').Config} Config */
 
 /**
- * Primitives the message handler uses to talk to a specific frame's content script.
- * `frameRef` is the isolated world's execution-context uniqueId the message arrived from;
- * the transport resolves it to the owning CDP session and the matching page-world context.
- * @typedef {Object} MessageTransport
- * @property {(frameRef: string, message: object) => Promise<void>} sendToContentScript
- * @property {(frameRef: string, code: string) => Promise<any>} evalInMainWorld
- */
-
-/**
- * @typedef {(transport: MessageTransport) => (msg: any, frameRef: string) => Promise<void>} MessageHandlerFactory
+ * @typedef {import('../../../playwright/isolated-world.mjs').MessageTransport} MessageTransport
+ * @typedef {import('../../../playwright/isolated-world.mjs').MessageHandlerFactory} MessageHandlerFactory
  */
 
 /**
@@ -86,6 +78,7 @@
 import fs from 'fs';
 import path from 'path';
 import { fileURLToPath } from 'url';
+import { injectIntoIsolatedWorld } from '../../../playwright/isolated-world.mjs';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 export const projectRoot = path.resolve(__dirname, '../../..');
@@ -185,7 +178,7 @@ export async function injectAutoconsent(page, options, provider, extension = {})
     if (browserName && browserName !== 'chromium') {
         throw new Error(`Regional testing supports Chromium only (got "${browserName}").`);
     }
-    await injectIntoIsolatedWorld(page, createMessageHandler, extension.script ?? '');
+    await injectIntoIsolatedWorld(page, createMessageHandler, contentScript, extension.script ?? '');
 
     function hasMessage(/** @type {string} */ type) {
         return received.some((m) => m.type === type);
@@ -259,167 +252,6 @@ export async function injectAutoconsent(page, options, provider, extension = {})
     return { received, hasMessage, waitForCompletion, waitForMessage, collectResult };
 }
 
-/**
- * Isolated-world injection via CDP (Chromium only). For each frame we create a dedicated isolated
- * world via `Page.createIsolatedWorld`, then inject the content script there and bridge messages
- * over a per-context CDP binding.
- *
- * @param {Page} page
- * @param {MessageHandlerFactory} createMessageHandler
- * @param {string} extraScript - Runs in each isolated world after the content script.
- */
-async function injectIntoIsolatedWorld(page, createMessageHandler, extraScript) {
-    // Isolated world name: `<prefix><pageWorldUniqueId><separator><frameId>`. Encoding the page-world
-    // uniqueId lets us later run eval snippets in that frame's main world.
-    const WORLD_PREFIX = 'autoconsent_iw_';
-    const WORLD_SEPARATOR = '_frame_';
-    const BINDING_PREFIX = 'autoconsentSendMessage_';
-
-    /** @type {Map<string, string>} isolated-world uniqueId -> page (main) world uniqueId */
-    const isolated2pageWorld = new Map();
-    /** @type {Map<string, any>} isolated-world uniqueId -> CDP session that owns it */
-    const sessionByContext = new Map();
-    /** @type {Map<string, string>} binding name -> isolated-world uniqueId */
-    const contextByBinding = new Map();
-
-    const handle = createMessageHandler({
-        sendToContentScript: async (isolatedUniqueId, message) => {
-            const client = sessionByContext.get(isolatedUniqueId);
-            if (!client) return;
-            try {
-                await client.send('Runtime.evaluate', {
-                    expression: `autoconsentReceiveMessage(${JSON.stringify(message)})`,
-                    uniqueContextId: isolatedUniqueId,
-                    awaitPromise: true,
-                    // Some pages' CSP would otherwise block evaluating in the isolated world.
-                    allowUnsafeEvalBlockedByCSP: true,
-                });
-            } catch {
-                // The context may be gone if the frame navigated or detached.
-            }
-        },
-        evalInMainWorld: async (isolatedUniqueId, code) => {
-            const client = sessionByContext.get(isolatedUniqueId);
-            const pageWorldUniqueId = isolated2pageWorld.get(isolatedUniqueId);
-            if (!client || !pageWorldUniqueId) return false;
-            const { result, exceptionDetails } = await client.send('Runtime.evaluate', {
-                expression: code,
-                uniqueContextId: pageWorldUniqueId,
-                returnByValue: true,
-                awaitPromise: true,
-                // Eval snippets must run even when the page's CSP disallows eval.
-                allowUnsafeEvalBlockedByCSP: true,
-            });
-            return exceptionDetails ? false : (result?.value ?? false);
-        },
-    });
-
-    async function attachToSession(/** @type {any} */ client) {
-        client.on('Runtime.executionContextCreated', async (/** @type {any} */ event) => {
-            const { context } = event;
-            const frameId = context.auxData?.frameId;
-
-            // Our isolated world finished initializing: wire up its binding and content script.
-            if (context.auxData?.type === 'isolated' && typeof context.name === 'string' && context.name.startsWith(WORLD_PREFIX)) {
-                const separatorIndex = context.name.indexOf(WORLD_SEPARATOR);
-                const pageWorldUniqueId = context.name.slice(WORLD_PREFIX.length, separatorIndex);
-                const intendedFrameId = context.name.slice(separatorIndex + WORLD_SEPARATOR.length);
-                // Chromium may create the named world in other frames too; keep only the one we asked for.
-                if (intendedFrameId !== frameId) return;
-
-                isolated2pageWorld.set(context.uniqueId, pageWorldUniqueId);
-                sessionByContext.set(context.uniqueId, client);
-
-                const bindingName = `${BINDING_PREFIX}${context.uniqueId.replace(/\W/g, '_')}`;
-                contextByBinding.set(bindingName, context.uniqueId);
-                try {
-                    await client.send('Runtime.addBinding', { name: bindingName, executionContextName: context.name });
-                    // CDP bindings take a single string arg, so wrap it in the shape the content script expects.
-                    await client.send('Runtime.evaluate', {
-                        expression: `window.autoconsentSendMessage = (m) => { window.${bindingName}(JSON.stringify(m)); return Promise.resolve(); };\n${contentScript}\n${extraScript}`,
-                        uniqueContextId: context.uniqueId,
-                        allowUnsafeEvalBlockedByCSP: true,
-                    });
-                } catch {
-                    // The frame may have navigated or detached before we finished wiring it up.
-                }
-                return;
-            }
-
-            // A regular page (main) world: request an isolated world for it, tagged with its id.
-            if (!frameId || context.auxData?.type !== 'default' || !context.origin || context.origin === '://') return;
-            try {
-                await client.send('Page.createIsolatedWorld', {
-                    frameId,
-                    worldName: `${WORLD_PREFIX}${context.uniqueId}${WORLD_SEPARATOR}${frameId}`,
-                });
-            } catch {
-                // The frame may have navigated or detached.
-            }
-        });
-
-        client.on('Runtime.executionContextDestroyed', (/** @type {any} */ event) => {
-            const uniqueId = event.executionContextUniqueId;
-            if (!uniqueId) return;
-            isolated2pageWorld.delete(uniqueId);
-            sessionByContext.delete(uniqueId);
-        });
-
-        client.on('Runtime.bindingCalled', (/** @type {any} */ event) => {
-            const isolatedUniqueId = contextByBinding.get(event.name);
-            if (!isolatedUniqueId) return;
-            let msg;
-            try {
-                msg = JSON.parse(event.payload);
-            } catch {
-                return;
-            }
-            handle(msg, isolatedUniqueId);
-        });
-
-        // Page must be enabled before createIsolatedWorld; Runtime.enable replays existing contexts.
-        await client.send('Page.enable');
-        await client.send('Runtime.enable');
-    }
-
-    // The page session covers the main frame and all same-process (in-process) iframes.
-    await attachToSession(await page.context().newCDPSession(page));
-
-    // Out-of-process iframes (OOPIFs) are separate CDP targets, so each needs its own session.
-    // newCDPSession throws for in-process frames, which are already handled above.
-    const attachedFrames = new WeakSet();
-    async function attachToOopif(/** @type {import('playwright').Frame} */ frame) {
-        if (!frame.parentFrame() || attachedFrames.has(frame)) return;
-        // Mark synchronously (before any await) so concurrent frameattached/framenavigated events
-        // can't open duplicate sessions for the same frame. Unmark on failure so a later event retries.
-        attachedFrames.add(frame);
-        let client;
-        try {
-            client = await page.context().newCDPSession(frame);
-        } catch {
-            // In-process frame (already covered by the page session) or the frame detached.
-            attachedFrames.delete(frame);
-            return;
-        }
-        try {
-            await attachToSession(client);
-        } catch {
-            // Wiring up the session failed (e.g. the OOPIF navigated/detached). Detach the
-            // half-initialized session before unmarking, otherwise a retry would open a second
-            // session for the same frame, leaving duplicate listeners and possible double injection.
-            try {
-                await client.detach();
-            } catch {
-                // The session may already be gone.
-            }
-            attachedFrames.delete(frame);
-        }
-    }
-    page.on('frameattached', attachToOopif);
-    page.on('framenavigated', attachToOopif);
-    await Promise.all(page.frames().map(attachToOopif));
-}
-
 /**
  * A result with nothing detected, optionally carrying an error.
  * @param {string} url

package/.agents/skills/oxylabs-testing/SKILL.md

@@ -69,7 +69,7 @@ Options: everything `testPage` takes in `proxy-testing` (`action`, `screenshotsD
 | Option | Default | Description |
 |--------|---------|-------------|
 | `device` | `'desktop'` | `'desktop' \| 'mobile'` — sets [`?p_device=`](https://developers.oxylabs.io/products/agent-browser/device-type) (viewport, touch, headers, User-Agent). Use it for mobile tests instead of Playwright device descriptors, which would contradict the Oxylabs fingerprint. |
-| `solveCaptcha` | `false` | Sends [`?solve_captcha=true`](https://developers.oxylabs.io/products/agent-browser/captcha-handling) so Oxylabs solves captchas on the page, and waits for the solver (see Gotchas). Accounts without the feature get HTTP 400 for every session, so leave it off unless captcha solving is enabled. |
+| `solveCaptcha` | `true` | Sends [`?solve_captcha=true`](https://developers.oxylabs.io/products/agent-browser/captcha-handling) so Oxylabs solves captchas on the page, and waits for the solver (see Gotchas). Our account has captcha solving enabled; an account without it gets HTTP 400 for every session. |
 
 Screenshots default to `test-results/oxylabs`, named `<domain>-<region>-oxylabs-final.jpg`.
 
@@ -85,13 +85,13 @@ Any two-letter country code maps to `?p_cc=` (e.g. `de` → `p_cc=DE`), per the
 
 ## Architecture
 
-Injection and result collection are shared with `proxy-testing` in [.agents/lib/regional-testing/harness.mjs](../../lib/regional-testing/harness.mjs): autoconsent runs in an isolated world of every frame (including out-of-process iframes) and talks to Node over CDP bindings, and `eval` snippets run in the page's main world. This skill only adds the Oxylabs connection and the captcha handling.
+Injection and result collection are shared with `proxy-testing` in [.agents/lib/regional-testing/harness.mjs](../../lib/regional-testing/harness.mjs): autoconsent runs in an isolated world of every frame (including out-of-process iframes) and talks to Node over CDP bindings, and `eval` snippets run in the page's main world. The isolated-world transport itself is [playwright/isolated-world.mjs](../../../playwright/isolated-world.mjs), which the Playwright E2E runner also uses for Chromium. The Oxylabs connection and the captcha tracking live in [playwright/oxylabs.mjs](../../../playwright/oxylabs.mjs), shared with the E2E runner's Oxylabs pass in CI.
 
-With `solveCaptcha: true`, a listener in autoconsent's isolated world forwards the Oxylabs runtime's captcha `window` messages to Node over the same CDP binding, so they arrive even if the page navigates right after.
+With captcha solving on, a listener in autoconsent's isolated world forwards the Oxylabs runtime's captcha `window` messages to Node over the same CDP binding, so they arrive even if the page navigates right after.
 
 ## Gotchas
 
-- **CAPTCHA events arrive via `window.postMessage`, not CDP.** With `solveCaptcha: true`, autoconsent's wait pauses whenever a start event arrives, until `oxylabs-captcha-end`, `oxylabs-captcha-solve-end` or `oxylabs-captcha-error`, for at most 60s from the first start (the timeout in Oxylabs' example). Oxylabs documents no deadline for the start event, so captcha-free pages aren't held up waiting for one. Agent Browser sends `oxylabs-captcha-solve-start` with a `captchaType` (e.g. `turnstile`); the documented `oxylabs-captcha-start` is accepted too. A Cloudflare Turnstile took about 35-40s to solve. Hard blocks without a captcha (e.g. DataDome "Access is temporarily restricted", Akamai "Access denied") send no events, so solving doesn't help there.
+- **CAPTCHA events arrive via `window.postMessage`, not CDP.** With captcha solving on, autoconsent's wait pauses whenever a start event arrives, until `oxylabs-captcha-end`, `oxylabs-captcha-solve-end` or `oxylabs-captcha-error`, for at most 60s from the first start (the timeout in Oxylabs' example). Oxylabs documents no deadline for the start event, so captcha-free pages aren't held up waiting for one. Agent Browser sends `oxylabs-captcha-solve-start` with a `captchaType` (e.g. `turnstile`); the documented `oxylabs-captcha-start` is accepted too. A Cloudflare Turnstile took about 35-40s to solve. Hard blocks without a captcha (e.g. DataDome "Access is temporarily restricted", Akamai "Access denied") send no events, so solving doesn't help there.
 - **Main-world CDP bindings don't work on Agent Browser.** `page.exposeBinding`, and `Runtime.addBinding` without `executionContextName`, never reach the page. Bindings scoped to an isolated world (what the harness uses) work, and `window` messages reach isolated-world listeners, so listen there.
 - **Limits:** 100 concurrent sessions and 10 new sessions per second per account. Going over the per-second limit fails `connectOverCDP` with `WebSocket error: ... 429 Too Many Requests`, and the library does not retry it. `testRegions` opens sessions one at a time, but a script that opens sessions in parallel (several `withOxylabsPage` calls in a `Promise.all`, or several such scripts at once) can easily send 10 within a second. Stagger the session opens (for example 200ms apart), and rerun any test that failed with a 429. A 429 is not a result about the site.
 - **Call injection before `page.goto`**, so autoconsent is in place before page scripts run.

package/.agents/skills/oxylabs-testing/scripts/oxylabs.mjs

@@ -12,20 +12,13 @@
 
 /**
  * @typedef {import('playwright').Page} Page
- * @typedef {import('playwright').Browser} Browser
  * @typedef {import('../../../lib/regional-testing/harness.mjs').TestResult} TestResult
  * @typedef {import('../../../lib/regional-testing/harness.mjs').AutoconsentContext} AutoconsentContext
  * @typedef {import('../../../lib/regional-testing/harness.mjs').Provider} Provider
  */
 
 /**
- * @typedef {'desktop'|'mobile'} OxylabsDevice
- */
-
-/**
- * @typedef {Object} OxylabsBrowserOptions
- * @property {OxylabsDevice} [device] - Oxylabs ?p_device= value. Oxylabs defaults to 'desktop'.
- * @property {boolean} [solveCaptcha=false] - Send ?solve_captcha=true: let Oxylabs solve captchas on the page. Needs the feature enabled on the account.
+ * @typedef {import('../../../../playwright/oxylabs.mjs').OxylabsBrowserOptions} OxylabsBrowserOptions
  */
 
 /**
@@ -40,7 +33,7 @@
  */
 
 import path from 'path';
-import { chromium } from 'playwright';
+import { connectOxylabs, createCaptchaTracker, redactCredentials } from '../../../../playwright/oxylabs.mjs';
 import {
     CORE_REGIONS,
     emptyResult,
@@ -50,81 +43,14 @@ import {
 } from '../../../lib/regional-testing/harness.mjs';
 
 export { ALL_REGIONS, CORE_REGIONS, EXPANDED_REGIONS, formatResult } from '../../../lib/regional-testing/harness.mjs';
-
-/**
- * US state- and city-targeted region keys, in addition to any two-letter country code. Query params
- * follow https://developers.oxylabs.io/products/agent-browser/geolocation-and-proxy-selection
- * (`p_state` is US-only; `p_city` needs `p_cc` or `p_state`).
- * @type {Record<string, Record<string, string>>}
- */
-export const OXYLABS_LOCAL_REGIONS = {
-    'us-california': { p_state: 'california' },
-    'us-newyork': { p_cc: 'US', p_city: 'new_york' },
-    'us-losangeles': { p_cc: 'US', p_city: 'los_angeles' },
-};
-
-/**
- * @param {string} regionKey - A two-letter country code (e.g. 'de', 'pt') or a key of OXYLABS_LOCAL_REGIONS.
- * @returns {Record<string, string>}
- */
-export function oxylabsRegionParams(regionKey) {
-    if (Object.prototype.hasOwnProperty.call(OXYLABS_LOCAL_REGIONS, regionKey)) {
-        return OXYLABS_LOCAL_REGIONS[regionKey];
-    }
-    if (/^[a-z]{2}$/i.test(regionKey)) {
-        return { p_cc: regionKey.toUpperCase() };
-    }
-    throw new Error(
-        `Unknown Oxylabs region "${regionKey}". Use a two-letter country code or one of: ${Object.keys(OXYLABS_LOCAL_REGIONS).join(', ')}`,
-    );
-}
-
-/**
- * @param {NodeJS.ProcessEnv} [env]
- */
-export function isOxylabsConfigured(env = process.env) {
-    return Boolean(env.OXYLABS_USER && env.OXYLABS_PASSWORD);
-}
-
-/**
- * Build the Oxylabs CDP endpoint for a region. The result contains credentials: never log it.
- * @param {string} regionKey
- * @param {OxylabsBrowserOptions} [opts]
- * @param {NodeJS.ProcessEnv} [env]
- */
-export function buildOxylabsEndpoint(regionKey, opts = {}, env = process.env) {
-    const user = env.OXYLABS_USER;
-    const password = env.OXYLABS_PASSWORD;
-    const host = env.OXYLABS_HOST || 'hb.oxylabs.io';
-    if (!user || !password) {
-        throw new Error('Missing Oxylabs credentials. Set OXYLABS_USER and OXYLABS_PASSWORD.');
-    }
-    const params = new URLSearchParams(oxylabsRegionParams(regionKey));
-    if (opts.device) {
-        params.set('p_device', opts.device);
-    }
-    // Accounts without captcha solving get HTTP 400 for this param, so only send it when asked.
-    if (opts.solveCaptcha) {
-        params.set('solve_captcha', 'true');
-    }
-    // user/password are interpolated raw — Oxylabs expects literal characters in
-    // userinfo (e.g. '+' must NOT be percent-encoded), matching their Python/JS
-    // SDK examples. If your credentials contain ':' or '@', escape upstream.
-    return `wss://${user}:${password}@${host}/?${params.toString()}`;
-}
-
-/**
- * Playwright puts the endpoint URL in connection error messages; strip the credentials.
- * @param {string} message
- * @param {NodeJS.ProcessEnv} [env]
- */
-export function redactCredentials(message, env = process.env) {
-    let out = message;
-    for (const secret of [env.OXYLABS_PASSWORD, env.OXYLABS_USER]) {
-        if (secret) out = out.split(secret).join('***');
-    }
-    return out;
-}
+export {
+    OXYLABS_LOCAL_REGIONS,
+    buildOxylabsEndpoint,
+    connectOxylabs,
+    isOxylabsConfigured,
+    oxylabsRegionParams,
+    redactCredentials,
+} from '../../../../playwright/oxylabs.mjs';
 
 /**
  * @param {TestResult} result
@@ -136,95 +62,22 @@ function redactResult(result) {
 }
 
 /**
- * Connect to an Oxylabs remote browser for a specific region.
- * @param {string} regionKey
- * @param {OxylabsBrowserOptions} [opts]
- * @returns {Promise<Browser>}
- */
-export async function connectOxylabs(regionKey, opts = {}) {
-    try {
-        return await chromium.connectOverCDP(buildOxylabsEndpoint(regionKey, opts), { timeout: 30000 });
-    } catch (e) {
-        // The first line has the cause; the call log after it repeats the endpoint URL.
-        const message = redactCredentials((e instanceof Error ? e.message : String(e)).split('\n')[0]);
-        const hint = opts.solveCaptcha && message.includes('400') ? ' (is captcha solving enabled for this Oxylabs account?)' : '';
-        throw new Error(message + hint);
-    }
-}
-
-// Captcha events arrive as window messages from the Oxylabs runtime. A listener in autoconsent's
-// isolated world forwards them over its CDP binding: Agent Browser drops main-world bindings, so
-// page.exposeBinding can't deliver them. See https://developers.oxylabs.io/products/agent-browser/captcha-handling
-const CAPTCHA_MESSAGE = 'oxylabsCaptcha';
-// Agent Browser sends `oxylabs-captcha-solve-start`; the docs list `-start`, `-end`, `-solve-end` and `-error`.
-const CAPTCHA_START_EVENTS = ['oxylabs-captcha-start', 'oxylabs-captcha-solve-start'];
-const CAPTCHA_END_EVENTS = ['oxylabs-captcha-end', 'oxylabs-captcha-solve-end'];
-const CAPTCHA_ERROR_EVENTS = ['oxylabs-captcha-error', 'oxylabs-captcha-solve-error'];
-// Longest a test pauses for solving, counted from the first start event: the wait in Oxylabs' example.
-const CAPTCHA_SOLVE_TIMEOUT_MS = 60000;
-const captchaListenerScript = `
-window.addEventListener("message", (e) => {
-    if (e?.data?.source === "oxylabs-runtime" && typeof e.data.type === "string") {
-        window.autoconsentSendMessage({ type: "${CAPTCHA_MESSAGE}", event: e.data.type });
-    }
-});
-`;
-
-/**
- * Inject autoconsent into a page's isolated world via CDP and, with `solveCaptcha`, forward
+ * Inject autoconsent into a page's isolated world via CDP and, unless `solveCaptcha` is false, forward
  * Oxylabs captcha events to the Node side. Call BEFORE navigating to the target URL.
  * @param {Page} page
  * @param {Partial<TestOptions>} [options]
  * @returns {Promise<OxylabsAutoconsentContext>}
  */
 export async function injectAutoconsent(page, options = {}) {
-    /** @type {{ detected: boolean, solved: boolean|null }} */
-    const captcha = { detected: false, solved: null };
-    /** @type {() => void} */
-    let resolveCaptcha = () => {};
-    /** @type {Promise<void>} */
-    const captchaPromise = new Promise((resolve) => {
-        resolveCaptcha = resolve;
-    });
-
-    let solving = false;
-    /** @type {number|null} */
-    let firstStartAt = null;
-
-    function handleCaptchaEvent(/** @type {string} */ type) {
-        if (CAPTCHA_START_EVENTS.includes(type)) {
-            captcha.detected = true;
-            solving = true;
-            firstStartAt ??= Date.now();
-        } else if (CAPTCHA_END_EVENTS.includes(type)) {
-            captcha.detected = true;
-            captcha.solved = true;
-            solving = false;
-            resolveCaptcha();
-        } else if (CAPTCHA_ERROR_EVENTS.includes(type)) {
-            captcha.detected = true;
-            captcha.solved = false;
-            solving = false;
-            resolveCaptcha();
-        }
-    }
-
-    // Oxylabs gives no deadline for a start event, so autoconsent's wait pauses whenever one arrives.
-    const isSolving = () => solving && firstStartAt !== null && Date.now() - firstStartAt < CAPTCHA_SOLVE_TIMEOUT_MS;
-
+    const tracker = createCaptchaTracker();
     /** @type {import('../../../lib/regional-testing/harness.mjs').IsolatedWorldExtension} */
-    const captchaListener = {
-        script: captchaListenerScript,
-        onMessage: (msg) => {
-            if (msg?.type === CAPTCHA_MESSAGE) handleCaptchaEvent(msg.event);
-        },
-    };
-    const ctx = await injectThrough(page, options, 'oxylabs', options.solveCaptcha ? captchaListener : {});
+    const captchaListener = { script: tracker.script, onMessage: tracker.handleMessage };
+    const ctx = await injectThrough(page, options, 'oxylabs', (options.solveCaptcha ?? true) ? captchaListener : {});
     return {
         ...ctx,
-        waitForCompletion: (timeout, detectionTimeout) => ctx.waitForCompletion(timeout, detectionTimeout, isSolving),
-        captcha,
-        captchaPromise,
+        waitForCompletion: (timeout, detectionTimeout) => ctx.waitForCompletion(timeout, detectionTimeout, tracker.isSolving),
+        captcha: tracker.state,
+        captchaPromise: tracker.done,
     };
 }
 

package/.agents/skills/oxylabs-testing/scripts/oxylabs.test.mjs

@@ -15,10 +15,13 @@ describe('oxylabs config', () => {
     });
 
     it('builds the endpoint with raw credentials and query params', () => {
-        assert.equal(buildOxylabsEndpoint('de', { device: 'mobile' }, env), 'wss://user:p+ss@hb.oxylabs.io/?p_cc=DE&p_device=mobile');
         assert.equal(
-            buildOxylabsEndpoint('us', { solveCaptcha: true }, { ...env, OXYLABS_HOST: 'proxy.example' }),
-            'wss://user:p+ss@proxy.example/?p_cc=US&solve_captcha=true',
+            buildOxylabsEndpoint('de', { device: 'mobile' }, env),
+            'wss://user:p+ss@hb.oxylabs.io/?p_cc=DE&p_device=mobile&solve_captcha=true',
+        );
+        assert.equal(
+            buildOxylabsEndpoint('us', { solveCaptcha: false }, { ...env, OXYLABS_HOST: 'proxy.example' }),
+            'wss://user:p+ss@proxy.example/?p_cc=US',
         );
     });
 

package/.agents/skills/verify-pr/SKILL.md

@@ -47,7 +47,7 @@ If any check fails, fix the issue and re-run before proceeding.
 
 **GitHub Actions** (`.github/workflows/checks.yml`): Runs `lint` and `test:lib`. Must pass.
 
-**Jenkins**: Runs Playwright E2E in 9 regions (US, GB, AU, CA, DE, FR, NL, CH, NO) for modified rules only. Posts a PR comment with artifact ZIP and [review tool](https://zok.pw/autoconsent-review-tool/) link for inspecting screenshots.
+**Jenkins**: Runs Playwright E2E in 9 regions (US, GB, AU, CA, DE, FR, NL, CH, NO) for modified rules only. On Chrome, tests that fail on a bot wall (navigation error, or no CMP on an error or challenge page) are rerun in an Oxylabs browser in the same region, which solves captchas, with one retry (`results-<REGION>-oxylabs.xml`, screenshots under `screenshots/oxylabs/`); a pass there counts as a pass. Posts a PR comment with artifact ZIP and [review tool](https://zok.pw/autoconsent-review-tool/) link for inspecting screenshots.
 
 ### Flaky E2E failures
 

package/AGENTS.md

@@ -77,7 +77,7 @@ Site-specific rules are rules scoped to specific sites with a `urlPattern`. Rule
 
 ### JSON Rules vs Code-based rules
 
-JSON rules live in `rules/autoconsent/` (hand-maintained) and `rules/generated/` (auto-generated). Each file defines one CMP rule following the `AutoConsentCMPRule` type in `lib/rules.ts`. **Add new JSON rules only to `rules/autoconsent/`.** Do not add new files to `rules/generated/`: only the crawler creates them. You may remove a generated rule, for example when it is stale.
+JSON rules live in `rules/autoconsent/` (hand-maintained) and `rules/generated/` (auto-generated). Each file defines one CMP rule following the `AutoConsentCMPRule` type in `lib/rules.ts`. **Add new JSON rules only to `rules/autoconsent/`.** Do not add new files to `rules/generated/`: only the crawler creates them. You may remove a generated rule, for example when it is stale. Before removing any rule, test every site in its spec with the rule removed, in the core regions and every region the spec runs in, and confirm that another rule or the heuristic handles each one, or that the site no longer shows the popup.
 
 ```json
 {

package/CHANGELOG.md

@@ -1,3 +1,110 @@
+# v16.61.0 (Sat Oct 24 2026)
+
+#### Rules
+
+- Rule update: studentapan [#2107](https://github.com/duckduckgo/autoconsent/pull/2107) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: AppConsent [#2108](https://github.com/duckduckgo/autoconsent/pull/2108) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: scb.se [#2095](https://github.com/duckduckgo/autoconsent/pull/2095) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: newsbreak, remove auto_US_newsbreak.com_0 [#2096](https://github.com/duckduckgo/autoconsent/pull/2096) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: fastweb-cookie-center [#2097](https://github.com/duckduckgo/autoconsent/pull/2097) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: gobasic [#2098](https://github.com/duckduckgo/autoconsent/pull/2098) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: com-consent-layer [#2099](https://github.com/duckduckgo/autoconsent/pull/2099) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: airbnb, remove auto_CH_airbnb.de_83z [#2100](https://github.com/duckduckgo/autoconsent/pull/2100) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: futurabolzano.it [#2101](https://github.com/duckduckgo/autoconsent/pull/2101) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: livsmedelsverket [#2102](https://github.com/duckduckgo/autoconsent/pull/2102) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: tv2reg-cookie-notice [#2104](https://github.com/duckduckgo/autoconsent/pull/2104) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: nhsapp [#2105](https://github.com/duckduckgo/autoconsent/pull/2105) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: sas, remove auto_AU_flysas.com_r1s [#2106](https://github.com/duckduckgo/autoconsent/pull/2106) ([@daxtheduck](https://github.com/daxtheduck))
+
+#### Authors: 1
+
+- Dax ([@daxtheduck](https://github.com/daxtheduck))
+
+---
+
+# v16.60.0 (Fri Oct 23 2026)
+
+#### Rules
+
+- Rule update: CPM crawl #529: autodoc.co.uk [#1534](https://github.com/duckduckgo/autoconsent/pull/1534) ([@cursoragent](https://github.com/cursoragent) [@muodov](https://github.com/muodov) [@daxtheduck](https://github.com/daxtheduck))
+- Rule update: Cookie popup on www.lovethesales.com [#2089](https://github.com/duckduckgo/autoconsent/pull/2089) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: Cookie notice on schumacher.com [#2088](https://github.com/duckduckgo/autoconsent/pull/2088) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: Cookie popup on www.elibrary.imf.org [#2091](https://github.com/duckduckgo/autoconsent/pull/2091) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: cookie pop up shop.mango.com [#2092](https://github.com/duckduckgo/autoconsent/pull/2092) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: CPM crawl #1276: parliament.uk [#2093](https://github.com/duckduckgo/autoconsent/pull/2093) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: CPM crawl #252: malmo.se [#2086](https://github.com/duckduckgo/autoconsent/pull/2086) ([@muodov](https://github.com/muodov) [@daxtheduck](https://github.com/daxtheduck))
+- Rule update: CPM crawl #29: svtplay.se [#2075](https://github.com/duckduckgo/autoconsent/pull/2075) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: CPM crawl #96: obsbygg.no [#2077](https://github.com/duckduckgo/autoconsent/pull/2077) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: CPM crawl #173: nordnet.dk [#2081](https://github.com/duckduckgo/autoconsent/pull/2081) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: CPM crawl #29: sundhed.dk [#2074](https://github.com/duckduckgo/autoconsent/pull/2074) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: CPM crawl cluster #225: asnbank.nl (sel:cookie+cookieconsent|none, 3 sites) [#2079](https://github.com/duckduckgo/autoconsent/pull/2079) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: CPM crawl #188: nederlandseloterij.nl [#2080](https://github.com/duckduckgo/autoconsent/pull/2080) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: ddg_mac_desktop (eun) CPM magic report: www.kaspersky.com [#2082](https://github.com/duckduckgo/autoconsent/pull/2082) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: CPM crawl #137: rejestr.io [#2083](https://github.com/duckduckgo/autoconsent/pull/2083) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: CPM crawl #261: bryk.pl [#2085](https://github.com/duckduckgo/autoconsent/pull/2085) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: CPM crawl #246: temu.com [#2087](https://github.com/duckduckgo/autoconsent/pull/2087) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: Onetrust [#2078](https://github.com/duckduckgo/autoconsent/pull/2078) ([@daxtheduck](https://github.com/daxtheduck) [@muodov](https://github.com/muodov))
+- Rule update: CPM crawl cluster #22: svt.se (sel:cookieconsent|none, 10 sites) [#2073](https://github.com/duckduckgo/autoconsent/pull/2073) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: dpgmedia-nl [#2042](https://github.com/duckduckgo/autoconsent/pull/2042) ([@daxtheduck](https://github.com/daxtheduck) [@muodov](https://github.com/muodov))
+- Rule update: gravito [#2046](https://github.com/duckduckgo/autoconsent/pull/2046) ([@daxtheduck](https://github.com/daxtheduck) [@muodov](https://github.com/muodov))
+- Add Italian heuristic detect patterns (x.com IT banner), remove stale twitter rule [#2072](https://github.com/duckduckgo/autoconsent/pull/2072) ([@muodov](https://github.com/muodov))
+- Rule update: bbc.com-orb-cosmetic, bbc.com [#2040](https://github.com/duckduckgo/autoconsent/pull/2040) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: cookieprompter [#2071](https://github.com/duckduckgo/autoconsent/pull/2071) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: ok [#2070](https://github.com/duckduckgo/autoconsent/pull/2070) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: amazon-music [#2069](https://github.com/duckduckgo/autoconsent/pull/2069) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: arbeidsplassen-nav-no [#2068](https://github.com/duckduckgo/autoconsent/pull/2068) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: bokadirekt.se [#2066](https://github.com/duckduckgo/autoconsent/pull/2066) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: pe-cookies [#2065](https://github.com/duckduckgo/autoconsent/pull/2065) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: att [#2064](https://github.com/duckduckgo/autoconsent/pull/2064) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: mainichi [#2063](https://github.com/duckduckgo/autoconsent/pull/2063) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: pensionsmyndigheten [#2062](https://github.com/duckduckgo/autoconsent/pull/2062) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: ing-de [#2061](https://github.com/duckduckgo/autoconsent/pull/2061) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: auctionet [#2059](https://github.com/duckduckgo/autoconsent/pull/2059) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: eroticiracconti-it [#2058](https://github.com/duckduckgo/autoconsent/pull/2058) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: sitevision [#2057](https://github.com/duckduckgo/autoconsent/pull/2057) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: zorginstituut-cookiewarning [#2056](https://github.com/duckduckgo/autoconsent/pull/2056) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: nordea [#2055](https://github.com/duckduckgo/autoconsent/pull/2055) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: bouvet-cookiepanel [#2054](https://github.com/duckduckgo/autoconsent/pull/2054) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: pro-euclp-cookies-box [#2053](https://github.com/duckduckgo/autoconsent/pull/2053) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: lantmateriet [#2051](https://github.com/duckduckgo/autoconsent/pull/2051) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: annunci69.it [#2050](https://github.com/duckduckgo/autoconsent/pull/2050) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: jobindex [#2048](https://github.com/duckduckgo/autoconsent/pull/2048) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: intesasanpaolo [#2044](https://github.com/duckduckgo/autoconsent/pull/2044) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: skatteverket-se [#2043](https://github.com/duckduckgo/autoconsent/pull/2043) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: wirtualna-polska [#2039](https://github.com/duckduckgo/autoconsent/pull/2039) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: onlineradiobox [#2038](https://github.com/duckduckgo/autoconsent/pull/2038) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: smile-insurances [#2037](https://github.com/duckduckgo/autoconsent/pull/2037) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: rambler [#2036](https://github.com/duckduckgo/autoconsent/pull/2036) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: cookielegit [#2035](https://github.com/duckduckgo/autoconsent/pull/2035) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: ersatzteile-24 [#2034](https://github.com/duckduckgo/autoconsent/pull/2034) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: aeolservice [#2033](https://github.com/duckduckgo/autoconsent/pull/2033) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: xenforo-cookie-consent [#2032](https://github.com/duckduckgo/autoconsent/pull/2032) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: hacon-next-gen [#2041](https://github.com/duckduckgo/autoconsent/pull/2041) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: consent-studio [#1999](https://github.com/duckduckgo/autoconsent/pull/1999) ([@daxtheduck](https://github.com/daxtheduck) [@muodov](https://github.com/muodov))
+- Rule update: puesc-gov-pl [#2031](https://github.com/duckduckgo/autoconsent/pull/2031) ([@daxtheduck](https://github.com/daxtheduck))
+- Rule update: eon.pl [#1991](https://github.com/duckduckgo/autoconsent/pull/1991) ([@daxtheduck](https://github.com/daxtheduck))
+
+#### AI / Agent Workflow
+
+- Test a rule's spec sites before removing it [#2094](https://github.com/duckduckgo/autoconsent/pull/2094) ([@muodov](https://github.com/muodov))
+
+#### Test Infrastructure
+
+- Run CI E2E tests in an isolated world and rerun bot-walled tests on Oxylabs [#1672](https://github.com/duckduckgo/autoconsent/pull/1672) ([@muodov](https://github.com/muodov))
+
+#### Internal
+
+- Detect more types of popup [#1769](https://github.com/duckduckgo/autoconsent/pull/1769) ([@sammacbeth](https://github.com/sammacbeth))
+
+#### Authors: 4
+
+- Cursor Agent ([@cursoragent](https://github.com/cursoragent))
+- Dax ([@daxtheduck](https://github.com/daxtheduck))
+- Maxim Tsoy ([@muodov](https://github.com/muodov))
+- Sam Macbeth ([@sammacbeth](https://github.com/sammacbeth))
+
+---
+
 # v16.59.0 (Thu Oct 22 2026)
 
 #### Rules