@duckduckgo/autoconsent 16.43.2 → 16.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/lib/regional-testing/harness.mjs +536 -0
- package/.agents/skills/oxylabs-testing/SKILL.md +99 -0
- package/.agents/skills/oxylabs-testing/scripts/oxylabs.mjs +300 -0
- package/.agents/skills/oxylabs-testing/scripts/oxylabs.test.mjs +37 -0
- package/.agents/skills/proxy-testing/SKILL.md +5 -2
- package/.agents/skills/proxy-testing/scripts/regional-proxy.mjs +22 -471
- package/.claude/skills/oxylabs-testing/SKILL.md +6 -0
- package/.claude/skills/proxy-testing/SKILL.md +1 -1
- package/.claude/user-settings.json.TEMPLATE +2 -0
- package/AGENTS.md +1 -1
- package/CHANGELOG.md +29 -0
- package/dist/addon-firefox/background.bundle.js +9 -3
- package/dist/addon-firefox/compact-rules.json +1 -1
- package/dist/addon-firefox/manifest.json +1 -1
- package/dist/addon-firefox/popup.bundle.js +9 -3
- package/dist/addon-firefox/rule-index.json +1 -1
- package/dist/addon-firefox/rules.json +1 -1
- package/dist/addon-mv3/background.bundle.js +9 -3
- package/dist/addon-mv3/compact-rules.json +1 -1
- package/dist/addon-mv3/manifest.json +1 -1
- package/dist/addon-mv3/popup.bundle.js +9 -3
- package/dist/addon-mv3/rule-index.json +1 -1
- package/dist/addon-mv3/rules.json +1 -1
- package/dist/autoconsent.standalone.js +1 -1
- package/package.json +2 -2
- package/readme.md +1 -1
- package/rules/autoconsent/bandcamp.json +42 -3
- package/rules/autoconsent/chatgpt.json +2 -2
- package/rules/autoconsent/elsevier.json +69 -0
- package/rules/autoconsent/fides.json +10 -2
- package/rules/autoconsent/lacoccinelle.json +97 -0
- package/rules/autoconsent/woo-commerce-com.json +28 -11
- package/rules/compact-rules.json +1 -1
- package/rules/rule-index.json +1 -1
- package/rules/rules.json +1 -1
- package/tests/bandcamp.spec.ts +7 -1
- package/tests/elsevier.spec.ts +3 -0
- package/tests/fides.spec.ts +7 -1
- package/tests/lacoccinelle.spec.ts +3 -0
- package/tests/woo-commerce-com.spec.ts +4 -1
|
@@ -15,80 +15,32 @@
|
|
|
15
15
|
* @typedef {import('playwright').Page} Page
|
|
16
16
|
* @typedef {import('playwright').Browser} Browser
|
|
17
17
|
* @typedef {import('playwright').LaunchOptions} LaunchOptions
|
|
18
|
+
* @typedef {import('../../../lib/regional-testing/harness.mjs').TestResult} TestResult
|
|
19
|
+
* @typedef {import('../../../lib/regional-testing/harness.mjs').AutoconsentContext} AutoconsentContext
|
|
20
|
+
* @typedef {import('../../../lib/regional-testing/harness.mjs').Provider} Provider
|
|
18
21
|
*/
|
|
19
22
|
|
|
20
23
|
/**
|
|
21
|
-
* @typedef {Object}
|
|
22
|
-
* @property {'optOut'|'optIn'|null} [action='optOut']
|
|
23
|
-
* @property {string} [screenshotsDir]
|
|
24
|
-
* @property {number} [navigationTimeout=45000]
|
|
25
|
-
* @property {number} [completionTimeout=45000]
|
|
26
|
-
* @property {number} [detectionTimeout] - How long to wait for `cmpDetected` before giving up. Defaults to `completionTimeout`.
|
|
24
|
+
* @typedef {Object} RegionalProxyOptions
|
|
27
25
|
* @property {boolean} [headless=true]
|
|
28
26
|
* @property {LaunchOptions} [launchOptions]
|
|
29
27
|
*/
|
|
30
28
|
|
|
31
29
|
/**
|
|
32
|
-
* @typedef {
|
|
33
|
-
* @property {string} url
|
|
34
|
-
* @property {string} region
|
|
35
|
-
* @property {string[]} cmpsDetected - All CMPs detected on the page.
|
|
36
|
-
* @property {string|null} cmpActedOn - The CMP that was actually opted out/in.
|
|
37
|
-
* @property {boolean} popupFound
|
|
38
|
-
* @property {boolean|null} optOutResult
|
|
39
|
-
* @property {boolean|null} optInResult
|
|
40
|
-
* @property {boolean|null} selfTestResult
|
|
41
|
-
* @property {boolean} autoconsentDone
|
|
42
|
-
* @property {boolean|null} isCosmetic
|
|
43
|
-
* @property {string[]} errors
|
|
44
|
-
* @property {number} duration
|
|
45
|
-
* @property {string[]} screenshotPaths
|
|
30
|
+
* @typedef {import('../../../lib/regional-testing/harness.mjs').TestOptions & RegionalProxyOptions} TestOptions
|
|
46
31
|
*/
|
|
47
32
|
|
|
48
|
-
/**
|
|
49
|
-
* @typedef {Object} AutoconsentContext
|
|
50
|
-
* @property {Object[]} received - All received autoconsent messages.
|
|
51
|
-
* @property {(type: string) => boolean} hasMessage
|
|
52
|
-
* @property {(timeout?: number, detectionTimeout?: number) => Promise<boolean>} waitForCompletion
|
|
53
|
-
* @property {(type: string, timeout?: number) => Promise<boolean>} waitForMessage
|
|
54
|
-
* @property {(url: string, region: string) => TestResult} collectResult
|
|
55
|
-
*/
|
|
56
|
-
|
|
57
|
-
/** @typedef {import('../../../../lib/types').Config} Config */
|
|
58
|
-
|
|
59
|
-
/**
|
|
60
|
-
* Primitives the message handler uses to talk to a specific frame's content script.
|
|
61
|
-
* `frameRef` is the isolated world's execution-context uniqueId the message arrived from;
|
|
62
|
-
* the transport resolves it to the owning CDP session and the matching page-world context.
|
|
63
|
-
* @typedef {Object} MessageTransport
|
|
64
|
-
* @property {(frameRef: string, message: object) => Promise<void>} sendToContentScript
|
|
65
|
-
* @property {(frameRef: string, code: string) => Promise<any>} evalInMainWorld
|
|
66
|
-
*/
|
|
67
|
-
|
|
68
|
-
/**
|
|
69
|
-
* @typedef {(transport: MessageTransport) => (msg: any, frameRef: string) => Promise<void>} MessageHandlerFactory
|
|
70
|
-
*/
|
|
71
|
-
|
|
72
|
-
import fs from 'fs';
|
|
73
33
|
import path from 'path';
|
|
74
|
-
import { fileURLToPath } from 'url';
|
|
75
34
|
import { chromium } from 'playwright';
|
|
35
|
+
import {
|
|
36
|
+
CORE_REGIONS,
|
|
37
|
+
emptyResult,
|
|
38
|
+
injectAutoconsent as injectThrough,
|
|
39
|
+
projectRoot,
|
|
40
|
+
runTest,
|
|
41
|
+
} from '../../../lib/regional-testing/harness.mjs';
|
|
76
42
|
|
|
77
|
-
|
|
78
|
-
const projectRoot = path.resolve(__dirname, '../../../..');
|
|
79
|
-
|
|
80
|
-
const contentScript = fs.readFileSync(path.join(projectRoot, 'dist/autoconsent.playwright.js'), 'utf8');
|
|
81
|
-
const rulesJson = JSON.parse(fs.readFileSync(path.join(projectRoot, 'rules/rules.json'), 'utf-8'));
|
|
82
|
-
const fullRules = rulesJson.autoconsent;
|
|
83
|
-
|
|
84
|
-
/** All supported proxy regions. */
|
|
85
|
-
export const ALL_REGIONS = ['us', 'gb', 'au', 'ca', 'de', 'fr', 'nl', 'ch', 'no', 'it', 'es', 'pl', 'se', 'dk', 'jp'];
|
|
86
|
-
|
|
87
|
-
/** Core pass: default verification set (CCPA, UK GDPR, EEA GDPR). Add the reported region when not already covered. */
|
|
88
|
-
export const CORE_REGIONS = ['us', 'gb', 'de'];
|
|
89
|
-
|
|
90
|
-
/** Expanded pass: escalation set covering all non-GDPR regimes plus GDPR representatives. */
|
|
91
|
-
export const EXPANDED_REGIONS = ['us', 'gb', 'de', 'fr', 'nl', 'pl', 'au', 'ca', 'jp'];
|
|
43
|
+
export { ALL_REGIONS, CORE_REGIONS, EXPANDED_REGIONS, formatResult } from '../../../lib/regional-testing/harness.mjs';
|
|
92
44
|
|
|
93
45
|
/**
|
|
94
46
|
* Build the Playwright proxy object for a region.
|
|
@@ -146,321 +98,15 @@ export async function launchRegionalProxyBrowser(regionKey, options = {}) {
|
|
|
146
98
|
* @returns {Promise<AutoconsentContext>}
|
|
147
99
|
*/
|
|
148
100
|
export async function injectAutoconsent(page, options = {}) {
|
|
149
|
-
|
|
150
|
-
/** @type {any[]} */
|
|
151
|
-
const received = [];
|
|
152
|
-
/** @type {Partial<Config>} */
|
|
153
|
-
const config = {
|
|
154
|
-
enabled: true,
|
|
155
|
-
autoAction: action,
|
|
156
|
-
disabledCmps: [],
|
|
157
|
-
enablePrehide: true,
|
|
158
|
-
detectRetries: 20,
|
|
159
|
-
enableCosmeticRules: true,
|
|
160
|
-
enableGeneratedRules: true,
|
|
161
|
-
enableHeuristicDetection: true,
|
|
162
|
-
heuristicMode: 'tier2',
|
|
163
|
-
// The engine runs in the isolated world; eval snippets are forwarded to the main world.
|
|
164
|
-
isMainWorld: false,
|
|
165
|
-
logs: { lifecycle: true, rulesteps: true, detectionsteps: false, evals: false, errors: true },
|
|
166
|
-
};
|
|
167
|
-
|
|
168
|
-
// Remembers which frame (and its transport's sender) asked for a self test, so the
|
|
169
|
-
// follow-up `selfTest` message is routed back to the same content-script context even
|
|
170
|
-
// when several frames/CDP sessions are involved.
|
|
171
|
-
/** @type {{ send: MessageTransport['sendToContentScript'], frameRef: any } | null} */
|
|
172
|
-
let selfTestTarget = null;
|
|
173
|
-
|
|
174
|
-
// The transport-agnostic message handler. A transport supplies two primitives:
|
|
175
|
-
// sendToContentScript(frameRef, message) - deliver to autoconsentReceiveMessage in the content-script world
|
|
176
|
-
// evalInMainWorld(frameRef, code) - run an eval snippet in the frame's MAIN world
|
|
177
|
-
// This split mirrors the browser extension: the engine lives in an isolated world,
|
|
178
|
-
// while `eval` snippets execute in the page's main world.
|
|
179
|
-
/** @type {MessageHandlerFactory} */
|
|
180
|
-
const createMessageHandler = ({ sendToContentScript, evalInMainWorld }) => {
|
|
181
|
-
return async function handleMessage(msg, frameRef) {
|
|
182
|
-
received.push(msg);
|
|
183
|
-
switch (msg.type) {
|
|
184
|
-
case 'init':
|
|
185
|
-
await sendToContentScript(frameRef, { type: 'initResp', config, rules: { autoconsent: fullRules } });
|
|
186
|
-
break;
|
|
187
|
-
case 'eval': {
|
|
188
|
-
let result = false;
|
|
189
|
-
try {
|
|
190
|
-
result = await evalInMainWorld(frameRef, msg.code);
|
|
191
|
-
} catch {}
|
|
192
|
-
await sendToContentScript(frameRef, { id: msg.id, type: 'evalResp', result });
|
|
193
|
-
break;
|
|
194
|
-
}
|
|
195
|
-
case 'optOutResult':
|
|
196
|
-
case 'optInResult':
|
|
197
|
-
if (msg.scheduleSelfTest) {
|
|
198
|
-
selfTestTarget = { send: sendToContentScript, frameRef };
|
|
199
|
-
}
|
|
200
|
-
break;
|
|
201
|
-
case 'autoconsentDone': {
|
|
202
|
-
const target = selfTestTarget ?? { send: sendToContentScript, frameRef };
|
|
203
|
-
await target.send(target.frameRef, { type: 'selfTest' });
|
|
204
|
-
break;
|
|
205
|
-
}
|
|
206
|
-
case 'autoconsentError':
|
|
207
|
-
console.error('autoconsent error:', msg.details);
|
|
208
|
-
break;
|
|
209
|
-
}
|
|
210
|
-
};
|
|
211
|
-
};
|
|
212
|
-
|
|
213
|
-
// Isolated-world injection requires CDP, which Playwright exposes for Chromium only.
|
|
214
|
-
const browserName = page.context().browser()?.browserType().name();
|
|
215
|
-
if (browserName && browserName !== 'chromium') {
|
|
216
|
-
throw new Error(`regional-proxy supports Chromium only (got "${browserName}").`);
|
|
217
|
-
}
|
|
218
|
-
await injectIntoIsolatedWorld(page, createMessageHandler);
|
|
219
|
-
|
|
220
|
-
function hasMessage(/** @type {string} */ type) {
|
|
221
|
-
return received.some((m) => m.type === type);
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
async function waitForCompletion(timeout = 45000, detectionTimeout = timeout) {
|
|
225
|
-
const start = Date.now();
|
|
226
|
-
while (Date.now() - start < timeout) {
|
|
227
|
-
if (hasMessage('optOutResult') || hasMessage('optInResult')) {
|
|
228
|
-
return true;
|
|
229
|
-
}
|
|
230
|
-
// Detection-only mode (action: null): no action result will arrive.
|
|
231
|
-
if (!action && hasMessage('popupFound')) {
|
|
232
|
-
return true;
|
|
233
|
-
}
|
|
234
|
-
// Bail out early only if no CMP was detected within the detection window. This defaults
|
|
235
|
-
// to the full timeout, so slow regional pages that detect late are not abandoned (which
|
|
236
|
-
// would otherwise yield a false "No CMP detected" and end before opt-out can finish).
|
|
237
|
-
if (Date.now() - start > detectionTimeout && !hasMessage('cmpDetected')) {
|
|
238
|
-
return false;
|
|
239
|
-
}
|
|
240
|
-
await new Promise((r) => setTimeout(r, 500));
|
|
241
|
-
}
|
|
242
|
-
return false;
|
|
243
|
-
}
|
|
244
|
-
|
|
245
|
-
async function waitForMessage(/** @type {string} */ type, timeout = 30000) {
|
|
246
|
-
const start = Date.now();
|
|
247
|
-
while (Date.now() - start < timeout) {
|
|
248
|
-
if (hasMessage(type)) return true;
|
|
249
|
-
await new Promise((r) => setTimeout(r, 500));
|
|
250
|
-
}
|
|
251
|
-
return false;
|
|
252
|
-
}
|
|
253
|
-
|
|
254
|
-
function collectResult(/** @type {string} */ url, /** @type {string} */ region) {
|
|
255
|
-
/** @type {TestResult} */
|
|
256
|
-
const result = {
|
|
257
|
-
url,
|
|
258
|
-
region,
|
|
259
|
-
cmpsDetected: [],
|
|
260
|
-
cmpActedOn: null,
|
|
261
|
-
popupFound: false,
|
|
262
|
-
optOutResult: null,
|
|
263
|
-
optInResult: null,
|
|
264
|
-
selfTestResult: null,
|
|
265
|
-
autoconsentDone: false,
|
|
266
|
-
isCosmetic: null,
|
|
267
|
-
errors: [],
|
|
268
|
-
duration: 0,
|
|
269
|
-
screenshotPaths: [],
|
|
270
|
-
};
|
|
271
|
-
for (const msg of received) {
|
|
272
|
-
switch (msg.type) {
|
|
273
|
-
case 'cmpDetected':
|
|
274
|
-
result.cmpsDetected.push(msg.cmp);
|
|
275
|
-
break;
|
|
276
|
-
case 'popupFound':
|
|
277
|
-
result.popupFound = true;
|
|
278
|
-
break;
|
|
279
|
-
case 'optOutResult':
|
|
280
|
-
result.optOutResult = msg.result;
|
|
281
|
-
result.cmpActedOn = msg.cmp;
|
|
282
|
-
break;
|
|
283
|
-
case 'optInResult':
|
|
284
|
-
result.optInResult = msg.result;
|
|
285
|
-
result.cmpActedOn = msg.cmp;
|
|
286
|
-
break;
|
|
287
|
-
case 'selfTestResult':
|
|
288
|
-
result.selfTestResult = msg.result;
|
|
289
|
-
break;
|
|
290
|
-
case 'autoconsentDone':
|
|
291
|
-
result.autoconsentDone = true;
|
|
292
|
-
result.isCosmetic = msg.isCosmetic;
|
|
293
|
-
break;
|
|
294
|
-
case 'autoconsentError':
|
|
295
|
-
result.errors.push(msg.details?.msg || JSON.stringify(msg.details));
|
|
296
|
-
break;
|
|
297
|
-
}
|
|
298
|
-
}
|
|
299
|
-
return result;
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
return { received, hasMessage, waitForCompletion, waitForMessage, collectResult };
|
|
101
|
+
return injectThrough(page, options, 'regional-proxy');
|
|
303
102
|
}
|
|
304
103
|
|
|
305
|
-
/**
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
* @param {MessageHandlerFactory} createMessageHandler
|
|
312
|
-
*/
|
|
313
|
-
async function injectIntoIsolatedWorld(page, createMessageHandler) {
|
|
314
|
-
// Isolated world name: `<prefix><pageWorldUniqueId><separator><frameId>`. Encoding the page-world
|
|
315
|
-
// uniqueId lets us later run eval snippets in that frame's main world.
|
|
316
|
-
const WORLD_PREFIX = 'autoconsent_iw_';
|
|
317
|
-
const WORLD_SEPARATOR = '_frame_';
|
|
318
|
-
const BINDING_PREFIX = 'autoconsentSendMessage_';
|
|
319
|
-
|
|
320
|
-
/** @type {Map<string, string>} isolated-world uniqueId -> page (main) world uniqueId */
|
|
321
|
-
const isolated2pageWorld = new Map();
|
|
322
|
-
/** @type {Map<string, any>} isolated-world uniqueId -> CDP session that owns it */
|
|
323
|
-
const sessionByContext = new Map();
|
|
324
|
-
/** @type {Map<string, string>} binding name -> isolated-world uniqueId */
|
|
325
|
-
const contextByBinding = new Map();
|
|
326
|
-
|
|
327
|
-
const handle = createMessageHandler({
|
|
328
|
-
sendToContentScript: async (isolatedUniqueId, message) => {
|
|
329
|
-
const client = sessionByContext.get(isolatedUniqueId);
|
|
330
|
-
if (!client) return;
|
|
331
|
-
try {
|
|
332
|
-
await client.send('Runtime.evaluate', {
|
|
333
|
-
expression: `autoconsentReceiveMessage(${JSON.stringify(message)})`,
|
|
334
|
-
uniqueContextId: isolatedUniqueId,
|
|
335
|
-
awaitPromise: true,
|
|
336
|
-
// Some pages' CSP would otherwise block evaluating in the isolated world.
|
|
337
|
-
allowUnsafeEvalBlockedByCSP: true,
|
|
338
|
-
});
|
|
339
|
-
} catch {
|
|
340
|
-
// The context may be gone if the frame navigated or detached.
|
|
341
|
-
}
|
|
342
|
-
},
|
|
343
|
-
evalInMainWorld: async (isolatedUniqueId, code) => {
|
|
344
|
-
const client = sessionByContext.get(isolatedUniqueId);
|
|
345
|
-
const pageWorldUniqueId = isolated2pageWorld.get(isolatedUniqueId);
|
|
346
|
-
if (!client || !pageWorldUniqueId) return false;
|
|
347
|
-
const { result, exceptionDetails } = await client.send('Runtime.evaluate', {
|
|
348
|
-
expression: code,
|
|
349
|
-
uniqueContextId: pageWorldUniqueId,
|
|
350
|
-
returnByValue: true,
|
|
351
|
-
awaitPromise: true,
|
|
352
|
-
// Eval snippets must run even when the page's CSP disallows eval.
|
|
353
|
-
allowUnsafeEvalBlockedByCSP: true,
|
|
354
|
-
});
|
|
355
|
-
return exceptionDetails ? false : (result?.value ?? false);
|
|
356
|
-
},
|
|
357
|
-
});
|
|
358
|
-
|
|
359
|
-
async function attachToSession(/** @type {any} */ client) {
|
|
360
|
-
client.on('Runtime.executionContextCreated', async (/** @type {any} */ event) => {
|
|
361
|
-
const { context } = event;
|
|
362
|
-
const frameId = context.auxData?.frameId;
|
|
363
|
-
|
|
364
|
-
// Our isolated world finished initializing: wire up its binding and content script.
|
|
365
|
-
if (context.auxData?.type === 'isolated' && typeof context.name === 'string' && context.name.startsWith(WORLD_PREFIX)) {
|
|
366
|
-
const separatorIndex = context.name.indexOf(WORLD_SEPARATOR);
|
|
367
|
-
const pageWorldUniqueId = context.name.slice(WORLD_PREFIX.length, separatorIndex);
|
|
368
|
-
const intendedFrameId = context.name.slice(separatorIndex + WORLD_SEPARATOR.length);
|
|
369
|
-
// Chromium may create the named world in other frames too; keep only the one we asked for.
|
|
370
|
-
if (intendedFrameId !== frameId) return;
|
|
371
|
-
|
|
372
|
-
isolated2pageWorld.set(context.uniqueId, pageWorldUniqueId);
|
|
373
|
-
sessionByContext.set(context.uniqueId, client);
|
|
374
|
-
|
|
375
|
-
const bindingName = `${BINDING_PREFIX}${context.uniqueId.replace(/\W/g, '_')}`;
|
|
376
|
-
contextByBinding.set(bindingName, context.uniqueId);
|
|
377
|
-
try {
|
|
378
|
-
await client.send('Runtime.addBinding', { name: bindingName, executionContextName: context.name });
|
|
379
|
-
// CDP bindings take a single string arg, so wrap it in the shape the content script expects.
|
|
380
|
-
await client.send('Runtime.evaluate', {
|
|
381
|
-
expression: `window.autoconsentSendMessage = (m) => { window.${bindingName}(JSON.stringify(m)); return Promise.resolve(); };\n${contentScript}`,
|
|
382
|
-
uniqueContextId: context.uniqueId,
|
|
383
|
-
allowUnsafeEvalBlockedByCSP: true,
|
|
384
|
-
});
|
|
385
|
-
} catch {
|
|
386
|
-
// The frame may have navigated or detached before we finished wiring it up.
|
|
387
|
-
}
|
|
388
|
-
return;
|
|
389
|
-
}
|
|
390
|
-
|
|
391
|
-
// A regular page (main) world: request an isolated world for it, tagged with its id.
|
|
392
|
-
if (!frameId || context.auxData?.type !== 'default' || !context.origin || context.origin === '://') return;
|
|
393
|
-
try {
|
|
394
|
-
await client.send('Page.createIsolatedWorld', {
|
|
395
|
-
frameId,
|
|
396
|
-
worldName: `${WORLD_PREFIX}${context.uniqueId}${WORLD_SEPARATOR}${frameId}`,
|
|
397
|
-
});
|
|
398
|
-
} catch {
|
|
399
|
-
// The frame may have navigated or detached.
|
|
400
|
-
}
|
|
401
|
-
});
|
|
402
|
-
|
|
403
|
-
client.on('Runtime.executionContextDestroyed', (/** @type {any} */ event) => {
|
|
404
|
-
const uniqueId = event.executionContextUniqueId;
|
|
405
|
-
if (!uniqueId) return;
|
|
406
|
-
isolated2pageWorld.delete(uniqueId);
|
|
407
|
-
sessionByContext.delete(uniqueId);
|
|
408
|
-
});
|
|
409
|
-
|
|
410
|
-
client.on('Runtime.bindingCalled', (/** @type {any} */ event) => {
|
|
411
|
-
const isolatedUniqueId = contextByBinding.get(event.name);
|
|
412
|
-
if (!isolatedUniqueId) return;
|
|
413
|
-
let msg;
|
|
414
|
-
try {
|
|
415
|
-
msg = JSON.parse(event.payload);
|
|
416
|
-
} catch {
|
|
417
|
-
return;
|
|
418
|
-
}
|
|
419
|
-
handle(msg, isolatedUniqueId);
|
|
420
|
-
});
|
|
421
|
-
|
|
422
|
-
// Page must be enabled before createIsolatedWorld; Runtime.enable replays existing contexts.
|
|
423
|
-
await client.send('Page.enable');
|
|
424
|
-
await client.send('Runtime.enable');
|
|
425
|
-
}
|
|
426
|
-
|
|
427
|
-
// The page session covers the main frame and all same-process (in-process) iframes.
|
|
428
|
-
await attachToSession(await page.context().newCDPSession(page));
|
|
429
|
-
|
|
430
|
-
// Out-of-process iframes (OOPIFs) are separate CDP targets, so each needs its own session.
|
|
431
|
-
// newCDPSession throws for in-process frames, which are already handled above.
|
|
432
|
-
const attachedFrames = new WeakSet();
|
|
433
|
-
async function attachToOopif(/** @type {import('playwright').Frame} */ frame) {
|
|
434
|
-
if (!frame.parentFrame() || attachedFrames.has(frame)) return;
|
|
435
|
-
// Mark synchronously (before any await) so concurrent frameattached/framenavigated events
|
|
436
|
-
// can't open duplicate sessions for the same frame. Unmark on failure so a later event retries.
|
|
437
|
-
attachedFrames.add(frame);
|
|
438
|
-
let client;
|
|
439
|
-
try {
|
|
440
|
-
client = await page.context().newCDPSession(frame);
|
|
441
|
-
} catch {
|
|
442
|
-
// In-process frame (already covered by the page session) or the frame detached.
|
|
443
|
-
attachedFrames.delete(frame);
|
|
444
|
-
return;
|
|
445
|
-
}
|
|
446
|
-
try {
|
|
447
|
-
await attachToSession(client);
|
|
448
|
-
} catch {
|
|
449
|
-
// Wiring up the session failed (e.g. the OOPIF navigated/detached). Detach the
|
|
450
|
-
// half-initialized session before unmarking, otherwise a retry would open a second
|
|
451
|
-
// session for the same frame, leaving duplicate listeners and possible double injection.
|
|
452
|
-
try {
|
|
453
|
-
await client.detach();
|
|
454
|
-
} catch {
|
|
455
|
-
// The session may already be gone.
|
|
456
|
-
}
|
|
457
|
-
attachedFrames.delete(frame);
|
|
458
|
-
}
|
|
459
|
-
}
|
|
460
|
-
page.on('frameattached', attachToOopif);
|
|
461
|
-
page.on('framenavigated', attachToOopif);
|
|
462
|
-
await Promise.all(page.frames().map(attachToOopif));
|
|
463
|
-
}
|
|
104
|
+
/** @type {Provider} */
|
|
105
|
+
const provider = {
|
|
106
|
+
name: 'regional-proxy',
|
|
107
|
+
defaultScreenshotsDir: path.join(projectRoot, 'test-results/regional-proxy'),
|
|
108
|
+
inject: injectAutoconsent,
|
|
109
|
+
};
|
|
464
110
|
|
|
465
111
|
/**
|
|
466
112
|
* Test a URL using an already-created Playwright page.
|
|
@@ -471,59 +117,7 @@ async function injectIntoIsolatedWorld(page, createMessageHandler) {
|
|
|
471
117
|
* @returns {Promise<TestResult>}
|
|
472
118
|
*/
|
|
473
119
|
export async function testPage(page, url, regionKey, options = {}) {
|
|
474
|
-
|
|
475
|
-
const completionTimeout = options.completionTimeout ?? 45000;
|
|
476
|
-
const detectionTimeout = options.detectionTimeout ?? completionTimeout;
|
|
477
|
-
const screenshotsDir = options.screenshotsDir ?? path.join(projectRoot, 'test-results/regional-proxy');
|
|
478
|
-
const startTime = Date.now();
|
|
479
|
-
|
|
480
|
-
try {
|
|
481
|
-
const ctx = await injectAutoconsent(page, options);
|
|
482
|
-
await page.goto(url, { waitUntil: 'commit', timeout: navTimeout });
|
|
483
|
-
|
|
484
|
-
const completed = await ctx.waitForCompletion(completionTimeout, detectionTimeout);
|
|
485
|
-
if (completed && !ctx.hasMessage('selfTestResult')) {
|
|
486
|
-
await ctx.waitForMessage('selfTestResult', 10000);
|
|
487
|
-
}
|
|
488
|
-
if (completed) {
|
|
489
|
-
await page.waitForTimeout(1500);
|
|
490
|
-
}
|
|
491
|
-
|
|
492
|
-
const result = ctx.collectResult(url, regionKey);
|
|
493
|
-
result.duration = Date.now() - startTime;
|
|
494
|
-
|
|
495
|
-
if (!completed && !ctx.hasMessage('cmpDetected')) {
|
|
496
|
-
result.errors.push('No CMP detected (site may not show a cookie banner in this region)');
|
|
497
|
-
} else if (!completed) {
|
|
498
|
-
result.errors.push('Timed out waiting for autoconsent to complete');
|
|
499
|
-
}
|
|
500
|
-
|
|
501
|
-
try {
|
|
502
|
-
const domain = new URL(url).hostname;
|
|
503
|
-
const filepath = path.join(screenshotsDir, `${domain}-${regionKey}-final.jpg`);
|
|
504
|
-
fs.mkdirSync(screenshotsDir, { recursive: true });
|
|
505
|
-
await page.screenshot({ path: filepath, quality: 50, scale: 'css', timeout: 5000, type: 'jpeg' });
|
|
506
|
-
result.screenshotPaths.push(filepath);
|
|
507
|
-
} catch {}
|
|
508
|
-
|
|
509
|
-
return result;
|
|
510
|
-
} catch (e) {
|
|
511
|
-
return {
|
|
512
|
-
url,
|
|
513
|
-
region: regionKey,
|
|
514
|
-
cmpsDetected: [],
|
|
515
|
-
cmpActedOn: null,
|
|
516
|
-
popupFound: false,
|
|
517
|
-
optOutResult: null,
|
|
518
|
-
optInResult: null,
|
|
519
|
-
selfTestResult: null,
|
|
520
|
-
autoconsentDone: false,
|
|
521
|
-
isCosmetic: null,
|
|
522
|
-
errors: [e instanceof Error ? e.message : String(e)],
|
|
523
|
-
duration: Date.now() - startTime,
|
|
524
|
-
screenshotPaths: [],
|
|
525
|
-
};
|
|
526
|
-
}
|
|
120
|
+
return runTest(page, url, regionKey, options, provider);
|
|
527
121
|
}
|
|
528
122
|
|
|
529
123
|
/**
|
|
@@ -542,21 +136,7 @@ export async function testUrl(url, regionKey, options = {}) {
|
|
|
542
136
|
const page = await browser.newPage();
|
|
543
137
|
return await testPage(page, url, regionKey, options);
|
|
544
138
|
} catch (e) {
|
|
545
|
-
return
|
|
546
|
-
url,
|
|
547
|
-
region: regionKey,
|
|
548
|
-
cmpsDetected: [],
|
|
549
|
-
cmpActedOn: null,
|
|
550
|
-
popupFound: false,
|
|
551
|
-
optOutResult: null,
|
|
552
|
-
optInResult: null,
|
|
553
|
-
selfTestResult: null,
|
|
554
|
-
autoconsentDone: false,
|
|
555
|
-
isCosmetic: null,
|
|
556
|
-
errors: [e instanceof Error ? e.message : String(e)],
|
|
557
|
-
duration: 0,
|
|
558
|
-
screenshotPaths: [],
|
|
559
|
-
};
|
|
139
|
+
return emptyResult(url, regionKey, 'regional-proxy', e);
|
|
560
140
|
} finally {
|
|
561
141
|
try {
|
|
562
142
|
await browser?.close();
|
|
@@ -578,32 +158,3 @@ export async function testRegions(url, regions = CORE_REGIONS, options = {}) {
|
|
|
578
158
|
}
|
|
579
159
|
return results;
|
|
580
160
|
}
|
|
581
|
-
|
|
582
|
-
/**
|
|
583
|
-
* Format a TestResult as a human-readable line.
|
|
584
|
-
* @param {TestResult} result
|
|
585
|
-
* @returns {string}
|
|
586
|
-
*/
|
|
587
|
-
export function formatResult(result) {
|
|
588
|
-
const actionResult = result.optOutResult ?? result.optInResult;
|
|
589
|
-
const actionAttempted = result.optOutResult !== null || result.optInResult !== null;
|
|
590
|
-
let status;
|
|
591
|
-
if (actionAttempted && actionResult) status = 'PASS';
|
|
592
|
-
else if (actionAttempted && !actionResult) status = 'ACTION FAILED';
|
|
593
|
-
else if (result.cmpsDetected.length > 0) status = 'PARTIAL';
|
|
594
|
-
else status = 'NO CMP';
|
|
595
|
-
|
|
596
|
-
const parts = [
|
|
597
|
-
`${status} [${result.region}] ${result.url}`,
|
|
598
|
-
` CMP: ${result.cmpActedOn || 'none'}${result.cmpsDetected.length > 1 ? ` (also detected: ${result.cmpsDetected.filter((c) => c !== result.cmpActedOn).join(', ')})` : ''}`,
|
|
599
|
-
` Popup: ${result.popupFound} | OptOut: ${result.optOutResult} | OptIn: ${result.optInResult} | SelfTest: ${result.selfTestResult}`,
|
|
600
|
-
` Done: ${result.autoconsentDone}${result.isCosmetic ? ' (cosmetic)' : ''} | ${result.duration}ms`,
|
|
601
|
-
];
|
|
602
|
-
if (result.errors.length > 0) {
|
|
603
|
-
parts.push(` Errors: ${result.errors.join('; ')}`);
|
|
604
|
-
}
|
|
605
|
-
if (result.screenshotPaths.length > 0) {
|
|
606
|
-
parts.push(` Screenshots: ${result.screenshotPaths.join(', ')}`);
|
|
607
|
-
}
|
|
608
|
-
return parts.join('\n');
|
|
609
|
-
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: oxylabs-testing
|
|
3
|
+
description: Test autoconsent rules across geographic regions using Oxylabs Agent Browser remote browsers. Use when a site shows our regional proxies a bot wall (captcha, challenge page, access denied), when a region has no regional proxy (e.g. pt, br), or for US state/city targeting. Default to proxy-testing otherwise.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
@.agents/skills/oxylabs-testing/SKILL.md
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: proxy-testing
|
|
3
|
-
description: Test autoconsent
|
|
3
|
+
description: Test autoconsent via regional proxies in Playwright. Use when verifying rule changes across regions (US, EU, UK, etc.), or investigating region-dependent CMP behavior.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
@.agents/skills/proxy-testing/SKILL.md
|
|
@@ -17,6 +17,8 @@
|
|
|
17
17
|
"REGIONAL_PROXY_US": "socks5://us-proxy.example.com:1080",
|
|
18
18
|
"REGIONAL_PROXY_USERNAME": "OPTIONAL_PROXY_USERNAME",
|
|
19
19
|
"REGIONAL_PROXY_PASSWORD": "OPTIONAL_PROXY_PASSWORD",
|
|
20
|
+
"OXYLABS_USER": "OPTIONAL_OXYLABS_USER",
|
|
21
|
+
"OXYLABS_PASSWORD": "OPTIONAL_OXYLABS_PASSWORD",
|
|
20
22
|
"PUBLICWWW_KEY": "REPLACE_WITH_YOUR_KEY"
|
|
21
23
|
}
|
|
22
24
|
}
|
package/AGENTS.md
CHANGED
|
@@ -66,7 +66,7 @@ CMPs behave differently by region:
|
|
|
66
66
|
|
|
67
67
|
Use `if`/`then`/`else` to handle regional variants within a single rule.
|
|
68
68
|
|
|
69
|
-
**Test all rule changes across the core region set** — US, GB, DE, plus the reported region from the task (or the closest supported proxy region, e.g. ES for PT) — using the `proxy-testing` skill. Escalate to the expanded set according to the skill's policy.
|
|
69
|
+
**Test all rule changes across the core region set** — US, GB, DE, plus the reported region from the task (or the closest supported proxy region, e.g. ES for PT) — using the `proxy-testing` skill. Escalate to the expanded set according to the skill's policy. If a site shows our proxies a bot wall, retest those regions with the `oxylabs-testing` skill.
|
|
70
70
|
|
|
71
71
|
### Generic vs Site-Specific Rules
|
|
72
72
|
|
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,32 @@
|
|
|
1
|
+
# v16.44.0 (Sat Oct 03 2026)
|
|
2
|
+
|
|
3
|
+
#### Rules
|
|
4
|
+
|
|
5
|
+
- Rule update: lacoccinelle [#1592](https://github.com/duckduckgo/autoconsent/pull/1592) ([@daxtheduck](https://github.com/daxtheduck))
|
|
6
|
+
- Rule update: elsevier [#1591](https://github.com/duckduckgo/autoconsent/pull/1591) ([@daxtheduck](https://github.com/daxtheduck))
|
|
7
|
+
- Rule update: Cpm fail nytimes.com [#1590](https://github.com/duckduckgo/autoconsent/pull/1590) ([@daxtheduck](https://github.com/daxtheduck))
|
|
8
|
+
- Rule update: chatgpt [#1589](https://github.com/duckduckgo/autoconsent/pull/1589) ([@daxtheduck](https://github.com/daxtheduck))
|
|
9
|
+
- Rule update: bandcamp.com [#1585](https://github.com/duckduckgo/autoconsent/pull/1585) ([@daxtheduck](https://github.com/daxtheduck))
|
|
10
|
+
- Rule update: woo-commerce-com [#1584](https://github.com/duckduckgo/autoconsent/pull/1584) ([@muodov](https://github.com/muodov))
|
|
11
|
+
|
|
12
|
+
#### Dependencies
|
|
13
|
+
|
|
14
|
+
- Bump tldts-experimental from 7.4.6 to 7.4.16 [#1580](https://github.com/duckduckgo/autoconsent/pull/1580) ([@dependabot[bot]](https://github.com/dependabot[bot]) [@muodov](https://github.com/muodov))
|
|
15
|
+
- Upgrade Playwright to 1.63.0 [#1588](https://github.com/duckduckgo/autoconsent/pull/1588) ([@muodov](https://github.com/muodov))
|
|
16
|
+
|
|
17
|
+
#### AI / Agent Workflow
|
|
18
|
+
|
|
19
|
+
- Fix Oxylabs captcha events on Agent Browser [#1595](https://github.com/duckduckgo/autoconsent/pull/1595) ([@muodov](https://github.com/muodov))
|
|
20
|
+
- Add Headless Browser support [#1593](https://github.com/duckduckgo/autoconsent/pull/1593) ([@muodov](https://github.com/muodov))
|
|
21
|
+
|
|
22
|
+
#### Authors: 3
|
|
23
|
+
|
|
24
|
+
- [@dependabot[bot]](https://github.com/dependabot[bot])
|
|
25
|
+
- Dax ([@daxtheduck](https://github.com/daxtheduck))
|
|
26
|
+
- Maxim Tsoy ([@muodov](https://github.com/muodov))
|
|
27
|
+
|
|
28
|
+
---
|
|
29
|
+
|
|
1
30
|
# v16.43.2 (Thu Oct 01 2026)
|
|
2
31
|
|
|
3
32
|
#### AI / Agent Workflow
|