@duckduckgo/autoconsent 14.2.0 → 14.4.0

package/lib/cmps/all.ts

@@ -9,7 +9,6 @@ import Klaro from './klaro';
 import Uniconsent from './uniconsent';
 import Conversant from './conversant';
 import Tiktok from './tiktok';
-import Airbnb from './airbnb';
 import Tumblr from './tumblr-com';
 import Admiral from './admiral';
 
@@ -25,7 +24,6 @@ export const dynamicCMPs = [
     Uniconsent,
     Conversant,
     Tiktok,
-    Airbnb,
     Tumblr,
     Admiral,
 ];

package/lib/encoding.ts

@@ -137,11 +137,29 @@ export function decodeRules(encoded: CompactCMPRuleset): AutoConsentCMPRule[] {
     if (encoded.v > 1) {
         throw new Error('Unsupported rule format.');
     }
-    function decodeRuleStep(step: CompactCMPRuleStep): AutoConsentRuleStep {
+    return encoded.r.filter((r) => r[0] <= SUPPORTED_RULE_STEP_VERSION).map((rule) => new CompactedCMPRule(rule, encoded.s));
+}
+
+class CompactedCMPRule implements AutoConsentCMPRule {
+    r: CompactCMPRule;
+    s: string[];
+    intermediate = false;
+    optIn = [];
+
+    constructor(rule: CompactCMPRule, strings: string[]) {
+        this.r = rule;
+        this.s = strings;
+        if (this.r[10] && this.r[10].intermediate) {
+            this.intermediate = this.r[10].intermediate;
+        }
+    }
+
+    _decodeRuleStep(step: CompactCMPRuleStep): AutoConsentRuleStep {
         const clonedStep: CompactCMPRuleStep = { ...step };
+        const decodeRuleStep = this._decodeRuleStep.bind(this);
         for (const [longKey, shortKey] of compactedRuleSteps) {
             if (clonedStep[shortKey] !== undefined) {
-                clonedStep[longKey] = encoded.s[clonedStep[shortKey]];
+                clonedStep[longKey] = this.s[clonedStep[shortKey]];
                 delete clonedStep[shortKey];
             }
         }
@@ -157,48 +175,50 @@ export function decodeRules(encoded: CompactCMPRuleset): AutoConsentCMPRule[] {
         }
         return { ...clonedStep };
     }
-    return encoded.r
-        .filter((r) => r[0] <= SUPPORTED_RULE_STEP_VERSION)
-        .map((rule) => {
-            const [
-                minimumRuleStepVersion,
-                name,
-                cosmetic,
-                urlPattern,
-                mainFrame,
-                prehideSelectors,
-                detectCmp,
-                detectPopup,
-                optOut,
-                test,
-                extra,
-            ] = rule;
-            const optIn: AutoConsentRuleStep[] = [];
-            const runContext: RunContext = {};
-            const runInMainFrame = decodeNullableBoolean((Math.floor(mainFrame / 10) % 10) as CompactNullableBoolean);
-            const runInSubFrame = decodeNullableBoolean((mainFrame % 10) as CompactNullableBoolean);
-            if (runInMainFrame !== undefined) {
-                runContext.main = runInMainFrame;
-            }
-            if (runInSubFrame !== undefined) {
-                runContext.frame = runInSubFrame;
-            }
-            if (urlPattern !== '') {
-                runContext.urlPattern = urlPattern;
-            }
 
-            return {
-                name,
-                cosmetic: decodeNullableBoolean(cosmetic),
-                runContext,
-                prehideSelectors: prehideSelectors.map((i) => encoded.s[i].toString()),
-                detectCmp: detectCmp.map(decodeRuleStep),
-                detectPopup: detectPopup.map(decodeRuleStep),
-                optOut: optOut.map(decodeRuleStep),
-                optIn,
-                test: test?.map(decodeRuleStep),
-                minimumRuleStepVersion: minimumRuleStepVersion || undefined,
-                ...extra,
-            };
-        });
+    get name() {
+        return this.r[1];
+    }
+
+    get cosmetic() {
+        return decodeNullableBoolean(this.r[2]);
+    }
+
+    get runContext() {
+        const runContext: RunContext = {};
+        const urlPattern = this.r[3];
+        const mainFrame = this.r[4];
+        const runInMainFrame = decodeNullableBoolean((Math.floor(mainFrame / 10) % 10) as CompactNullableBoolean);
+        const runInSubFrame = decodeNullableBoolean((mainFrame % 10) as CompactNullableBoolean);
+        if (runInMainFrame !== undefined) {
+            runContext.main = runInMainFrame;
+        }
+        if (runInSubFrame !== undefined) {
+            runContext.frame = runInSubFrame;
+        }
+        if (urlPattern !== '') {
+            runContext.urlPattern = urlPattern;
+        }
+        return runContext;
+    }
+
+    get prehideSelectors() {
+        return this.r[5].map((i) => this.s[i].toString());
+    }
+
+    get detectCmp() {
+        return this.r[6].map(this._decodeRuleStep.bind(this));
+    }
+
+    get detectPopup() {
+        return this.r[7].map(this._decodeRuleStep.bind(this));
+    }
+
+    get optOut() {
+        return this.r[8].map(this._decodeRuleStep.bind(this));
+    }
+
+    get test() {
+        return this.r[9].map(this._decodeRuleStep.bind(this));
+    }
 }

package/lib/rules.ts

@@ -4,20 +4,20 @@ import { snippets } from './eval-snippets';
 export const SUPPORTED_RULE_STEP_VERSION = 1;
 
 export type AutoConsentCMPRule = {
-    name: string;
-    vendorUrl?: string;
-    prehideSelectors?: string[];
-    runContext?: RunContext;
-    intermediate?: boolean;
-    cosmetic?: boolean;
-    detectCmp: AutoConsentRuleStep[];
-    detectPopup: AutoConsentRuleStep[];
-    optOut: AutoConsentRuleStep[];
-    optIn: AutoConsentRuleStep[];
-    openCmp?: AutoConsentRuleStep[];
-    test?: AutoConsentRuleStep[];
-    comment?: string;
-    minimumRuleStepVersion?: number;
+    readonly name: string;
+    readonly vendorUrl?: string;
+    readonly prehideSelectors?: string[];
+    readonly runContext?: RunContext;
+    readonly intermediate?: boolean;
+    readonly cosmetic?: boolean;
+    readonly detectCmp: AutoConsentRuleStep[];
+    readonly detectPopup: AutoConsentRuleStep[];
+    readonly optOut: AutoConsentRuleStep[];
+    readonly optIn: AutoConsentRuleStep[];
+    readonly openCmp?: AutoConsentRuleStep[];
+    readonly test?: AutoConsentRuleStep[];
+    readonly comment?: string;
+    readonly minimumRuleStepVersion?: number;
 };
 
 export type RunContext = {

package/lib/types.ts

@@ -54,6 +54,7 @@ export type Config = {
     disabledCmps: string[];
     enablePrehide: boolean;
     enableCosmeticRules: boolean;
+    enableGeneratedRules: boolean;
     detectRetries: number;
     isMainWorld: boolean;
     prehideTimeout: number;

package/lib/utils.ts

@@ -75,6 +75,7 @@ export function normalizeConfig(providedConfig: any): Config {
         disabledCmps: [],
         enablePrehide: true,
         enableCosmeticRules: true,
+        enableGeneratedRules: true,
         enableHeuristicDetection: false,
         detectRetries: 20,
         isMainWorld: false,

package/lib/web.ts

@@ -20,7 +20,8 @@ function filterCMPs(rules: AutoCMP[], config: Config) {
     return rules.filter((cmp) => {
         return (
             (!config.disabledCmps || !config.disabledCmps.includes(cmp.name)) && // CMP is not disabled
-            (config.enableCosmeticRules || !cmp.isCosmetic) // CMP is not cosmetic or cosmetic rules are enabled
+            (config.enableCosmeticRules || !cmp.isCosmetic) && // CMP is not cosmetic or cosmetic rules are enabled
+            (config.enableGeneratedRules || !cmp.name.startsWith('auto_')) // CMP is not generated or generated rules are enabled
         );
     });
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@duckduckgo/autoconsent",
-  "version": "14.2.0",
+  "version": "14.4.0",
   "description": "",
   "exports": {
     ".": {
@@ -21,9 +21,10 @@
   "scripts": {
     "clean": "rm -r dist",
     "lint": "eslint . && prettier . --check && npm run rule-syntax-check",
+    "lint-fix": "eslint . --fix && prettier . --write && npm run rule-syntax-check",
     "bundle": "./build.sh",
     "rule-syntax-check": "node scripts/validate-json-rules.js",
-    "watch": "npm run prepublish && chokidar \"lib\" \"addon\" \"rules/autoconsent\" \"rules/filterlist.txt\" --ignore 'lib/filterlist-engine.ts' -c \"npm run prepublish\"",
+    "watch": "npm run prepublish && chokidar \"lib\" \"addon\" \"rules/autoconsent\" \"rules/generated\" \"rules/filterlist.txt\" --ignore 'lib/filterlist-engine.ts' -c \"npm run prepublish\"",
     "create-rule": "node rules/create-rule.mjs",
     "test": "playwright test",
     "test:webkit": "playwright test --project webkit",

package/playwright/content.ts

@@ -1,7 +1,8 @@
 import AutoConsent from '../lib/web';
 import { BackgroundMessage } from '../lib/messages';
 import { MessageSender, RuleBundle } from '../lib/types';
-import * as rules from '../rules/rules.json';
+import { autoconsent } from '../rules/rules.json';
+import { consentomatic } from '../rules/consentomatic.json';
 
 declare global {
     interface Window {
@@ -11,7 +12,7 @@ declare global {
 }
 
 if (!window.autoconsentReceiveMessage) {
-    const consent = new AutoConsent(window.autoconsentSendMessage, null, <RuleBundle>rules);
+    const consent = new AutoConsent(window.autoconsentSendMessage, null, <RuleBundle>{ autoconsent, consentomatic });
 
     window.autoconsentReceiveMessage = (message: BackgroundMessage) => {
         return Promise.resolve(consent.receiveMessageCallback(message));

package/playwright/runner.ts

@@ -1,3 +1,4 @@
+import crypto from 'crypto';
 import fs from 'fs';
 import path from 'path';
 import { test, expect, Page, Frame } from '@playwright/test';
@@ -43,9 +44,14 @@ export async function injectContentScript(page: Page | Frame) {
     }
 }
 
+const screenshotsDir = path.join(__dirname, '../test-results/screenshots');
+
 export function generateTest(url: string, expectedCmp: string, options: TestOptions = defaultOptions) {
+    const domain = new URL(url).hostname;
+    const urlHash = crypto.createHash('md5').update(url).digest('hex').slice(0, 4);
     function genTest(autoAction: AutoAction) {
-        test(`${url.split('://')[1]} .${testRegion} ${autoAction} ${options.mobile ? 'mobile' : ''}`, async ({ page }, { project }) => {
+        test(`${domain} ${urlHash} .${testRegion} ${autoAction} ${options.mobile ? 'mobile' : ''}`, async ({ page }, { project }) => {
+            let screenshotCounter = 0;
             if (options.onlyRegions && options.onlyRegions.length > 0 && !options.onlyRegions.includes(testRegion)) {
                 test.skip();
             }
@@ -73,6 +79,20 @@ export function generateTest(url: string, expectedCmp: string, options: TestOpti
                 });
             }
 
+            async function takeScreenshot(name: string) {
+                try {
+                    await page.screenshot({
+                        path: path.join(screenshotsDir, `${expectedCmp}-${autoAction}-${domain}-${urlHash}-${name}.jpg`),
+                        quality: 50,
+                        scale: 'css',
+                        timeout: 2000,
+                        type: 'jpeg',
+                    });
+                } catch (e) {
+                    console.error(`Failed to take screenshot ${name}`, e);
+                }
+            }
+
             let selfTestFrame: Frame | null = null;
             async function messageCallback({ frame }: { frame: Frame }, msg: ContentScriptMessage) {
                 LOG_MESSAGES.includes(msg.type) && console.log(msg);
@@ -84,21 +104,32 @@ export function generateTest(url: string, expectedCmp: string, options: TestOpti
                                 enabled: true,
                                 autoAction,
                                 disabledCmps: [],
-                                enablePrehide: true,
+                                enablePrehide: false,
                                 detectRetries: 20,
                                 enableCosmeticRules: true,
+                                visualTest: true,
                             })} })`,
                         );
                         break;
                     }
+                    case 'cmpDetected': {
+                        await takeScreenshot(`${screenshotCounter++}-cmpDetected`);
+                        break;
+                    }
+                    case 'popupFound': {
+                        await takeScreenshot(`${screenshotCounter++}-popupFound`);
+                        break;
+                    }
                     case 'optInResult':
                     case 'optOutResult': {
+                        await takeScreenshot(`${screenshotCounter++}-result`);
                         if (msg.scheduleSelfTest) {
                             selfTestFrame = frame;
                         }
                         break;
                     }
                     case 'autoconsentDone': {
+                        await takeScreenshot(`${screenshotCounter++}-done`);
                         if (selfTestFrame && options.testSelfTest) {
                             await selfTestFrame.evaluate(`autoconsentReceiveMessage({ type: "selfTest" })`);
                         }
@@ -111,6 +142,10 @@ export function generateTest(url: string, expectedCmp: string, options: TestOpti
                         );
                         break;
                     }
+                    case 'visualDelay': {
+                        await takeScreenshot(`${screenshotCounter++}`);
+                        break;
+                    }
                     case 'autoconsentError': {
                         console.error(url, msg.details);
                         break;
@@ -123,31 +158,40 @@ export function generateTest(url: string, expectedCmp: string, options: TestOpti
             page.frames().forEach(injectContentScript);
             page.on('framenavigated', injectContentScript);
 
-            // wait for all messages and assertions
-            await waitFor(() => isMessageReceived({ type: 'cmpDetected', cmp: expectedCmp }), 50, 500);
-            expect(isMessageReceived({ type: 'cmpDetected', cmp: expectedCmp })).toBe(true);
+            try {
+                // wait for all messages and assertions
+                await waitFor(() => isMessageReceived({ type: 'cmpDetected', cmp: expectedCmp }), 50, 500);
+                expect(isMessageReceived({ type: 'cmpDetected', cmp: expectedCmp })).toBe(true);
 
-            await waitFor(() => isMessageReceived({ type: 'popupFound', cmp: expectedCmp }), options.expectPopupOpen ? 50 : 5, 500);
-            expect(isMessageReceived({ type: 'popupFound', cmp: expectedCmp })).toBe(options.expectPopupOpen);
+                await waitFor(() => isMessageReceived({ type: 'popupFound', cmp: expectedCmp }), options.expectPopupOpen ? 50 : 5, 500);
+                expect(isMessageReceived({ type: 'popupFound', cmp: expectedCmp })).toBe(options.expectPopupOpen);
 
-            if (options.expectPopupOpen) {
-                if (autoAction === 'optOut') {
-                    await waitFor(() => isMessageReceived({ type: 'optOutResult', result: true }), 50, 300);
-                    expect(isMessageReceived({ type: 'optOutResult', result: true })).toBe(true);
-                }
-                if (autoAction === 'optIn') {
-                    await waitFor(() => isMessageReceived({ type: 'optInResult', result: true }), 50, 300);
-                    expect(isMessageReceived({ type: 'optInResult', result: true })).toBe(true);
-                }
-                if (options.testSelfTest && selfTestFrame) {
-                    await waitFor(() => isMessageReceived({ type: 'selfTestResult', result: true }), 50, 300);
-                    expect(isMessageReceived({ type: 'selfTestResult', result: true })).toBe(true);
+                if (options.expectPopupOpen) {
+                    if (autoAction === 'optOut') {
+                        await waitFor(() => isMessageReceived({ type: 'optOutResult', result: true }), 50, 300);
+                        expect(isMessageReceived({ type: 'optOutResult', result: true })).toBe(true);
+                    }
+                    if (autoAction === 'optIn') {
+                        await waitFor(() => isMessageReceived({ type: 'optInResult', result: true }), 50, 300);
+                        expect(isMessageReceived({ type: 'optInResult', result: true })).toBe(true);
+                    }
+                    if (options.testSelfTest && selfTestFrame) {
+                        await waitFor(() => isMessageReceived({ type: 'selfTestResult', result: true }), 50, 300);
+                        expect(isMessageReceived({ type: 'selfTestResult', result: true })).toBe(true);
+                    }
+                    await waitFor(() => isMessageReceived({ type: 'autoconsentDone' }), 10, 500);
+                    expect(isMessageReceived({ type: 'autoconsentDone' })).toBe(true);
                 }
-                await waitFor(() => isMessageReceived({ type: 'autoconsentDone' }), 10, 500);
-                expect(isMessageReceived({ type: 'autoconsentDone' })).toBe(true);
-            }
 
-            expect(isMessageReceived({ type: 'autoconsentError' })).toBe(false);
+                received.forEach((msg) => {
+                    if (msg.type === 'autoconsentError') {
+                        expect(msg.details.msg, 'only "multiple CMPs" errors are allowed').toContain('Found multiple CMPs');
+                    }
+                });
+            } catch (e) {
+                await takeScreenshot(`${screenshotCounter++}-failure`);
+                throw e;
+            }
         });
     }
 

package/rules/autoconsent/cookieyes.json

@@ -13,11 +13,17 @@
                     "if": { "exists": ".cky-consent-container [data-cky-tag=settings-button]" },
                     "then": [
                         { "click": ".cky-consent-container [data-cky-tag=settings-button]" },
-                        { "waitFor": ".cky-modal-open input[type=checkbox]" },
-                        { "click": ".cky-modal-open input[type=checkbox]:checked", "all": true, "optional": true },
-                        { "waitForThenClick": ".cky-modal [data-cky-tag=detail-save-button]" }
+                        { "waitFor": ".cky-modal-open input[type=checkbox],.cky-consent-bar-expand input[type=checkbox]" },
+                        {
+                            "click": ".cky-modal-open input[type=checkbox]:checked,.cky-consent-bar-expand input[type=checkbox]:checked",
+                            "all": true,
+                            "optional": true
+                        },
+                        {
+                            "waitForThenClick": ".cky-modal [data-cky-tag=detail-save-button],.cky-consent-bar-expand [data-cky-tag=detail-save-button]"
+                        }
                     ],
-                    "else": [{ "hide": ".cky-consent-container,.cky-overlay" }]
+                    "else": [{ "hide": ".cky-consent-container,.cky-overlay,.cky-consent-bar-expand" }]
                 }
             ]
         }

package/rules/autoconsent/ct-ultimate-gdpr.json

@@ -20,8 +20,8 @@
     ],
     "optOut": [
         {
-            "if": { "visible": "#ct_ultimate-gdpr-cookie-reject a" },
-            "then": [{ "click": "#ct_ultimate-gdpr-cookie-reject a" }],
+            "if": { "visible": "#ct_ultimate-gdpr-cookie-reject a,#ct-ultimate-gdpr-cookie-reject" },
+            "then": [{ "click": "#ct_ultimate-gdpr-cookie-reject a,#ct-ultimate-gdpr-cookie-reject" }],
             "else": [
                 {
                     "if": { "visible": "#ct-ultimate-gdpr-cookie-change-settings" },

package/rules/build.ts

@@ -9,14 +9,24 @@ export const rulesDir = __dirname;
 
 async function readFileJSON(filePath: string) {
     const data = await fs.promises.readFile(filePath, 'utf-8');
-    return JSON.parse(data);
+    try {
+        return JSON.parse(data);
+    } catch (e) {
+        console.error(`Error parsing ${filePath}:`, e);
+        throw e;
+    }
 }
 
 export async function buildAutoconsentRules(): Promise<AutoConsentCMPRule[]> {
     // merge rules from ./autoconsent into rules.autoconsent array
     const autoconsentDir = path.join(rulesDir, 'autoconsent');
     const files = fs.readdirSync(autoconsentDir);
-    return await Promise.all(files.map((file) => readFileJSON(path.join(autoconsentDir, file))));
+    const generatedRulesDir = path.join(rulesDir, 'generated');
+    const generatedFiles = fs.readdirSync(generatedRulesDir);
+    const normalRules = await Promise.all(files.map((file) => readFileJSON(path.join(autoconsentDir, file))));
+    const generatedRules = await Promise.all(generatedFiles.map((file) => readFileJSON(path.join(generatedRulesDir, file))));
+
+    return [...normalRules, ...generatedRules];
 }
 
 export async function buildConsentOMaticRules() {