@duckduckgo/autoconsent 16.62.0 → 17.0.0

package/tests-wtr/heuristics/get-potential-popups.test.ts

@@ -1,5 +1,6 @@
 import { expect } from '@esm-bundle/chai';
 import { getActionablePopups, getButtonData, getFramePopupElement, getPotentialPopups } from '../../lib/heuristics';
+import { PopupData } from '../../lib/types';
 import { getHidingStyle } from '../../lib/utils';
 
 const BANNER = `
@@ -44,14 +45,18 @@ describe('getPotentialPopups', () => {
         return attachShadow(container.querySelector('#host')!, html);
     }
 
-    function popupIds(): string[] {
-        return getPotentialPopups(1000).map((p) => p.element.id);
+    function popupIds(hitTest = false): string[] {
+        return getPotentialPopups(1000, hitTest).map((p) => p.element.id);
     }
 
     function findPopup(id: string) {
         return getPotentialPopups(1000).find((p) => p.element.id === id);
     }
 
+    function inContainer(popups: PopupData[]): PopupData[] {
+        return popups.filter((p) => container.contains(p.element));
+    }
+
     it('returns positioned popups with their buttons', () => {
         container.innerHTML = fixedBanner('popup');
         const popup = findPopup('popup');
@@ -240,7 +245,7 @@ describe('getPotentialPopups', () => {
                     <div style="position: fixed; bottom: 0;"><button>Reject all</button><button>Accept all</button></div>
                 </div>
             `;
-            const popups = getPotentialPopups(1000).filter((p) => container.contains(p.element));
+            const popups = inContainer(getPotentialPopups(1000));
             expect(popups.map((p) => p.element.id)).to.deep.equal(['banner']);
         });
 
@@ -298,10 +303,236 @@ describe('getPotentialPopups', () => {
                     <div style="position: sticky; bottom: 0;"><button>Accept all</button></div>
                 </div>
             `;
-            const popups = getActionablePopups('tier2', 1000).filter((p) => container.contains(p.element));
+            const popups = inContainer(getActionablePopups('tier2', 1000));
             expect(popups).to.have.length(0);
         });
     });
+    describe('banners in the normal page flow', () => {
+        const IN_FLOW_BANNER = `
+            <div id="banner">
+                <p>We use cookies to improve your experience. Read our <a href="#">Privacy Policy</a>.</p>
+                <button>Reject all</button>
+                <button>Accept all</button>
+            </div>
+        `;
+
+        it('only hit-tests when asked to', () => {
+            container.innerHTML = IN_FLOW_BANNER;
+            expect(popupIds()).not.to.include('banner');
+            expect(inContainer(getActionablePopups('reject', 1000))).to.have.length(0);
+        });
+
+        it('finds a banner at the top of the page that is not positioned', () => {
+            container.innerHTML = IN_FLOW_BANNER;
+            expect(popupIds(true)).to.include('banner');
+        });
+
+        it('makes an in-flow banner actionable', () => {
+            container.innerHTML = IN_FLOW_BANNER;
+            const popup = getActionablePopups('reject', 1000, true).find((p) => p.element.id === 'banner');
+            expect(popup).to.exist;
+            expect(popup!.regexClassification).to.equal('reject');
+        });
+
+        it('ignores an in-flow banner below the fold', () => {
+            container.innerHTML = `<div style="height: 3000px"></div>${IN_FLOW_BANNER}`;
+            expect(popupIds(true)).not.to.include('banner');
+        });
+
+        it('ignores page copy about cookies without consent buttons', () => {
+            container.innerHTML =
+                '<div id="copy"><p>We use cookies on this site.</p><a href="#">Privacy Policy</a> <a href="#">Contact</a></div>';
+            expect(popupIds(true)).not.to.include('copy');
+        });
+
+        it('ignores a login form with consent wording', () => {
+            container.innerHTML = `
+                <form id="login">
+                    <input type="email" placeholder="Email">
+                    <p>By continuing, you agree to our Terms and acknowledge our Privacy Policy and use of cookies.</p>
+                    <button>Continue</button>
+                </form>
+            `;
+            expect(popupIds(true)).not.to.include('login');
+        });
+
+        it('does not report a positioned popup twice', () => {
+            container.innerHTML = fixedBanner('banner');
+            expect(popupIds(true).filter((id) => id === 'banner')).to.have.length(1);
+        });
+
+        it('prefers a positioned popup with reject over an in-flow accept-only banner', () => {
+            container.innerHTML = `
+                <div id="inflow"><p>We use cookies to improve your experience.</p><button>Accept all</button></div>
+                <div id="fixed" style="position: fixed; bottom: 0;"><p>We use cookies.</p><button>Reject all</button><button>Accept all</button></div>
+            `;
+            const popups = inContainer(getActionablePopups('tier2', 1000, true));
+            expect(popups.map((p) => [p.element.id, p.regexClassification])).to.deep.equal([
+                ['fixed', 'reject'],
+                ['inflow', 'tier2'],
+            ]);
+        });
+
+        it('skips the DOM walk when hit-testing finds a popup with reject', () => {
+            // pointer-events: none keeps the fixed popup out of hit-testing, so only the walk could find it
+            container.innerHTML = `
+                ${IN_FLOW_BANNER}
+                <div id="fixed" style="position: fixed; bottom: 0; pointer-events: none;"><p>We use cookies.</p><button>Reject all</button></div>
+            `;
+            const actionable = inContainer(getActionablePopups('tier2', 1000, true));
+            expect(actionable.map((p) => p.element.id)).to.deep.equal(['banner']);
+            expect(popupIds(true)).to.include.members(['banner', 'fixed']);
+        });
+
+        it('finds an in-flow banner inside a shadow root', () => {
+            shadowHost('', IN_FLOW_BANNER);
+            expect(popupIds(true)).to.include('banner');
+        });
+
+        describe('inside a walk candidate that is not a consent popup', () => {
+            // the fixed wrapper is found by the walk, but its age gate text excludes it
+            function ageGateWrapper(bannerFirst: boolean): string {
+                const banner =
+                    '<div id="banner"><p>We use cookies to improve your experience on our website.</p><button>Accept all</button></div>';
+                const ageGate = '<p>You must be over 18 years old to enter.</p>';
+                return `<div id="wrapper" style="position: fixed; top: 0; left: 0; right: 0;">${bannerFirst ? banner + ageGate : ageGate + banner}</div>`;
+            }
+
+            it('keeps a consent popup found by hit-testing next to the walk candidate', () => {
+                // the top grid row hits the banner, so hit-testing finds the banner rather than the wrapper
+                container.innerHTML = ageGateWrapper(true);
+                const popups = inContainer(getPotentialPopups(1000, true));
+                expect(popups.map((p) => [p.element.id, p.source])).to.deep.equal([
+                    ['wrapper', 'walk'],
+                    ['banner', 'hitTest'],
+                ]);
+                expect(popups[0].hitPoints).to.be.undefined;
+                expect(popups[1].hitPoints).not.to.be.empty;
+            });
+
+            it('makes the consent popup found by hit-testing actionable', () => {
+                container.innerHTML = ageGateWrapper(true);
+                const actionable = inContainer(getActionablePopups('tier2', 1000, true));
+                expect(actionable.map((p) => [p.element.id, p.regexClassification])).to.deep.equal([['banner', 'tier2']]);
+            });
+
+            it('reports the wrapper once when hit-testing also reaches it', () => {
+                // the top grid row hits the age gate text, so the climb reaches the wrapper
+                container.innerHTML = ageGateWrapper(false);
+                const popups = inContainer(getPotentialPopups(1000, true));
+                expect(popups.map((p) => [p.element.id, p.source])).to.deep.equal([['wrapper', 'walk']]);
+                expect(popups[0].hitPoints).not.to.be.empty;
+                expect(inContainer(getActionablePopups('tier2', 1000, true))).to.be.empty;
+            });
+        });
+
+        it('keeps a banner with reject found by hit-testing next to its accept-only fragment found by the walk', () => {
+            // the walk keeps the fixed bar, an accept-only fragment that passes on its own; hit-testing the reject button reaches the banner
+            container.innerHTML = `
+                <div id="banner" style="position: fixed; bottom: 0; left: 0; right: 0; height: 200px;">
+                    <div id="bar" style="position: fixed; bottom: 120px; left: 0; right: 0; height: 40px;"><span>We use cookies to improve your experience.</span><button>Accept all</button></div>
+                    <button style="position: absolute; bottom: 0; left: 0; width: 100%; height: 100px;">Reject all</button>
+                </div>
+            `;
+            const popups = inContainer(getPotentialPopups(1000, true));
+            expect(popups.map((p) => [p.element.id, p.source, p.regexClassification])).to.deep.equal([
+                ['bar', 'walk', 'tier2'],
+                ['banner', 'hitTest', 'reject'],
+            ]);
+            const actionable = inContainer(getActionablePopups('tier2', 1000, true));
+            expect(actionable.map((p) => p.element.id)).to.deep.equal(['banner']);
+        });
+    });
+
+    describe('order', () => {
+        it('sorts actionable popups by classification, and keeps search order within one', () => {
+            container.innerHTML = `
+                <div id="accept1" style="position: fixed; top: 0;"><p>We use cookies on this website.</p><button>Accept all</button></div>
+                <div id="ok" style="position: fixed; top: 100px;"><p>We use cookies on this website.</p><button>OK</button></div>
+                <div id="accept2" style="position: fixed; top: 200px;"><p>We use cookies on this website.</p><button>Accept all</button></div>
+            `;
+            const searchOrder = inContainer(getPotentialPopups(1000))
+                .filter((p) => p.regexClassification === 'tier2')
+                .map((p) => p.element.id);
+            expect(searchOrder).to.have.members(['accept1', 'accept2']);
+            const popups = inContainer(getActionablePopups('tier2', 1000));
+            expect(popups.map((p) => p.element.id)).to.deep.equal(['ok', ...searchOrder]);
+        });
+    });
+
+    describe('labels', () => {
+        it('labels every potential popup with a classification from its buttons', () => {
+            container.innerHTML = fixedBanner('consent');
+            // outside the container, so that the walk does not expand it to the container that holds the consent banner
+            const chat = document.createElement('div');
+            chat.innerHTML = '<div id="chat" style="position: fixed; top: 0;"><p>Chat with us</p><button>Start chat</button></div>';
+            document.body.appendChild(chat);
+            try {
+                const labels = Object.fromEntries(
+                    getPotentialPopups(1000)
+                        .filter((p) => container.contains(p.element) || chat.contains(p.element))
+                        .map((p) => [p.element.id, p.regexClassification]),
+                );
+                expect(labels).to.deep.equal({ consent: 'reject', chat: 'none' });
+            } finally {
+                chat.remove();
+            }
+        });
+    });
+
+    describe('hit-test grid and hit points', () => {
+        const GRID = { columns: [0.25, 0.75], edgeRows: [40] };
+        const columnXs = GRID.columns.map((c) => Math.round(c * window.innerWidth));
+
+        function findGridPopup(id: string) {
+            return getPotentialPopups(1000, true, GRID).find((p) => p.element.id === id);
+        }
+
+        it('records every grid point that reached an in-flow banner', () => {
+            container.innerHTML = `<div id="banner" style="padding: 60px 0;">${BANNER}</div>`;
+            const popup = findGridPopup('banner');
+            expect(popup?.source).to.equal('hitTest');
+            expect(popup!.hitPoints).to.deep.equal(columnXs.map((x) => [x, 40]));
+        });
+
+        it('keeps the walk candidate of a positioned popup, with its hit points', () => {
+            container.innerHTML = `<div id="fixed" style="position: fixed; bottom: 0; left: 0; right: 0; padding: 30px 0;">${BANNER}</div>`;
+            const popup = findGridPopup('fixed');
+            expect(popup?.source).to.equal('walk');
+            const bottomRow = Math.round(window.innerHeight - 40);
+            expect(popup!.hitPoints).to.deep.equal(columnXs.map((x) => [x, bottomRow]));
+        });
+
+        it('leaves out hit points of a positioned popup that is not hit-tested', () => {
+            container.innerHTML = fixedBanner('fixed');
+            const popup = getPotentialPopups(1000).find((p) => p.element.id === 'fixed');
+            expect(popup?.source).to.equal('walk');
+            expect(popup!.hitPoints).to.be.undefined;
+        });
+
+        it('tests rows 30 and 90px from the edges by default', () => {
+            // the banner lies between 100 and 200px from the top; the spacer is outside the container, so climbs from it stop at body
+            const spacer = document.createElement('div');
+            spacer.style.height = '100px';
+            document.body.insertBefore(spacer, document.body.firstChild);
+            document.body.style.margin = '0';
+            container.innerHTML = `<div id="banner" style="height: 100px; overflow: hidden;">${BANNER}</div>`;
+            try {
+                expect(container.querySelector('#banner')!.getBoundingClientRect().top).to.equal(100);
+                expect(inContainer(getPotentialPopups(1000, true))).to.have.length(0);
+                const withRow150 = getPotentialPopups(1000, true, { columns: [0.1, 0.5, 0.9], edgeRows: [30, 90, 150] });
+                expect(withRow150.map((p) => p.element.id)).to.include('banner');
+            } finally {
+                spacer.remove();
+                document.body.style.margin = '';
+            }
+        });
+
+        it('only tests the points of the grid it is given', () => {
+            container.innerHTML = `<div id="banner" style="padding: 60px 0;">${BANNER}</div>`;
+            expect(getPotentialPopups(1000, true, { columns: [0.5], edgeRows: [] }).map((p) => p.element.id)).not.to.include('banner');
+        });
+    });
 });
 
 describe('getFramePopupElement', () => {

package/tests-wtr/heuristics/heuristic-cmp.ts

@@ -5,11 +5,12 @@ import { HeuristicLevel } from '../../lib/types';
 
 // must be run from heuristic-cmp.html
 describe('AutoConsentHeuristicCMP', () => {
-    function createAutoconsent(heuristicMode: HeuristicLevel) {
+    function createAutoconsent(heuristicMode: HeuristicLevel, enableHeuristicHitTest = false) {
         return new Autoconsent(() => Promise.resolve(), {
             enabled: false,
             autoAction: null,
             heuristicMode,
+            enableHeuristicHitTest,
         });
     }
 
@@ -40,6 +41,20 @@ describe('AutoConsentHeuristicCMP', () => {
             expect(cmp.name).to.equal('HEURISTIC-REJECT');
         });
 
+        it('hit-tests for an in-flow banner only when enableHeuristicHitTest is set', async () => {
+            const banner = document.createElement('div');
+            banner.innerHTML = '<p>We use cookies to improve your experience.</p><button>Reject all</button><button>Accept all</button>';
+            document.body.prepend(banner);
+            try {
+                expect(await new AutoConsentHeuristicCMP(createAutoconsent('tier2'), 'tier2').detectCmp()).to.be.false;
+                const cmp = new AutoConsentHeuristicCMP(createAutoconsent('tier2', true), 'tier2');
+                expect(await cmp.detectCmp()).to.be.true;
+                expect(cmp.name).to.equal('HEURISTIC-REJECT');
+            } finally {
+                banner.remove();
+            }
+        });
+
         it('detects a Russian popup using the Cyrillic word for cookies', async () => {
             const autoconsent = createAutoconsent('tier2');
             const cmp = new AutoConsentHeuristicCMP(autoconsent, 'tier2');

package/tests-wtr/heuristics/heuristics-utils.test.ts

@@ -4,6 +4,7 @@ import {
     cleanButtonText,
     classifyButtonTextRegex,
     classifyButtons,
+    classifyPopup,
     isExcludedPopup,
 } from '../../lib/heuristic-classify';
 import { getActionablePopups, isDisabled, excludeContainers, getButtonData, isDialogLikeElement } from '../../lib/heuristics';
@@ -387,6 +388,38 @@ describe('classifyButtonTextRegex', () => {
         expect(classifyButtonTextRegex('Сохранить и закрыть')).to.equal('acknowledge');
     });
 
+    it('matches Catalan acknowledge buttons', () => {
+        expect(classifyButtonTextRegex('Entenc')).to.equal('acknowledge');
+        expect(classifyButtonTextRegex('Ho entenc')).to.equal('acknowledge');
+        expect(classifyButtonTextRegex('Entès!')).to.equal('acknowledge');
+        expect(classifyButtonTextRegex('Entesos')).to.equal('acknowledge');
+    });
+
+    it('matches Catalan necessary-only buttons as reject', () => {
+        expect(classifyButtonTextRegex('Només les necessàries')).to.equal('reject');
+        expect(classifyButtonTextRegex('Només necessàries')).to.equal('reject');
+        expect(classifyButtonTextRegex('Necessàries')).to.equal('reject');
+        expect(classifyButtonTextRegex('Acceptar només les galetes necessàries')).to.equal('reject');
+        expect(classifyButtonTextRegex('Només galetes tècniques')).to.equal('reject');
+        expect(classifyButtonTextRegex('Només les essencials')).to.equal('reject');
+        expect(classifyButtonTextRegex('Continuar sense acceptar')).to.equal('reject');
+    });
+
+    it('matches Catalan settings buttons', () => {
+        expect(classifyButtonTextRegex('Configuració')).to.equal('settings');
+        expect(classifyButtonTextRegex('Personalitzar')).to.equal('settings');
+        expect(classifyButtonTextRegex('Preferències')).to.equal('settings');
+        expect(classifyButtonTextRegex('Més opcions')).to.equal('settings');
+        expect(classifyButtonTextRegex('Configuració de galetes')).to.equal('settings');
+        expect(classifyButtonTextRegex('Gestionar les meves preferències')).to.equal('settings');
+        expect(classifyButtonTextRegex('Gestiona les galetes')).to.equal('settings');
+    });
+
+    it('does not treat Catalan save or accept buttons as settings or reject', () => {
+        expect(classifyButtonTextRegex('Desar preferències')).to.not.equal('settings');
+        expect(classifyButtonTextRegex('Acceptar')).to.equal('accept');
+    });
+
     it('does not treat Russian subscription buttons as reject', () => {
         expect(classifyButtonTextRegex('Отказаться от подписки')).to.equal('other');
     });
@@ -412,6 +445,22 @@ describe('classifyButtonTextRegex', () => {
     });
 });
 
+describe('classifyPopup', () => {
+    function classify(texts: string[]) {
+        return classifyPopup(texts.map((text) => ({ text, regexClassification: classifyButtonTextRegex(text) })));
+    }
+
+    it('dismisses a Catalan notice-only banner as tier1', () => {
+        expect(classify(['Entenc'])).to.equal('tier1');
+    });
+
+    it('does not acknowledge a Catalan banner that offers necessary-only or settings', () => {
+        expect(classify(['Entès', 'Només les necessàries'])).to.equal('reject');
+        expect(classify(['Entès', 'Preferències'])).to.equal('none');
+        expect(classify(['Entès', 'Personalitzar'])).to.equal('none');
+    });
+});
+
 describe('classifyButtons', () => {
     it('separates reject buttons from other buttons', () => {
         const buttons: ButtonData[] = [

package/rules/generated/auto_CA_plex.tv_ee4.json

@@ -1,40 +0,0 @@
-{
-    "name": "auto_CA_plex.tv_ee4",
-    "cosmetic": false,
-    "_metadata": {
-        "vendorUrl": "http://www.plex.tv/"
-    },
-    "runContext": {
-        "main": true,
-        "frame": false,
-        "urlPattern": "^https?://(www\\.)?plex\\.tv/"
-    },
-    "prehideSelectors": [],
-    "detectCmp": [
-        {
-            "exists": "body > div:not([id]) > div:nth-child(3):not([id]) > div:nth-child(1):not([id]) > div:nth-child(2):not([id]) > a:not([id])"
-        }
-    ],
-    "detectPopup": [
-        {
-            "visible": "body > div:not([id]) > div:nth-child(3):not([id]) > div:nth-child(1):not([id]) > div:nth-child(2):not([id]) > a:not([id])"
-        }
-    ],
-    "optIn": [],
-    "optOut": [
-        {
-            "wait": 500
-        },
-        {
-            "waitForThenClick": "body > div:not([id]) > div:nth-child(3):not([id]) > div:nth-child(1):not([id]) > div:nth-child(2):not([id]) > a:not([id])",
-            "comment": "I Reject"
-        }
-    ],
-    "test": [
-        {
-            "waitForVisible": "body > div:not([id]) > div:nth-child(3):not([id]) > div:nth-child(1):not([id]) > div:nth-child(2):not([id]) > a:not([id])",
-            "timeout": 1000,
-            "check": "none"
-        }
-    ]
-}

package/rules/generated/auto_CA_support.plex.tv_gn3.json

@@ -1,40 +0,0 @@
-{
-    "name": "auto_CA_support.plex.tv_gn3",
-    "cosmetic": false,
-    "_metadata": {
-        "vendorUrl": "http://support.plex.tv/"
-    },
-    "runContext": {
-        "main": true,
-        "frame": false,
-        "urlPattern": "^https?://(www\\.)?support\\.plex\\.tv/"
-    },
-    "prehideSelectors": [],
-    "detectCmp": [
-        {
-            "exists": "body > div:not([id]) > div:nth-child(3):not([id]) > div:nth-child(1):not([id]) > div:nth-child(2):not([id]) > a:not([id])"
-        }
-    ],
-    "detectPopup": [
-        {
-            "visible": "body > div:not([id]) > div:nth-child(3):not([id]) > div:nth-child(1):not([id]) > div:nth-child(2):not([id]) > a:not([id])"
-        }
-    ],
-    "optIn": [],
-    "optOut": [
-        {
-            "wait": 500
-        },
-        {
-            "waitForThenClick": "body > div:not([id]) > div:nth-child(3):not([id]) > div:nth-child(1):not([id]) > div:nth-child(2):not([id]) > a:not([id])",
-            "comment": "I Reject"
-        }
-    ],
-    "test": [
-        {
-            "waitForVisible": "body > div:not([id]) > div:nth-child(3):not([id]) > div:nth-child(1):not([id]) > div:nth-child(2):not([id]) > a:not([id])",
-            "timeout": 1000,
-            "check": "none"
-        }
-    ]
-}

package/tests/generated/auto_CA_plex.tv_ee4.spec.ts

@@ -1,2 +0,0 @@
-import generateCMPTests from '../../playwright/runner';
-generateCMPTests('auto_CA_plex.tv_ee4', ['https://www.plex.tv/'], { testOptIn: false, testSelfTest: true, onlyRegions: ['CA'] });

package/tests/generated/auto_CA_support.plex.tv_gn3.spec.ts

@@ -1,6 +0,0 @@
-import generateCMPTests from '../../playwright/runner';
-generateCMPTests('auto_CA_support.plex.tv_gn3', ['https://support.plex.tv/'], {
-    testOptIn: false,
-    testSelfTest: true,
-    onlyRegions: ['CA'],
-});