@duckduckgo/autoconsent 16.49.0 → 16.51.0

package/dist/types/eval-snippets.d.ts

@@ -40,6 +40,7 @@ export declare const snippets: {
     EVAL_COOKIEFIRST_1: () => boolean;
     EVAL_COOKIEINFORMATION_0: () => any;
     EVAL_COOKIEINFORMATION_1: () => any;
+    EVAL_EE_CONSENT_DECLINE_0: () => boolean;
     EVAL_ETSY_0: () => boolean;
     EVAL_ETSY_1: () => any;
     EVAL_EZOIC_0: () => any;
@@ -87,5 +88,6 @@ export declare const snippets: {
     EVAL_USERCENTRICS_API_6: () => boolean;
     EVAL_USERCENTRICS_BUTTON_0: () => any;
     EVAL_WAITROSE_0: () => boolean;
+    EVAL_WBD_LTP_CLOSE: () => any;
 };
 export declare function getFunctionBody(snippetFunc: () => any): string;

package/dist/types/heuristic-classify.d.ts

@@ -0,0 +1,16 @@
+export * from './heuristic-patterns';
+export type ButtonRegexClassification = 'reject' | 'settings' | 'accept' | 'acknowledge' | 'other';
+export type PopupClassification = 'none' | 'reject' | 'tier1' | 'tier2';
+export type ClassifiableButton = {
+    text: string;
+    regexClassification?: ButtonRegexClassification;
+};
+export declare function checkHeuristicPatterns(allText: string, detectPatterns?: RegExp[]): {
+    patterns: string[];
+    snippets: string[];
+};
+export declare function classifyButtons(buttons: ClassifiableButton[]): void;
+export declare function isExcludedPopup(popupText: string, excludePatterns?: RegExp[]): boolean;
+export declare function classifyPopup(buttons: ClassifiableButton[]): PopupClassification;
+export declare function cleanButtonText(buttonText: string): string;
+export declare function classifyButtonTextRegex(buttonText: string): ButtonRegexClassification;

package/dist/types/heuristics.d.ts

@@ -1,13 +1,6 @@
-import { ButtonData, ButtonRegexClassification, HeuristicLevel, PopupData } from './types';
-export declare function checkHeuristicPatterns(allText: string, detectPatterns?: RegExp[]): {
-    patterns: string[];
-    snippets: string[];
-};
+import { ButtonData, HeuristicLevel, PopupData } from './types';
 export declare function getActionablePopups(mode?: HeuristicLevel, timeout?: number): PopupData[];
-export declare function classifyButtons(buttons: ButtonData[]): void;
-export declare function isExcludedPopup(popupText: string, excludePatterns?: RegExp[]): boolean;
-export declare function cleanButtonText(buttonText: string): string;
-export declare function classifyButtonTextRegex(buttonText: string): ButtonRegexClassification;
+export declare function getPotentialPopups(timeout?: number): PopupData[];
 export declare function isDialogLikeElement(node: HTMLElement): boolean;
 /**
  * Serialize all actionable buttons on the page

package/dist/types/types.d.ts

@@ -1,6 +1,7 @@
 import { ContentScriptMessage } from './messages';
 import { AutoConsentCMPRule, ElementSelector, HideMethod, RunContext, VisibilityCheck } from './rules';
 import { CompactCMPRuleset } from './encoding';
+import type { ButtonRegexClassification, ClassifiableButton, PopupClassification } from './heuristic-classify';
 export type MessageSender = (message: ContentScriptMessage) => Promise<void>;
 export interface AutoCMP {
     name: string;
@@ -85,11 +86,9 @@ export type ConsentState = {
     endTime: number;
     performance?: Record<string, number[]>;
 };
-export type ButtonRegexClassification = 'reject' | 'settings' | 'accept' | 'acknowledge' | 'other';
-export interface ButtonData {
-    text: string;
+export type { ButtonRegexClassification, PopupClassification };
+export interface ButtonData extends ClassifiableButton {
     element: HTMLElement;
-    regexClassification?: ButtonRegexClassification;
 }
 export interface PopupData {
     text: string;
@@ -105,4 +104,3 @@ export interface PopupData {
  *  - tier2: Heuristic detection is enabled, and will click the accept button on a popup if it exists, and no Reject, Acknowledge, or Settings button exists.
  */
 export type HeuristicLevel = 'off' | 'reject' | 'tier1' | 'tier2';
-export type PopupClassification = 'none' | 'reject' | 'tier1' | 'tier2';

package/lib/eval-snippets.ts

@@ -148,6 +148,9 @@ export const snippets = {
             .forEach((i) => i.getAttribute('aria-checked') === 'true' && i.click()) || true,
     EVAL_COOKIEINFORMATION_0: () => CookieInformation.declineAllCategories() || true,
     EVAL_COOKIEINFORMATION_1: () => CookieInformation.submitAllCategories() || true,
+    // turn the single-button form into a decline submission, same fields as the CMP's own decline form
+    EVAL_EE_CONSENT_DECLINE_0: () =>
+        document.querySelectorAll('#cookieConsentFormAccept input[name^="ee:cookies_"]').forEach((i) => (i.value = 'n')) || true,
     EVAL_ETSY_0: () =>
         document.querySelectorAll('.gdpr-overlay-body input').forEach((toggle) => {
             toggle.checked = false;
@@ -264,6 +267,7 @@ export const snippets = {
     EVAL_USERCENTRICS_BUTTON_0: () =>
         JSON.parse(localStorage.getItem('usercentrics')).consents.every((c) => c.isEssential || !c.consentStatus),
     EVAL_WAITROSE_0: () => Array.from(document.querySelectorAll('label[id$=cookies-deny-label]')).forEach((e) => e.click()) || true,
+    EVAL_WBD_LTP_CLOSE: () => document.querySelector('dialog#ltp-dialog').close() || true,
 };
 
 export function getFunctionBody(snippetFunc: () => any) {

package/lib/heuristic-classify.ts

@@ -0,0 +1,136 @@
+/**
+ * Text-only heuristic classification: popup text and button labels in, classifications out.
+ * Must not depend on DOM APIs, so that it can be used from Node (e.g. by tracker-radar-collector).
+ */
+import {
+    ACCEPT_PATTERNS,
+    ACKNOWLEDGE_PATTERNS,
+    BUTTON_NEVER_MATCH_PATTERNS,
+    DETECT_NEVER_MATCH_PATTERNS,
+    DETECT_PATTERNS,
+    REJECT_PATTERNS,
+    SETTINGS_PATTERNS,
+} from './heuristic-patterns';
+export * from './heuristic-patterns';
+
+export type ButtonRegexClassification = 'reject' | 'settings' | 'accept' | 'acknowledge' | 'other';
+export type PopupClassification = 'none' | 'reject' | 'tier1' | 'tier2';
+
+export type ClassifiableButton = {
+    text: string;
+    regexClassification?: ButtonRegexClassification;
+};
+
+const TEXT_LIMIT = 100000;
+
+export function checkHeuristicPatterns(allText: string, detectPatterns = DETECT_PATTERNS) {
+    allText = allText.slice(0, TEXT_LIMIT);
+    const patterns = [];
+    const snippets = [];
+
+    for (const p of detectPatterns) {
+        const matches = allText?.match(p);
+        if (matches) {
+            patterns.push(p.toString());
+            // When a non-global regex has capture groups, `String.prototype.match`
+            // returns undefined entries for optional groups that did not match.
+            for (const m of matches) {
+                if (typeof m === 'string') {
+                    snippets.push(m.substring(0, 200));
+                }
+            }
+        }
+    }
+    return { patterns, snippets };
+}
+
+export function classifyButtons(buttons: ClassifiableButton[]) {
+    for (const button of buttons) {
+        button.regexClassification = classifyButtonTextRegex(button.text);
+    }
+}
+
+export function isExcludedPopup(popupText: string, excludePatterns = DETECT_NEVER_MATCH_PATTERNS): boolean {
+    if (!popupText) {
+        return false;
+    }
+    const truncated = popupText.slice(0, TEXT_LIMIT);
+    return excludePatterns.some((p) => p.test(truncated));
+}
+
+export function classifyPopup(buttons: ClassifiableButton[]): PopupClassification {
+    const { reject, settings, accept, acknowledge } = buttons.reduce(
+        (acc, button) => {
+            if (button.regexClassification && button.regexClassification !== 'other') {
+                acc[button.regexClassification]++;
+            }
+            return acc;
+        },
+        { reject: 0, settings: 0, accept: 0, acknowledge: 0 },
+    );
+    if (reject > 0) {
+        return 'reject';
+    }
+    if (settings > 0) {
+        return 'none';
+    }
+    if (acknowledge > 0) {
+        return 'tier1';
+    }
+    if (accept > 0) {
+        return accept === 1 ? 'tier2' : 'none';
+    }
+    return 'none';
+}
+
+/**
+ * @param {string} buttonText
+ * @param {Array<string|RegExp>} matchPatterns
+ * @param {Array<string|RegExp>} neverMatchPatterns
+ * @returns {boolean}
+ */
+function testButtonMatches(buttonText: string, matchPatterns: (string | RegExp)[], neverMatchPatterns: (string | RegExp)[]): boolean {
+    if (!buttonText) {
+        return false;
+    }
+    const cleanedButtonText = cleanButtonText(buttonText);
+    return (
+        !neverMatchPatterns.some((p) => (p instanceof RegExp && p.test(cleanedButtonText)) || p === cleanedButtonText) &&
+        matchPatterns.some((p) => (p instanceof RegExp && p.test(cleanedButtonText)) || p === cleanedButtonText)
+    );
+}
+
+export function cleanButtonText(buttonText: string): string {
+    // lowercase
+    let result = buttonText.toLowerCase();
+    // remove special characters
+    result = result.replace(/[“”"'/#&[\]→✕×⟩❯><✗ב’›«»]+/g, '');
+    // remove emojis
+    result = result.replace(
+        /[\u{1F600}-\u{1F64F}\u{1F300}-\u{1F5FF}\u{1F680}-\u{1F6FF}\u{1F1E0}-\u{1F1FF}\u2600-\u26FF\u2700-\u27BF\u{1F900}-\u{1F9FF}\u{1FA70}-\u{1FAFF}]/gu,
+        '',
+    );
+    // remove newlines
+    result = result.replace(/\n+/g, ' ');
+    // remove multiple spaces
+    result = result.replace(/\s+/g, ' ');
+    // strip whitespace around the text
+    result = result.trim();
+    return result;
+}
+
+export function classifyButtonTextRegex(buttonText: string): ButtonRegexClassification {
+    if (testButtonMatches(buttonText, REJECT_PATTERNS, BUTTON_NEVER_MATCH_PATTERNS)) {
+        return 'reject';
+    }
+    if (testButtonMatches(buttonText, SETTINGS_PATTERNS, BUTTON_NEVER_MATCH_PATTERNS)) {
+        return 'settings';
+    }
+    if (testButtonMatches(buttonText, ACCEPT_PATTERNS, BUTTON_NEVER_MATCH_PATTERNS)) {
+        return 'accept';
+    }
+    if (testButtonMatches(buttonText, ACKNOWLEDGE_PATTERNS, BUTTON_NEVER_MATCH_PATTERNS)) {
+        return 'acknowledge';
+    }
+    return 'other';
+}

package/lib/heuristic-patterns.ts

@@ -7,7 +7,8 @@ export const DETECT_PATTERNS = [
     /only necessary cookies/gi, // "only necessary" is probably too broad
     /(?:by continuing.{0,100}cookie)|(?:cookie.{0,100}by continuing)/gi,
     /(?:by continuing.{0,100}privacy)|(?:privacy.{0,100}by continuing)/gi,
-    /we (?:use|serve)(?: optional| essential)? cookies/gi,
+    // a single qualifier is common: "optional", "essential", "necessary", "analytics", ...
+    /we (?:use|serve)(?: \w+)? cookies/gi,
     /we are using cookies/gi,
     /use of cookies/gi,
     /website uses cookies to enhance your browsing experience/gi,
@@ -165,6 +166,10 @@ export const DETECT_PATTERNS = [
 
     // Italian (IT)
     /usiamo.{0,20}cookie/gi,
+
+    // Japanese (JA)
+    // e.g. "当サイトはCookieを使用しています", "クッキー（Cookie）を利用しています"
+    /(?:cookie|クッキー).{0,10}を(?:使用|利用)/gi,
 ];
 
 const REJECT_PATTERNS_ENGLISH = [
@@ -185,8 +190,9 @@ const REJECT_PATTERNS_ENGLISH = [
     /^\s*(i\s+)?(want\s+to\s+)?only\s+(strictly\s+)?(necessary|essential|essentials|functional|required|minimal)\s*(cookies)?\s*(continue|only)?\s*$/is,
     /^\s*(strictly\s+)?(necessary|essential|essentials|functional|required|minimal)\s+((cookies\s+)?only|only\s+cookies)\s*$/is,
 
-    // e.g. "do not sell or share my personal information", "opt out of sale ..." (CCPA)
-    /do\s+not\s+sell|opt\s+out\s+of\s+sale/is,
+    // e.g. "do not sell or share my personal information", "do not share my personal information", "opt out of sale ..." (CCPA)
+    // "do not sell page" is a link; "do not sellcookie settings" is two concatenated buttons
+    /do\s+not\s+sell(?!\s*(page|cookie settings))|do\s+not\s+share\s+my\s+(personal\s+)?(information|info|data)|opt\s+out\s+of\s+sale/is,
 
     // e.g. "opt-out of sale/share or targeted advertising", "opt-out of advertising/social media cookies"
     /^opt[ -]?out of /is,
@@ -201,7 +207,6 @@ const REJECT_PATTERNS_ENGLISH = [
     /^no,? thank(s| you)$/is,
     /^opt[ -]out$/is,
     'dont enable',
-    'withdraw consent',
     'i do not agree',
 ];
 
@@ -236,10 +241,11 @@ const REJECT_PATTERNS_DUTCH = [
 const REJECT_PATTERNS_FRENCH = [
     // refuser / rejeter / interdire / décliner (reject verbs, any position)
     // "refuser et s'abonner" / "refuser et payer" are excluded via BUTTON_NEVER_MATCH_PATTERNS
-    /(^|\s)(refus|rejet|rejeter|interdire|interdis|déclin|declin)/is,
+    // skip English prose such as "cookies that you can refuse"
+    /^(?!.*\bcan refuse\b)(.*\s)?(refus|rejet|rejeter|interdire|interdis|déclin|declin)/is,
 
     // only necessary / essential / technical / functional
-    /(uniquement|seulement|indispensable|strictement nécessaire|que les cookies\s+(nécessaires|techniques|essentiels|indispensables|fonctionnels))/is,
+    /^(?=.*(cookie|nécessaire|essentiel|techni|requis|obligatoire|fonctionnel|indispensable|sdk)).*(uniquement|seulement|indispensable|strictement nécessaire|que les cookies\s+(nécessaires|techniques|essentiels|indispensables|fonctionnels))/is,
 
     // continue/proceed without accepting; refuse everything; disable purposes
     /(sans accepter|ne pas accepter|je naccepte rien|je désactive)/is,
@@ -318,7 +324,7 @@ const REJECT_PATTERNS_SPANISH = [
     /^(no,?\s+)?(solo|sólo|només)\s+(usar\s+|las?\s+|los\s+|lo\s+)?.{0,20}(necesari|esencial|estrictamente)/is,
 
     // refusals / opt-outs
-    /^(no acept|no consentir|no permitir|no estoy de acuerdo|no,? gracias|sin consentimiento|revocar consentimiento|continuar sin aceptar|prefiero rechazarlas|descartar todas)/is,
+    /^(no acept|no consentir|no permitir|no estoy de acuerdo|no,? gracias|sin consentimiento|continuar sin aceptar|prefiero rechazarlas|descartar todas)/is,
 
     'acceptar només les necessàries',
     'només sutilitzen cookies quan és necessari',
@@ -390,6 +396,13 @@ const REJECT_PATTERNS_TURKISH = ['reddet', 'çerezleri reddet'];
 
 const REJECT_PATTERNS_INDONESIAN = ['tolak cookie'];
 
+const REJECT_PATTERNS_JAPANESE = [
+    // e.g. "拒否", "すべて拒否", "全て拒否する", "cookieを拒否"
+    /^(すべて|全て)?\s?(の)?(cookie|クッキー)?\s?(を)?拒否(する)?$/i,
+    '同意しない',
+    '同意しません',
+];
+
 /**
  * @type {Array<string|RegExp>}
  */
@@ -411,9 +424,13 @@ export const REJECT_PATTERNS = [
     ...REJECT_PATTERNS_RUSSIAN,
     ...REJECT_PATTERNS_TURKISH,
     ...REJECT_PATTERNS_INDONESIAN,
+    ...REJECT_PATTERNS_JAPANESE,
 ];
 
 export const BUTTON_NEVER_MATCH_PATTERNS = [
+    // text of several concatenated buttons, or popup body text rather than a button
+    /accept all.*reject all|reject all.*accept all/is,
+    /^.{300,}$/s,
     /pay|subscribe/is,
     /abonneer/is,
     /abonnier/is,
@@ -461,9 +478,11 @@ export const SETTINGS_PATTERNS = [
     // Multilingual "open customization" patterns: a customization verb next to a
     // cookie/preference/settings/options/details/purposes noun (both word orders).
     // The negative lookahead avoids policy links and save/confirm/accept phrases (those are accept/acknowledge/other).
-    /^(?!.*\b(policy|policies|notice|statement|impressum|richtlinie|beleid|politique|política|polityk|speichern|guardar|opslaan|zapisz|enregistrer|sauvegarder|bevestig|bestätig|confirm|save|submit|akzeptier|accept|zaakcept)\b)(customi[sz]e|manage|adjust|configure|personali[sz]e|let me choose|edit|change|set|select|view|see|review|update|open|show|choose|anpassen|verwalten|konfigurieren|bearbeiten|öffnen|anzeigen|einblenden|festlegen|auswählen|wählen|aanpassen|beheren|instellen|wijzig\w*|personaliseren|personaliseer|kies|bekijk|toon|personnaliser|paramétrer|gérer|configurer|choisir|afficher|définir|modifier|configurar|personalizar|gestionar|administrar|ajustar|seleccionar|modificar|establecer|dostosuj|zarządzaj|personalizuj|ustaw\w*|zmień|pokaż|wybierz)\b.{0,20}(cookies?|preferences?|settings?|options?|choices?|controls?|details?|purposes?|services?|consent|einstellung\w*|optionen|präferenzen|einzelheiten|zwecke|dienste|datenschutz\w*|auswahl|voorkeur\w*|instelling\w*|opties|diensten|préférences|paramètres|réglages|choix|détails|finalités|témoins|preferencia\w*|opciones|ajustes|configuraci\w*|detalles|servicios|elección|preferencj\w*|ustawie\w*|opcje|szczegół\w*|cele|galetes)\b/is,
-    /^(?!.*\b(policy|policies|notice|statement|impressum|richtlinie|beleid|politique|política|polityk|speichern|guardar|opslaan|zapisz|enregistrer|sauvegarder|bevestig|bestätig|confirm|save|submit|akzeptier|accept|zaakcept)\b)(cookies?|preferences?|settings?|options?|choices?|controls?|details?|purposes?|services?|consent|einstellung\w*|optionen|präferenzen|einzelheiten|zwecke|dienste|datenschutz\w*|auswahl|voorkeur\w*|instelling\w*|opties|diensten|préférences|paramètres|réglages|choix|détails|finalités|témoins|preferencia\w*|opciones|ajustes|configuraci\w*|detalles|servicios|elección|preferencj\w*|ustawie\w*|opcje|szczegół\w*|cele|galetes)\b.{0,15}(customi[sz]e|manage|adjust|configure|personali[sz]e|let me choose|edit|change|set|select|view|see|review|update|open|show|choose|anpassen|verwalten|konfigurieren|bearbeiten|öffnen|anzeigen|einblenden|festlegen|auswählen|wählen|aanpassen|beheren|instellen|wijzig\w*|personaliseren|personaliseer|kies|bekijk|toon|personnaliser|paramétrer|gérer|configurer|choisir|afficher|définir|modifier|configurar|personalizar|gestionar|administrar|ajustar|seleccionar|modificar|establecer|dostosuj|zarządzaj|personalizuj|ustaw\w*|zmień|pokaż|wybierz)\b/is,
+    /^(?!.*\b(policy|policies|notice|statement|impressum|\w*richtlinie\w*|\w*erklärung|page|beschrijving|privacy$|beleid|politique|política|polityk|speichern|guardar|opslaan|zapisz|enregistrer|sauvegarder|bevestig|bestätig|confirm|save|submit|akzeptier|accept|zaakcept)\b)(customi[sz]e|manage|adjust|configure|personali[sz]e|let me choose|edit|change|set|select|view|see|review|update|open|show|choose|anpassen|verwalten|konfigurieren|bearbeiten|öffnen|anzeigen|einblenden|einstellen|festlegen|auswählen|wählen|aanpassen|beheren|instellen|wijzig\w*|personaliseren|personaliseer|kies|bekijk|toon|personnaliser|paramétrer|gérer|configurer|choisir|afficher|définir|modifier|configurar|personalizar|gestionar|administrar|ajustar|seleccionar|modificar|establecer|dostosuj|zarządzaj|personalizuj|ustaw\w*|zmień|pokaż|wybierz)\b.{0,20}(cookies?|preferences?|settings?|options?|choices?|controls?|details?|purposes?|services?|consent|einstellung\w*|optionen|präferenzen|einzelheiten|zwecke|dienste|datenschutz\w*|auswahl|voorkeur\w*|instelling\w*|opties|diensten|préférences|paramètres|réglages|choix|détails|finalités|témoins|preferencia\w*|opciones|ajustes|configuraci\w*|detalles|servicios|elección|preferencj\w*|ustawie\w*|opcje|szczegół\w*|cele|galetes)\b/is,
+    /^(?!.*\b(policy|policies|notice|statement|impressum|\w*richtlinie\w*|\w*erklärung|page|beschrijving|privacy$|beleid|politique|política|polityk|speichern|guardar|opslaan|zapisz|enregistrer|sauvegarder|bevestig|bestätig|confirm|save|submit|akzeptier|accept|zaakcept)\b)(cookies?|preferences?|settings?|options?|choices?|controls?|details?|purposes?|services?|consent|einstellung\w*|optionen|präferenzen|einzelheiten|zwecke|dienste|datenschutz\w*|auswahl|voorkeur\w*|instelling\w*|opties|diensten|préférences|paramètres|réglages|choix|détails|finalités|témoins|preferencia\w*|opciones|ajustes|configuraci\w*|detalles|servicios|elección|preferencj\w*|ustawie\w*|opcje|szczegół\w*|cele|galetes)\b(.{0,14}\W)?(customi[sz]e|manage|adjust|configure|personali[sz]e|let me choose|edit|change|set|select|view|see|review|update|open|show|choose|anpassen|verwalten|konfigurieren|bearbeiten|öffnen|anzeigen|einblenden|einstellen|festlegen|auswählen|wählen|aanpassen|beheren|instellen|wijzig\w*|personaliseren|personaliseer|kies|bekijk|toon|personnaliser|paramétrer|gérer|configurer|choisir|afficher|définir|modifier|configurar|personalizar|gestionar|administrar|ajustar|seleccionar|modificar|establecer|dostosuj|zarządzaj|personalizuj|ustaw\w*|zmień|pokaż|wybierz)\b/is,
 
+    // settings noun + cookies, e.g. "paramètres des cookies", "ajustes de cookies", "instellingen voor cookies en diensten"
+    /^(paramètres|préférences|réglages|ajustes|opciones|elección|preferencias|instellingen)( (de|des|les|voor|van))? cookies( (et des|en|y) (services|diensten|servicios))?$/is,
     'settings',
     'preferences',
     /customi(s|z)e/is,
@@ -477,8 +496,8 @@ export const SETTINGS_PATTERNS = [
     'cookieconsent preferences',
     /privacy choices/is,
     /^(privacy|cookie|custom) settings$/is,
-    /^cookies? (settings|preferences|setting)$/is,
-    /(manage|customize|customise|opt-out|edit).*(cookies|preferences|settings|options)/is,
+    /^cookies? (settings|preferences|setting)(?!.*\bdo not\b)/is,
+    /^(?!.*\bpolic(y|ies)\b).*(manage|customize|customise|opt-out|edit).*(cookies|preferences|settings|options)/is,
     'cookie consent options',
     'privacy controls',
     // German
@@ -669,7 +688,7 @@ export const ACCEPT_PATTERNS = [
     'yes',
     /accept all above/is,
     'close and accept',
-    /accept all$/is,
+    /(^|\s)accept all$/is,
     'im ok with that',
 
     // Spanish (ES)
@@ -697,6 +716,8 @@ export const ACCEPT_PATTERNS = [
     'accepta',
     'accepta totes les cookies',
     'accepta-ho tot',
+    // FR/CA "d'accord" / "d'acord"
+    /^dac?cord( fermer)?$/is,
     'accepta-les totes',
     'acceptar galetes',
     'acceptar i tancar',
@@ -745,7 +766,6 @@ export const ACCEPT_PATTERNS = [
     'akceptuję!',
     'ok, zgadzam się',
     'potwierdzam wszystkie',
-    'przejdź do serwisu',
     'tak',
     'tak, zgadzam się na wszystkie pliki cookie',
     'tak, zgadzam się',
@@ -828,9 +848,6 @@ export const ACKNOWLEDGE_PATTERNS = [
     'vale',
     '¡vamos!',
 
-    // Catalan (CA)
-    'dacord',
-
     // Polish (PL)
     'kontynuuj',
     'ok, zrozumiałem',

package/lib/heuristics.ts

@@ -1,40 +1,10 @@
-import {
-    ACCEPT_PATTERNS,
-    ACKNOWLEDGE_PATTERNS,
-    BUTTON_NEVER_MATCH_PATTERNS,
-    DETECT_NEVER_MATCH_PATTERNS,
-    DETECT_PATTERNS,
-    REJECT_PATTERNS,
-    SETTINGS_PATTERNS,
-} from './heuristic-patterns';
-import { ButtonData, ButtonRegexClassification, HeuristicLevel, PopupClassification, PopupData } from './types';
+import { checkHeuristicPatterns, classifyButtons, classifyPopup, isExcludedPopup } from './heuristic-classify';
+import { ButtonData, HeuristicLevel, PopupClassification, PopupData } from './types';
 import { isElementVisible, isTopFrame } from './utils';
 
 const BUTTON_LIKE_ELEMENT_SELECTOR = 'button, input[type="button"], input[type="submit"], a, [role="button"], [class*="button"]';
-const TEXT_LIMIT = 100000;
 const POPUP_SEARCH_MAX_TIME = 100;
 
-export function checkHeuristicPatterns(allText: string, detectPatterns = DETECT_PATTERNS) {
-    allText = allText.slice(0, TEXT_LIMIT);
-    const patterns = [];
-    const snippets = [];
-
-    for (const p of detectPatterns) {
-        const matches = allText?.match(p);
-        if (matches) {
-            patterns.push(p.toString());
-            // When a non-global regex has capture groups, `String.prototype.match`
-            // returns undefined entries for optional groups that did not match.
-            for (const m of matches) {
-                if (typeof m === 'string') {
-                    snippets.push(m.substring(0, 200));
-                }
-            }
-        }
-    }
-    return { patterns, snippets };
-}
-
 export function getActionablePopups(mode: HeuristicLevel = 'reject', timeout = POPUP_SEARCH_MAX_TIME): PopupData[] {
     const acceptedLevels: PopupClassification[] =
         mode === 'tier2' ? ['reject', 'tier1', 'tier2'] : mode === 'tier1' ? ['reject', 'tier1'] : ['reject'];
@@ -65,98 +35,7 @@ export function getActionablePopups(mode: HeuristicLevel = 'reject', timeout = P
         .sort((a, b) => ((a.regexClassification ?? '') > (b.regexClassification ?? '') ? 1 : -1));
 }
 
-export function classifyButtons(buttons: ButtonData[]) {
-    for (const button of buttons) {
-        button.regexClassification = classifyButtonTextRegex(button.text);
-    }
-}
-
-export function isExcludedPopup(popupText: string, excludePatterns = DETECT_NEVER_MATCH_PATTERNS): boolean {
-    if (!popupText) {
-        return false;
-    }
-    const truncated = popupText.slice(0, TEXT_LIMIT);
-    return excludePatterns.some((p) => p.test(truncated));
-}
-
-function classifyPopup(buttons: ButtonData[]): PopupClassification {
-    const { reject, settings, accept, acknowledge } = buttons.reduce(
-        (acc, button) => {
-            if (button.regexClassification && button.regexClassification !== 'other') {
-                acc[button.regexClassification]++;
-            }
-            return acc;
-        },
-        { reject: 0, settings: 0, accept: 0, acknowledge: 0 },
-    );
-    if (reject > 0) {
-        return 'reject';
-    }
-    if (settings > 0) {
-        return 'none';
-    }
-    if (acknowledge > 0) {
-        return 'tier1';
-    }
-    if (accept > 0) {
-        return accept === 1 ? 'tier2' : 'none';
-    }
-    return 'none';
-}
-
-/**
- * @param {string} buttonText
- * @param {Array<string|RegExp>} matchPatterns
- * @param {Array<string|RegExp>} neverMatchPatterns
- * @returns {boolean}
- */
-function testButtonMatches(buttonText: string, matchPatterns: (string | RegExp)[], neverMatchPatterns: (string | RegExp)[]): boolean {
-    if (!buttonText) {
-        return false;
-    }
-    const cleanedButtonText = cleanButtonText(buttonText);
-    return (
-        !neverMatchPatterns.some((p) => (p instanceof RegExp && p.test(cleanedButtonText)) || p === cleanedButtonText) &&
-        matchPatterns.some((p) => (p instanceof RegExp && p.test(cleanedButtonText)) || p === cleanedButtonText)
-    );
-}
-
-export function cleanButtonText(buttonText: string): string {
-    // lowercase
-    let result = buttonText.toLowerCase();
-    // remove special characters
-    result = result.replace(/[“”"'/#&[\]→✕×⟩❯><✗ב’›«»]+/g, '');
-    // remove emojis
-    result = result.replace(
-        /[\u{1F600}-\u{1F64F}\u{1F300}-\u{1F5FF}\u{1F680}-\u{1F6FF}\u{1F1E0}-\u{1F1FF}\u2600-\u26FF\u2700-\u27BF\u{1F900}-\u{1F9FF}\u{1FA70}-\u{1FAFF}]/gu,
-        '',
-    );
-    // remove newlines
-    result = result.replace(/\n+/g, ' ');
-    // remove multiple spaces
-    result = result.replace(/\s+/g, ' ');
-    // strip whitespace around the text
-    result = result.trim();
-    return result;
-}
-
-export function classifyButtonTextRegex(buttonText: string): ButtonRegexClassification {
-    if (testButtonMatches(buttonText, REJECT_PATTERNS, BUTTON_NEVER_MATCH_PATTERNS)) {
-        return 'reject';
-    }
-    if (testButtonMatches(buttonText, SETTINGS_PATTERNS, BUTTON_NEVER_MATCH_PATTERNS)) {
-        return 'settings';
-    }
-    if (testButtonMatches(buttonText, ACCEPT_PATTERNS, BUTTON_NEVER_MATCH_PATTERNS)) {
-        return 'accept';
-    }
-    if (testButtonMatches(buttonText, ACKNOWLEDGE_PATTERNS, BUTTON_NEVER_MATCH_PATTERNS)) {
-        return 'acknowledge';
-    }
-    return 'other';
-}
-
-function getPotentialPopups(timeout = POPUP_SEARCH_MAX_TIME) {
+export function getPotentialPopups(timeout = POPUP_SEARCH_MAX_TIME): PopupData[] {
     const isFramed = !isTopFrame();
     // do not inspect frames that are more than one level deep
     if (isFramed && window.parent && window.parent !== window.top) {

package/lib/types.ts

@@ -1,6 +1,7 @@
 import { ContentScriptMessage } from './messages';
 import { AutoConsentCMPRule, ElementSelector, HideMethod, RunContext, VisibilityCheck } from './rules';
 import { CompactCMPRuleset } from './encoding';
+import type { ButtonRegexClassification, ClassifiableButton, PopupClassification } from './heuristic-classify';
 
 export type MessageSender = (message: ContentScriptMessage) => Promise<void>;
 
@@ -114,12 +115,10 @@ export type ConsentState = {
     performance?: Record<string, number[]>;
 };
 
-export type ButtonRegexClassification = 'reject' | 'settings' | 'accept' | 'acknowledge' | 'other';
+export type { ButtonRegexClassification, PopupClassification };
 
-export interface ButtonData {
-    text: string;
+export interface ButtonData extends ClassifiableButton {
     element: HTMLElement;
-    regexClassification?: ButtonRegexClassification;
 }
 
 export interface PopupData {
@@ -137,4 +136,3 @@ export interface PopupData {
  *  - tier2: Heuristic detection is enabled, and will click the accept button on a popup if it exists, and no Reject, Acknowledge, or Settings button exists.
  */
 export type HeuristicLevel = 'off' | 'reject' | 'tier1' | 'tier2';
-export type PopupClassification = 'none' | 'reject' | 'tier1' | 'tier2';

package/lib/web.ts

@@ -7,7 +7,7 @@ import { dynamicCMPs } from './cmps/all';
 import { AutoConsentCMP, AutoConsentHeuristicCMP } from './cmps/base';
 import { DomActions } from './dom-actions';
 import { isTopFrame, normalizeConfig, scheduleWhenIdle } from './utils';
-import { checkHeuristicPatterns } from './heuristics';
+import { checkHeuristicPatterns } from './heuristic-classify';
 import { decodeRules } from './encoding';
 
 // Re-export types and functions for external use

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@duckduckgo/autoconsent",
-  "version": "16.49.0",
+  "version": "16.51.0",
   "description": "",
   "types": "./dist/types/web.d.ts",
   "exports": {
@@ -9,6 +9,11 @@
       "import": "./dist/autoconsent.esm.js",
       "require": "./dist/autoconsent.cjs.js"
     },
+    "./heuristics": {
+      "types": "./dist/types/heuristic-classify.d.ts",
+      "import": "./dist/heuristics.mjs",
+      "require": "./dist/heuristics.cjs.js"
+    },
     "./rules/rules.json": "./rules/rules.json",
     "./rules/compact-rules.json": "./rules/compact-rules.json"
   },
@@ -17,7 +22,7 @@
   },
   "scripts": {
     "clean": "rm -r dist",
-    "lint": "eslint . && prettier . --check && npm run rule-syntax-check",
+    "lint": "eslint . && prettier . --check && npm run rule-syntax-check && tsc -p tsconfig.heuristics.json",
     "lint-fix": "eslint . --fix && prettier . --write && npm run rule-syntax-check",
     "bundle": "./build.sh",
     "rule-syntax-check": "node scripts/validate-json-rules.js",

package/playwright/content.ts

@@ -1,14 +1,19 @@
 import AutoConsent from '../lib/web';
 import { BackgroundMessage } from '../lib/messages';
 import { MessageSender, RuleBundle } from '../lib/types';
+import { getButtonData, getPotentialPopups } from '../lib/heuristics';
 
 declare global {
     interface Window {
         autoconsentSendMessage: MessageSender;
         autoconsentReceiveMessage: (message: BackgroundMessage) => Promise<void>;
+        autoconsentHeuristics: { getPotentialPopups: typeof getPotentialPopups; getButtonData: typeof getButtonData };
     }
 }
 
+// lets other scripts in the same world (e.g. crawler scrapers) reuse popup and button discovery
+window.autoconsentHeuristics = { getPotentialPopups, getButtonData };
+
 if (!window.autoconsentReceiveMessage) {
     const consent = new AutoConsent(window.autoconsentSendMessage, null, <RuleBundle>{ autoconsent: [] });
 

package/readme.md

@@ -32,6 +32,18 @@ In some environments (e.g. browser extensions), it's useful to access [eval snip
 import { evalSnippets } from '@duckduckgo/autoconsent';
 ```
 
+### Heuristics
+
+The heuristic popup and button classification is available on its own, without DOM dependencies, so it can run in Node (e.g. to post-process crawl data):
+
+```javascript
+import { checkHeuristicPatterns, classifyButtonTextRegex, classifyButtons, classifyPopup } from '@duckduckgo/autoconsent/heuristics';
+
+classifyButtonTextRegex('Reject all'); // 'reject'
+```
+
+`dist/autoconsent.playwright.js` also exposes `window.autoconsentHeuristics.getPotentialPopups(timeout)` and `getButtonData(element)`, so other scripts injected into the same world can reuse autoconsent's popup and button discovery.
+
 ## Browser extension
 
 Autoconsent comes with a reference extension implementation. It is not published in stores since the feature is available in all [DuckDuckGo apps](https://duckduckgo.com/app), but you can build it yourself and use for testing.