@duckduckgo/autoconsent 1.0.6 → 2.0.0

package/lib/types.ts

@@ -0,0 +1,35 @@
+import { ContentScriptMessage } from "./messages";
+import { ConsentOMaticConfig } from "./cmps/consentomatic";
+import { AutoConsentCMPRule, RunContext } from "./rules";
+
+export type MessageSender = (message: ContentScriptMessage) => Promise<void>;
+
+export interface AutoCMP {
+  name: string
+  hasSelfTest: boolean
+  isIntermediate: boolean;
+  prehideSelectors?: string[];
+  runContext: RunContext;
+  checkRunContext(): boolean;
+  detectCmp(): Promise<boolean>
+  detectPopup(): Promise<boolean>
+  optOut(): Promise<boolean>
+  optIn(): Promise<boolean>
+  openCmp(): Promise<boolean>
+  test(): Promise<boolean>
+}
+
+export type RuleBundle = {
+  autoconsent: AutoConsentCMPRule[];
+  consentomatic: { [name: string]: ConsentOMaticConfig };
+};
+
+export type AutoAction = 'optOut' | 'optIn' | null;
+
+export type Config = {
+  enabled: boolean;
+  autoAction: AutoAction;
+  disabledCmps: string[];
+  enablePrehide: boolean;
+  detectRetries: number;
+}

package/lib/utils.ts

@@ -0,0 +1,64 @@
+import { HideMethod } from "./rules";
+
+// get or create a style container for CSS overrides
+export function getStyleElement(styleOverrideElementId = "autoconsent-css-rules"): HTMLStyleElement {
+  const styleSelector = `style#${styleOverrideElementId}`;
+  const existingElement = document.querySelector(styleSelector);
+  if (existingElement && existingElement instanceof HTMLStyleElement) {
+    return existingElement;
+  } else {
+    const parent =
+      document.head ||
+      document.getElementsByTagName("head")[0] ||
+      document.documentElement;
+    const css = document.createElement("style");
+    css.id = styleOverrideElementId;
+    parent.appendChild(css);
+    return css;
+  }
+}
+
+// hide elements with a CSS rule
+export function hideElements(
+  styleEl: HTMLStyleElement,
+  selectors: string[],
+  method: HideMethod = 'display',
+): boolean {
+  const hidingSnippet = method === "opacity" ? `opacity: 0` : `display: none`; // use display by default
+  const rule = `${selectors.join(
+    ","
+  )} { ${hidingSnippet} !important; z-index: -1 !important; pointer-events: none !important; } `;
+
+  if (styleEl instanceof HTMLStyleElement) {
+    styleEl.innerText += rule;
+    return selectors.length > 0;
+  }
+  return false;
+}
+
+export async function waitFor(predicate: () => Promise<boolean> | boolean, maxTimes: number, interval: number): Promise<boolean> {
+  const result = await predicate();
+  if (!result && maxTimes > 0) {
+    return new Promise((resolve) => {
+      setTimeout(async () => {
+        resolve(waitFor(predicate, maxTimes - 1, interval));
+      }, interval);
+    });
+  }
+  return Promise.resolve(result);
+}
+
+export function isElementVisible(elem: HTMLElement): boolean {
+  if (!elem) {
+    return false;
+  }
+  if (elem.offsetParent !== null) {
+    return true;
+  } else {
+    const css = window.getComputedStyle(elem);
+    if (css.position === 'fixed' && css.display !== "none") { // fixed elements may be visible even if the parent is not
+      return true;
+    }
+  }
+  return false;
+}

package/lib/web.ts

@@ -1,35 +1,89 @@
-import Tab from './web/tab';
-import handleContentMessage from './web/content';
-import TabConsent from './tabwrapper';
-import detectDialog from './detector';
-import { rules, createAutoCMP } from './index';
-import { Browser, MessageSender, AutoCMP, TabActor } from './types';
-import { ConsentOMaticCMP, ConsentOMaticConfig } from './consentomatic/index';
+import { rules as dynamicRules, createAutoCMP } from './index';
+import { MessageSender, AutoCMP, RuleBundle, Config } from './types';
+import { ConsentOMaticCMP, ConsentOMaticConfig } from './cmps/consentomatic';
 import { AutoConsentCMPRule } from './rules';
-import prehideElements from './hider';
 import { enableLogs } from './config';
+import { BackgroundMessage, InitMessage } from './messages';
+import { prehide, undoPrehide } from './rule-executors';
+import { evalState, resolveEval } from './eval-handler';
 
 export * from './index';
-export {
-  Tab,
-  handleContentMessage,
-}
 
 export default class AutoConsent {
-  consentFrames: Map<number, any> = new Map()
-  tabCmps: Map<number, TabConsent> = new Map()
-  rules: AutoCMP[]
+  rules: AutoCMP[] = [];
+  config: Config;
+  foundCmp: AutoCMP = null;
+  protected sendContentMessage: MessageSender;
 
-  constructor(protected browser: Browser, protected sendContentMessage: MessageSender) {
+  constructor(sendContentMessage: MessageSender, config: Config = null, declarativeRules: RuleBundle = null) {
+    evalState.sendContentMessage = sendContentMessage;
     this.sendContentMessage = sendContentMessage;
-    this.rules = [...rules];
+    this.rules = [...dynamicRules];
+
+    enableLogs && console.log('autoconsent init', window.location.href);
+    if (config) {
+      this.initialize(config, declarativeRules);
+    } else {
+      const initMsg: InitMessage = {
+        type: "init",
+        url: window.location.href,
+      };
+      sendContentMessage(initMsg);
+    }
+  }
+
+  initialize(config: Config, declarativeRules: RuleBundle) {
+    this.config = config;
+    if (!config.enabled) {
+      enableLogs && console.log("autoconsent is disabled");
+      return;
+    }
+
+    this.parseRules(declarativeRules);
+    if (config.disabledCmps?.length > 0) {
+      this.disableCMPs(config.disabledCmps);
+    }
+
+    if (config.enablePrehide) {
+      if (document.documentElement) {
+        this.prehideElements(); // prehide as early as possible to prevent flickering
+      } else {
+        // we're injected really early
+        const delayedPrehide = () => {
+          window.removeEventListener('DOMContentLoaded', delayedPrehide);
+          this.prehideElements();
+        }
+        window.addEventListener('DOMContentLoaded', delayedPrehide);
+      }
+    }
+
+    // start detection
+    if (document.readyState === 'loading') {
+      const onReady = () => {
+        window.removeEventListener('DOMContentLoaded', onReady);
+        this.start();
+      }
+      window.addEventListener('DOMContentLoaded', onReady);
+    } else {
+      this.start();
+    }
+  }
+
+  parseRules(declarativeRules: RuleBundle) {
+    Object.keys(declarativeRules.consentomatic).forEach((name) => {
+      this.addConsentomaticCMP(name, declarativeRules.consentomatic[name]);
+    });
+    declarativeRules.autoconsent.forEach((rule) => {
+      this.addCMP(rule);
+    });
+    enableLogs && console.log("added rules", this.rules);
   }
 
   addCMP(config: AutoConsentCMPRule) {
     this.rules.push(createAutoCMP(config));
   }
 
-  disableCMPs(cmpNames: String[]) {
+  disableCMPs(cmpNames: string[]) {
     this.rules = this.rules.filter((cmp) => !cmpNames.includes(cmp.name))
   }
 
@@ -37,78 +91,233 @@ export default class AutoConsent {
     this.rules.push(new ConsentOMaticCMP(`com_${name}`, config));
   }
 
-  createTab(tabId: number) {
-    return new Tab(tabId,
-      this.consentFrames.get(tabId),
-      this.sendContentMessage,
-      this.browser);
+  // start the detection process, possibly with a delay
+  start() {
+    if (window.requestIdleCallback) {
+      window.requestIdleCallback(() => this._start(), { timeout: 500 });
+    } else {
+      this._start();
+    }
   }
 
-  async checkTab(tabId: number, prehide = true) {
-    enableLogs && console.log('checking tab', tabId, this.consentFrames, this.tabCmps);
-    const tab = this.createTab(tabId);
-    if (prehide) {
-      this.prehideElements(tab);
-    }
-    const consent = new TabConsent(tab, this.detectDialog(tab, 20));
-    this.tabCmps.set(tabId, consent);
-    // check tabs
-    consent.checked.then((rule) => {
-      if (this.consentFrames.has(tabId) && rule) {
-        const frame = this.consentFrames.get(tabId);
-        enableLogs && console.log(`Found ${rule.name} in a nested iframe ${frame.id} inside tab ${tabId}`);
-        if (frame.type === rule.name) {
-          consent.tab.frame = frame;
+  async _start() {
+    enableLogs && console.log(`Detecting CMPs on ${window.location.href}`)
+    const cmp = await this.findCmp(this.config.detectRetries);
+    if (cmp) {
+      enableLogs && console.log("detected CMP:", cmp.name, window.location.href);
+      this.sendContentMessage({
+        type: 'cmpDetected',
+        url: location.href,
+        cmp: cmp.name,
+      }); // notify the browser
+      const isOpen = await this.waitForPopup(cmp);
+      if (!isOpen) {
+        enableLogs && console.log('no popup found');
+        if (this.config.enablePrehide) {
+          undoPrehide();
         }
+        return false;
       }
-      enableLogs && console.log('finished checking tab', tabId, this.consentFrames, this.tabCmps);
-      // no CMP detected, undo hiding
-      if (!rule && prehide) {
-        enableLogs && console.log('no CMP detected, undo hiding');
-        tab.undoHideElements();
+
+      this.foundCmp = cmp;
+      this.sendContentMessage({
+        type: 'popupFound',
+        cmp: cmp.name,
+        url: location.href,
+      }); // notify the browser
+
+      if (this.config.autoAction === 'optOut') {
+        return await this.doOptOut();
+      } else if (this.config.autoAction === 'optIn') {
+        return await this.doOptIn();
+      }
+
+      enableLogs && console.log("waiting for opt-out signal...", location.href);
+      return true;
+    } else {
+      enableLogs && console.log("no CMP found", location.href);
+      if (this.config.enablePrehide) {
+        undoPrehide();
       }
+      return false;
+    }
+  }
+
+  async findCmp(retries: number): Promise<AutoCMP> {
+    let foundCmp: AutoCMP = null;
+    const allFoundCmps: AutoCMP[] = [];
+
+    for (const cmp of this.rules) {
+      try {
+        if (!cmp.checkRunContext()) {
+          continue;
+        }
+        const result = await cmp.detectCmp();
+        if (result) {
+          enableLogs && console.log(`Found CMP: ${cmp.name}`);
+          allFoundCmps.push(cmp);
+          if (!foundCmp) {
+            foundCmp = cmp;
+          }
+        }
+      } catch (e) {
+        enableLogs && console.warn(`error detecting ${cmp.name}`, e);
+      }
+    }
+
+    if (allFoundCmps.length > 1) {
+      const errorDetails = {
+        msg: `Found multiple CMPs, check the detection rules.`,
+        cmps: allFoundCmps.map((cmp) => cmp.name),
+      };
+      enableLogs && console.warn(errorDetails.msg, errorDetails.cmps);
+      this.sendContentMessage({
+        type: 'autoconsentError',
+        details: errorDetails,
+      });
+    }
+
+    if (!foundCmp && retries > 0) {
+      return new Promise((resolve) => {
+        setTimeout(async () => {
+          const result = this.findCmp(retries - 1);
+          resolve(result);
+        }, 500);
+      });
+    }
+
+    return foundCmp;
+  }
+
+  async doOptOut(): Promise<boolean> {
+    let optOutResult;
+    if (!this.foundCmp) {
+      enableLogs && console.log('no CMP to opt out');
+      optOutResult = false;
+    } else {
+      enableLogs && console.log(`CMP ${this.foundCmp.name}: opt out on ${window.location.href}`);
+      optOutResult = await this.foundCmp.optOut();
+      enableLogs && console.log(`${this.foundCmp.name}: opt out result ${optOutResult}`);
+    }
+
+    if (this.config.enablePrehide) {
+      undoPrehide();
+    }
+
+    this.sendContentMessage({
+      type: 'optOutResult',
+      cmp: this.foundCmp ? this.foundCmp.name : 'none',
+      result: optOutResult,
+      scheduleSelfTest: this.foundCmp && this.foundCmp.hasSelfTest,
+      url: location.href,
     });
 
-    return this.tabCmps.get(tabId);
+    if (optOutResult && !this.foundCmp.isIntermediate) {
+      this.sendContentMessage({
+        type: 'autoconsentDone',
+        cmp: this.foundCmp.name,
+        url: location.href,
+      });
+    }
+
+    return optOutResult;
   }
 
-  removeTab(tabId: number) {
-    this.tabCmps.delete(tabId);
-    this.consentFrames.delete(tabId);
+  async doOptIn(): Promise<boolean> {
+    let optInResult;
+    if (!this.foundCmp) {
+      enableLogs && console.log('no CMP to opt in');
+      optInResult = false;
+    } else {
+      enableLogs && console.log(`CMP ${this.foundCmp.name}: opt in on ${window.location.href}`);
+      optInResult = await this.foundCmp.optIn();
+    }
+
+    if (this.config.enablePrehide) {
+      undoPrehide();
+    }
+
+    this.sendContentMessage({
+      type: 'optInResult',
+      cmp: this.foundCmp ? this.foundCmp.name : 'none',
+      result: optInResult,
+      scheduleSelfTest: this.foundCmp && this.foundCmp.hasSelfTest,
+      url: location.href,
+    });
+
+    if (optInResult && !this.foundCmp.isIntermediate) {
+      this.sendContentMessage({
+        type: 'autoconsentDone',
+        cmp: this.foundCmp.name,
+        url: location.href,
+      });
+    }
+
+    return optInResult;
   }
 
-  onFrame({ tabId, url, frameId }: { tabId: number, url: string, frameId: number }) {
-    // ignore main frames
-    if (frameId === 0) {
-      return;
+  async doSelfTest(): Promise<boolean> {
+    let selfTestResult;
+    if (!this.foundCmp) {
+      enableLogs && console.log('no CMP to self test');
+      selfTestResult = false;
+    } else {
+      enableLogs && console.log(`CMP ${this.foundCmp.name}: self-test on ${window.location.href}`);
+      selfTestResult = await this.foundCmp.test();
     }
-    try {
-      const frame = {
-        id: frameId,
-        url: url,
-      };
-      const tab = this.createTab(tabId);
-      const frameMatch = this.rules.findIndex(r => r.detectFrame(tab, frame));
-      if (frameMatch > -1) {
-        this.consentFrames.set(tabId, {
-          type: this.rules[frameMatch].name,
-          url,
-          id: frameId,
-        });
-        if (this.tabCmps.has(tabId)) {
-          this.tabCmps.get(tabId).tab.frame = this.consentFrames.get(tabId);
-        }
-      }
-    } catch (e) {
-      console.error(e);
+
+    this.sendContentMessage({
+      type: 'selfTestResult',
+      cmp: this.foundCmp ? this.foundCmp.name : 'none',
+      result: selfTestResult,
+      url: location.href,
+    });
+    return selfTestResult;
+  }
+
+  async waitForPopup(cmp: AutoCMP, retries = 5, interval = 500): Promise<boolean> {
+    enableLogs && console.log('checking if popup is open...', cmp.name);
+    const isOpen = await cmp.detectPopup();
+    if (!isOpen && retries > 0) {
+      return new Promise((resolve) => setTimeout(() => resolve(this.waitForPopup(cmp, retries - 1, interval)), interval));
     }
+    enableLogs && console.log(`popup is ${isOpen ? 'open' : 'not open'}`);
+    return isOpen;
   }
 
-  async detectDialog(tab: TabActor, retries: number): Promise<AutoCMP> {
-    return detectDialog(tab, retries, this.rules);
+  prehideElements(): boolean {
+    // hide rules not specific to a single CMP rule
+    const globalHidden = [
+      "#didomi-popup,.didomi-popup-container,.didomi-popup-notice,.didomi-consent-popup-preferences,#didomi-notice,.didomi-popup-backdrop,.didomi-screen-medium",
+    ]
+
+    const selectors = this.rules.reduce((selectorList, rule) => {
+      if (rule.prehideSelectors) {
+        return [...selectorList, ...rule.prehideSelectors];
+      }
+      return selectorList;
+    }, globalHidden);
+
+    return prehide(selectors);
   }
 
-  async prehideElements(tab: TabActor): Promise<void> {
-    return prehideElements(tab, this.rules);
+  async receiveMessageCallback(message: BackgroundMessage) {
+    if (enableLogs && message.type !== 'evalResp' /* evals are noisy */) {
+      console.log('received from background', message, window.location.href);
+    }
+    switch (message.type) {
+      case 'initResp':
+        this.initialize(message.config, message.rules);
+        break;
+      case 'optOut':
+        await this.doOptOut();
+        break;
+      case 'selfTest':
+        await this.doSelfTest();
+        break;
+      case 'evalResp':
+        resolveEval(message.id, message.result);
+        break;
+    }
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@duckduckgo/autoconsent",
-  "version": "1.0.6",
+  "version": "2.0.0",
   "description": "",
   "main": "dist/autoconsent.cjs.js",
   "module": "dist/autoconsent.esm.js",
@@ -8,34 +8,37 @@
     "lib": "lib"
   },
   "scripts": {
-    "clean": "rm -r dist lib/**/*.js lib/*.js",
-    "build": "tsc",
+    "clean": "rm -r dist lib/**/*.js lib/*.js addon/*.js",
     "bundle": "rollup -c",
     "watch": "rollup -c -w",
-    "start": "web-ext run -s ./addon --firefox firefoxdeveloperedition",
     "test": "playwright test",
     "test:webkit": "playwright test --project webkit",
     "test:firefox": "playwright test --project firefox",
+    "test:chrome": "playwright test --project chrome",
     "fetch-fanboy-list": "wget https://www.fanboy.co.nz/fanboy-cookiemonster.txt",
     "fetch-site-list": "curl https://s3.amazonaws.com/cdn.cliqz.com/re-consent/supported_sites.txt > sites.txt",
     "build-rules": "node rules/build.js && cp rules/rules.json addon/",
     "version": "node update_version.js && git add addon/manifest.json",
     "vendor-copy": "mkdir -p addon/vendor && cp node_modules/mocha/mocha.* addon/vendor/ && cp node_modules/chai/chai.js addon/vendor/",
-    "prepublish": "npm run build-rules && npm run build && npm run bundle"
+    "prepublish": "npm run build-rules && npm run bundle"
   },
   "author": "Sam Macbeth",
   "license": "MPL-2.0",
-  "dependencies": {},
   "devDependencies": {
     "@playwright/test": "^1.17.1",
-    "@rollup/plugin-typescript": "^4.0.0",
-    "@types/chai": "^4.2.11",
-    "@types/mocha": "^8.0.0",
+    "@rollup/plugin-json": "^4.1.0",
+    "@rollup/plugin-typescript": "^8.3.2",
+    "@types/chai": "^4.3.1",
+    "@types/chrome": "^0.0.188",
+    "@types/mocha": "^9.1.1",
+    "@typescript-eslint/eslint-plugin": "^5.25.0",
+    "@typescript-eslint/parser": "^5.25.0",
     "chai": "^4.2.0",
-    "mocha": "^8.0.1",
-    "rollup": "^1.32.1",
-    "typescript": "^3.8.3",
-    "web-ext": "^4.2.0",
-    "web-ext-types": "^3.2.1"
+    "eslint-config-airbnb": "^19.0.4",
+    "mocha": "^10.0.0",
+    "rollup": "^2.73.0",
+    "rollup-plugin-terser": "^7.0.2",
+    "tslib": "^2.4.0",
+    "typescript": "^4.6.4"
   }
 }

package/playwright/content.ts

@@ -0,0 +1,27 @@
+import AutoConsent from "../lib/web";
+import { BackgroundMessage } from "../lib/messages";
+import { MessageSender, RuleBundle } from "../lib/types";
+import * as rules from '../rules/rules.json';
+
+declare global {
+  interface Window {
+    autoconsentSendMessage: MessageSender;
+    autoconsentReceiveMessage: (message: BackgroundMessage) => Promise<void>;
+  }
+}
+
+if (!window.autoconsentReceiveMessage) {
+  const consent = new AutoConsent(window.autoconsentSendMessage, {
+    enabled: true,
+    autoAction: 'optOut',
+    disabledCmps: [],
+    enablePrehide: true,
+    detectRetries: 20,
+  }, <RuleBundle>rules);
+
+  window.autoconsentReceiveMessage = (message: BackgroundMessage) => {
+    return Promise.resolve(consent.receiveMessageCallback(message));
+  };
+} else {
+  console.warn('autoconsent already initialized', window.autoconsentReceiveMessage);
+}