@duckduckgo/autoconsent 1.0.2

package/lib/web.js

@@ -0,0 +1,90 @@
+import Tab from './web/tab';
+import handleContentMessage from './web/content';
+import TabConsent from './tabwrapper';
+import detectDialog from './detector';
+import { rules, createAutoCMP } from './index';
+import { ConsentOMaticCMP } from './consentomatic/index';
+import prehideElements from './hider';
+export * from './index';
+export { Tab, handleContentMessage, };
+export default class AutoConsent {
+    constructor(browser, sendContentMessage) {
+        this.browser = browser;
+        this.sendContentMessage = sendContentMessage;
+        this.consentFrames = new Map();
+        this.tabCmps = new Map();
+        this.sendContentMessage = sendContentMessage;
+        this.rules = [...rules];
+    }
+    addCMP(config) {
+        this.rules.push(createAutoCMP(config));
+    }
+    disableCMPs(cmpNames) {
+        this.rules = this.rules.filter((cmp) => !cmpNames.includes(cmp.name));
+    }
+    addConsentomaticCMP(name, config) {
+        this.rules.push(new ConsentOMaticCMP(`com_${name}`, config));
+    }
+    createTab(tabId) {
+        return new Tab(tabId, this.consentFrames.get(tabId), this.sendContentMessage, this.browser);
+    }
+    async checkTab(tabId, prehide = true) {
+        const tab = this.createTab(tabId);
+        if (prehide) {
+            this.prehideElements(tab);
+        }
+        const consent = new TabConsent(tab, this.detectDialog(tab, 20));
+        this.tabCmps.set(tabId, consent);
+        // check tabs
+        consent.checked.then((rule) => {
+            if (this.consentFrames.has(tabId) && rule) {
+                const frame = this.consentFrames.get(tabId);
+                if (frame.type === rule.name) {
+                    consent.tab.frame = frame;
+                }
+            }
+            // no CMP detected, undo hiding
+            if (!rule && prehide) {
+                tab.undoHideElements();
+            }
+        });
+        return this.tabCmps.get(tabId);
+    }
+    removeTab(tabId) {
+        this.tabCmps.delete(tabId);
+        this.consentFrames.delete(tabId);
+    }
+    onFrame({ tabId, url, frameId }) {
+        // ignore main frames
+        if (frameId === 0) {
+            return;
+        }
+        try {
+            const frame = {
+                id: frameId,
+                url: url,
+            };
+            const tab = this.createTab(tabId);
+            const frameMatch = this.rules.findIndex(r => r.detectFrame(tab, frame));
+            if (frameMatch > -1) {
+                this.consentFrames.set(tabId, {
+                    type: this.rules[frameMatch].name,
+                    url,
+                    id: frameId,
+                });
+                if (this.tabCmps.has(tabId)) {
+                    this.tabCmps.get(tabId).tab.frame = this.consentFrames.get(tabId);
+                }
+            }
+        }
+        catch (e) {
+            console.error(e);
+        }
+    }
+    async detectDialog(tab, retries) {
+        return detectDialog(tab, retries, this.rules);
+    }
+    async prehideElements(tab) {
+        return prehideElements(tab, this.rules);
+    }
+}

package/lib/web.ts

@@ -0,0 +1,109 @@
+import Tab from './web/tab';
+import handleContentMessage from './web/content';
+import TabConsent from './tabwrapper';
+import detectDialog from './detector';
+import { rules, createAutoCMP } from './index';
+import { Browser, MessageSender, AutoCMP, TabActor } from './types';
+import { ConsentOMaticCMP, ConsentOMaticConfig } from './consentomatic/index';
+import { AutoConsentCMPRule } from './rules';
+import prehideElements from './hider';
+
+export * from './index';
+export {
+  Tab,
+  handleContentMessage,
+}
+
+export default class AutoConsent {
+  consentFrames: Map<number, any> = new Map()
+  tabCmps: Map<number, TabConsent> = new Map()
+  rules: AutoCMP[]
+
+  constructor(protected browser: Browser, protected sendContentMessage: MessageSender) {
+    this.sendContentMessage = sendContentMessage;
+    this.rules = [...rules];
+  }
+
+  addCMP(config: AutoConsentCMPRule) {
+    this.rules.push(createAutoCMP(config));
+  }
+
+  disableCMPs(cmpNames: String[]) {
+    this.rules = this.rules.filter((cmp) => !cmpNames.includes(cmp.name))
+  }
+
+  addConsentomaticCMP(name: string, config: ConsentOMaticConfig) {
+    this.rules.push(new ConsentOMaticCMP(`com_${name}`, config));
+  }
+
+  createTab(tabId: number) {
+    return new Tab(tabId,
+      this.consentFrames.get(tabId),
+      this.sendContentMessage,
+      this.browser);
+  }
+
+  async checkTab(tabId: number, prehide = true) {
+    const tab = this.createTab(tabId);
+    if (prehide) {
+      this.prehideElements(tab);
+    }
+    const consent = new TabConsent(tab, this.detectDialog(tab, 20));
+    this.tabCmps.set(tabId, consent);
+    // check tabs
+    consent.checked.then((rule) => {
+      if (this.consentFrames.has(tabId) && rule) {
+        const frame = this.consentFrames.get(tabId);
+        if (frame.type === rule.name) {
+          consent.tab.frame = frame;
+        }
+      }
+      // no CMP detected, undo hiding
+      if (!rule && prehide) {
+        tab.undoHideElements();
+      }
+    });
+
+    return this.tabCmps.get(tabId);
+  }
+
+  removeTab(tabId: number) {
+    this.tabCmps.delete(tabId);
+    this.consentFrames.delete(tabId);
+  }
+
+  onFrame({ tabId, url, frameId }: { tabId: number, url: string, frameId: number }) {
+    // ignore main frames
+    if (frameId === 0) {
+      return;
+    }
+    try {
+      const frame = {
+        id: frameId,
+        url: url,
+      };
+      const tab = this.createTab(tabId);
+      const frameMatch = this.rules.findIndex(r => r.detectFrame(tab, frame));
+      if (frameMatch > -1) {
+        this.consentFrames.set(tabId, {
+          type: this.rules[frameMatch].name,
+          url,
+          id: frameId,
+        });
+        if (this.tabCmps.has(tabId)) {
+          this.tabCmps.get(tabId).tab.frame = this.consentFrames.get(tabId);
+        }
+      }
+    } catch (e) {
+      console.error(e);
+    }
+  }
+
+  async detectDialog(tab: TabActor, retries: number): Promise<AutoCMP> {
+    return detectDialog(tab, retries, this.rules);
+  }
+
+  async prehideElements(tab: TabActor): Promise<void> {
+    return prehideElements(tab, this.rules);
+  }
+}

package/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "@duckduckgo/autoconsent",
+  "version": "1.0.2",
+  "description": "",
+  "main": "dist/autoconsent.cjs.js",
+  "module": "dist/autoconsent.esm.js",
+  "directories": {
+    "lib": "lib"
+  },
+  "scripts": {
+    "clean": "rm -r dist lib/**/*.js lib/*.js",
+    "build": "tsc",
+    "bundle": "rollup -c",
+    "watch": "rollup -c -w",
+    "start": "web-ext run -s ./addon --firefox firefoxdeveloperedition",
+    "test": "playwright test",
+    "test:webkit": "playwright test --project webkit",
+    "test:firefox": "playwright test --project firefox",
+    "fetch-fanboy-list": "wget https://www.fanboy.co.nz/fanboy-cookiemonster.txt",
+    "fetch-site-list": "curl https://s3.amazonaws.com/cdn.cliqz.com/re-consent/supported_sites.txt > sites.txt",
+    "build-rules": "node rules/build.js && cp rules/rules.json addon/",
+    "version": "node update_version.js && git add addon/manifest.json",
+    "vendor-copy": "mkdir -p addon/vendor && cp node_modules/mocha/mocha.* addon/vendor/ && cp node_modules/chai/chai.js addon/vendor/",
+    "prepublish": "npm run build-rules && npm run build && npm run bundle"
+  },
+  "author": "Sam Macbeth",
+  "license": "MPL-2.0",
+  "dependencies": {},
+  "devDependencies": {
+    "@playwright/test": "^1.17.1",
+    "@rollup/plugin-typescript": "^4.0.0",
+    "@types/chai": "^4.2.11",
+    "@types/mocha": "^8.0.0",
+    "chai": "^4.2.0",
+    "mocha": "^8.0.1",
+    "rollup": "^1.32.1",
+    "typescript": "^3.8.3",
+    "web-ext": "^4.2.0",
+    "web-ext-types": "^3.2.1"
+  }
+}

package/playwright.config.ts

@@ -0,0 +1,31 @@
+import { PlaywrightTestConfig, devices } from '@playwright/test';
+
+const proxy = process.env.PROXY_SERVER ? { server: process.env.PROXY_SERVER } : undefined
+
+const config: PlaywrightTestConfig = {
+  forbidOnly: !!process.env.CI,
+  retries: process.env.CI ? 2 : 0,
+  testDir: 'tests',
+  use: {
+    trace: 'on-first-retry',
+  },
+  projects: [
+    {
+      name: 'webkit',
+      use: {
+        ...devices['Desktop Safari'],
+        proxy,
+        screenshot: 'only-on-failure',
+        userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15',
+      },
+    },
+    {
+      name: 'firefox',
+      use: {
+        ...devices['Desktop Firefox'],
+        proxy,
+      },
+    }
+  ],
+};
+export default config;

package/readme.md

@@ -0,0 +1,151 @@
+## Autoconsent
+
+This is a library of rules for navigating through common consent popups on the web. These rules
+can be run in a Firefox webextension, or in a puppeteer orchestrated headless browser. Using
+these rules, opt-in and opt-out options can be selected automatically, without requiring
+user-input.
+
+### Install Standalone
+
+The standalone addon can be built with the following steps:
+
+```bash
+# Download dependencies
+npm ci
+# Build JS bundles
+npm run bundle
+# Build consent ruleset
+npm run build-rules
+```
+
+The standalone addon can be found in the `addon` directory and can be run with `npm start`.
+Alternatively, you can use `web-ext build -s addon/` to generate a packaged addon that can
+be installed in an existing Firefox profile.
+
+### Rules
+
+The library's functionality is implemented as a set of rules that define how to manage consent on
+a subset of sites. These generally correspond to specific Consent Management Providers (CMPs)
+that are installed on multiple sites. Each CMP ruleset defines:
+
+ * If the site is using that CMP
+ * If a popup is displayed
+ * Steps to specify an 'opt-in' or 'opt-out' consent for the CMP.
+ * Optionally, a test if the consent was correctly applied.
+
+There are currently three ways of implementing a CMP:
+ 
+ 1. As a [JSON ruleset](./rules/autoconsent/), intepreted by the `AutoConsent` class. 
+ 1. As a class implementing the `AutoCMP` interface. This enables more complex logic than the linear AutoConsent
+ rulesets allow.
+ 3. As a [Consent-O-Matic](https://github.com/cavi-au/Consent-O-Matic) rule. The `ConsentOMaticCMP` class implements
+ compability with rules written for the Consent-O-Matic extension.
+
+### Rule Syntax
+
+An autoconsent CMP rule can be written as either:
+ * a class implementing the `AutoCMP` interface, or
+ * a JSON file adhering to the `AutoConsentCMPRule` type.
+
+In most cases the JSON syntax should be sufficient, unless non-linear logic is required, in which case a class is required.
+
+Both JSON and class implementations require 5 main components:
+ * `name` - to identify this CMP.
+ * `detectCMP` - which determines if this CMP is included on the page.
+ * `detectPopup` - which determines if a popup is being shown by the CMP.
+ * `optOut` - executes actions to do an 'opt-out' from the popup screen. i.e. denying all consents possible.
+ * `optIn` - execut actions for an 'opt-in' from the popup screen.
+
+Except for `name` this are defined as a set of checks or actions on the page. In the JSON syntax this is a list of `AutoConsentRuleStep` objects. For `detect` checks, we return true for the check if all steps return true. For opt in and out, we execute actions in order, exiting if one fails. The following checks/actions are supported:
+
+#### Element exists
+
+```json
+{
+  "exists": "selector"
+}
+```
+Returns true if `document.querySelect(selector)` returns elements.
+
+#### Element visible
+
+```json
+{
+  "visible": "selector",
+  "check": "any" | "all" | "none"
+}
+```
+Returns true if an element returned from `document.querySelect(selector)` is current visible on the page. If `check` is `all`, every element must be visible. If `check` is `none`, no element should be visible.
+
+#### Eval
+
+```json
+{
+  "eval": "code"
+}
+```
+Evaluates `code` in the context of the page and returns the truthiness of the result.
+
+#### Wait for element
+
+```json
+{
+  "waitFor": "selector",
+  "timeout": 1000
+}
+```
+Waits until `selector` exists in the page. After `timeout` ms the step fails.
+
+#### Click and element
+```json
+{
+  "click": "selector",
+  "all": true | false,
+}
+```
+Click on an element returned by `selector`. If `all` is `true`, all matching elements are clicked.
+
+#### Wait for then click
+```json
+{
+  "waitForThenClick": "selector",
+  "timeout": 1000
+}
+```
+Combines `waitFor` and `click`.
+
+#### Wait
+```json
+{
+  "wait": 1000,
+}
+```
+Wait for the specified number of milliseconds.
+
+#### Go to URL
+```json
+{
+  "goto": "url"
+}
+```
+Navigate the page to the given URL.
+
+#### Hide rule
+```json
+{
+  "hide": ["selector", ...]
+}
+```
+Set the elements matched by the selectors to `display: none`.
+
+#### Frames
+
+In some cases, rules have to interact with `iframes` in the page. The CMP rule defintion can optionally include a `frame` component that should be the _prefix_ of the expected frame URL. Checks and actions can then add `"frame": true` to indicate that the check or action should be done on the iframe's document (rather than main frame).
+
+#### Optional actions
+
+Any rule can include the `"optional": true` to ignore failure. 
+
+### License
+
+MPLv2.

package/rollup.config.js

@@ -0,0 +1,54 @@
+import typescript from '@rollup/plugin-typescript';
+import pkg from './package.json';
+
+export default [{
+  input: './lib/node.ts',
+  output: [{
+    file: 'dist/autoconsent.puppet.js',
+    format: 'cjs'
+  }],
+  plugins: [
+    typescript(),
+  ]
+}, {
+  input: './lib/web.ts',
+  output: [{
+    file: pkg.module,
+    format: 'es',
+    globals: ['browser'],
+  }, {
+    file: pkg.main,
+    format: 'cjs',
+  }],
+  plugins: [
+    typescript(),
+  ]
+}, {
+  input: './addon/background.ts',
+  output: [{
+    file: './addon/background.bundle.js',
+    format: 'iife',
+  }],
+  plugins: [
+    typescript(),
+  ]
+}, {
+  input: './addon/content.ts',
+  output: [{
+    file: './addon/content.bundle.js',
+    format: 'iife',
+  }],
+  plugins: [
+    typescript(),
+  ]
+},  {
+  input: './addon/test.ts',
+  output: [{
+    file: './addon/test.bundle.js',
+    format: 'iife',
+    external: ['chai', 'mocha']
+  }],
+  plugins: [
+    typescript(),
+  ]
+}];

package/rules/autoconsent/asus.json

@@ -0,0 +1,7 @@
+{
+  "name": "asus",
+  "detectCmp": [{ "exists": "#cookie-policy-info" }],
+  "detectPopup": [{ "visible": "#cookie-policy-info" }],
+  "optIn": [{ "click": ".btn-read-ck" }],
+  "optOut": [{ "click": ".btn-setting" }, { "click": ".btn-save" }]
+}

package/rules/autoconsent/cc-banner.json

@@ -0,0 +1,9 @@
+{
+  "name": "cc_banner",
+  "prehideSelectors": [".cc_banner-wrapper"],
+  "isHidingRule": true,
+  "detectCmp": [{ "exists": ".cc_banner-wrapper" }],
+  "detectPopup": [{ "visible": ".cc_banner" }],
+  "optIn": [{ "click": ".cc_btn_accept_all" }],
+  "optOut": [{ "hide": [".cc_banner-wrapper"] }]
+}

package/rules/autoconsent/cookie-law-info.json

@@ -0,0 +1,14 @@
+{
+  "name": "cookie-law-info",
+  "prehideSelectors": ["#cookie-law-info-bar"],
+  "detectCmp": [{ "exists": "#cookie-law-info-bar" }],
+  "detectPopup": [{ "visible": "#cookie-law-info-bar" }],
+  "optIn": [{ "click": "[data-cli_action=\"accept\"]" }],
+  "optOut": [
+    { "hide": ["#cookie-law-info-bar"] },
+    {
+      "eval": "CLI.disableAllCookies() || CLI.reject_close() || true"
+    }
+  ],
+  "test": [{ "eval": "document.cookie.indexOf('cookielawinfo-checkbox-non-necessary=yes') === -1" }]
+}

package/rules/autoconsent/cookie-notice.json

@@ -0,0 +1,9 @@
+{
+  "name": "cookie-notice",
+  "prehideSelectors": ["#cookie-notice"],
+  "isHidingRule": true,
+  "detectCmp": [{ "exists": "#cookie-notice" }],
+  "detectPopup": [{ "visible": "#cookie-notice" }],
+  "optIn": [{ "hide": ["#cn-accept-cookie"] }],
+  "optOut": [{ "hide": ["#cookie-notice"] }]
+}

package/rules/autoconsent/cookieconsent.json

@@ -0,0 +1,9 @@
+{
+    "name": "cookieconsent",
+    "prehideSelectors": ["[aria-label=\"cookieconsent\"]"],
+    "isHidingRule": true,
+    "detectCmp": [{ "exists": "[aria-label=\"cookieconsent\"]" }],
+    "detectPopup": [{ "visible": "[aria-label=\"cookieconsent\"]" }],
+    "optIn": [{ "click": ".cc-dismiss" }],
+    "optOut": [{ "hide": ["[aria-label=\"cookieconsent\"]"] }]
+  }

package/rules/autoconsent/drupal.json

@@ -0,0 +1,7 @@
+{
+  "name": "Drupal",
+  "detectCmp": [{ "exists": "#drupalorg-crosssite-gdpr" }],
+  "detectPopup": [{ "visible": "#drupalorg-crosssite-gdpr" }],
+  "optOut": [{ "click": ".no" }],
+  "optIn": [{ "click": ".yes" }]
+}

package/rules/autoconsent/eu-cookie-compliance.json

@@ -0,0 +1,14 @@
+{
+  "name": "eu-cookie-compliance-banner",
+  "isHidingRule": true,
+  "detectCmp": [{ "exists": ".eu-cookie-compliance-banner-info" }],
+  "detectPopup": [{ "visible": ".eu-cookie-compliance-banner-info" }],
+  "optIn": [{ "click": ".agree-button" }],
+  "optOut": [
+    { "click": ".decline-button,.eu-cookie-compliance-save-preferences-button", "optional": true },
+    { "hide": [".eu-cookie-compliance-banner-info", "#sliding-popup"] }
+  ],
+  "test": [
+      { "eval": "document.cookie.indexOf('cookie-agreed=2') === -1" }
+  ]
+}

package/rules/autoconsent/fundingchoices.json

@@ -0,0 +1,12 @@
+{
+    "name": "funding-choices",
+    "prehideSelectors": [".fc-consent-root,.fc-dialog-container,.fc-dialog-overlay,.fc-dialog-content"],
+    "detectCmp": [{ "exists": ".fc-consent-root" }],
+    "detectPopup": [{ "exists": ".fc-dialog-container" }],
+    "optOut": [
+      { "click": ".fc-cta-do-not-consent,.fc-cta-manage-options" },
+      { "click": ".fc-preference-consent:checked,.fc-preference-legitimate-interest:checked", "all": true, "optional": true },
+      { "click": ".fc-confirm-choices", "optional": true }
+    ],
+    "optIn": [{ "click": ".fc-cta-consent" }]
+  }

package/rules/autoconsent/hubspot.json

@@ -0,0 +1,7 @@
+{
+  "name": "hubspot",
+  "detectCmp": [{ "exists": "#hs-eu-cookie-confirmation" }],
+  "detectPopup": [{ "visible": "#hs-eu-cookie-confirmation" }],
+  "optIn": [{ "click": "#hs-eu-confirmation-button" }],
+  "optOut": [{ "click": "#hs-eu-decline-button" }]
+}

package/rules/autoconsent/klaro.json

@@ -0,0 +1,10 @@
+{
+  "name": "klaro",
+  "detectCmp": [{ "exists": ".klaro > .cookie-notice" }],
+  "detectPopup": [{ "visible": ".klaro > .cookie-notice" }],
+  "optIn": [{ "click": ".cm-btn-success" }],
+  "optOut": [{ "click": ".cn-decline" }],
+  "test": [
+    { "eval": "Object.values(klaro.getManager().consents).every(c => !c)" }
+  ]
+}

package/rules/autoconsent/notice-cookie.json

@@ -0,0 +1,9 @@
+{
+  "name": "notice-cookie",
+  "prehideSelectors": [".button--notice"],
+  "isHidingRule": true,
+  "detectCmp": [{ "exists": ".notice--cookie" }],
+  "detectPopup": [{ "visible": ".notice--cookie" }],
+  "optIn": [{ "click": ".button--notice" }],
+  "optOut": [{ "hide": [".notice--cookie"] }]
+}

package/rules/autoconsent/onetrust.json

@@ -0,0 +1,24 @@
+{
+  "name": "Onetrust",
+  "prehideSelectors": ["#onetrust-banner-sdk,#onetrust-consent-sdk,.optanon-alert-box-wrapper,.onetrust-pc-dark-filter,.js-consent-banner"],
+  "isHidingRule": true,
+  "detectCmp": [{ "exists": "#onetrust-banner-sdk,.optanon-alert-box-wrapper" }],
+  "detectPopup": [{ "visible": "#onetrust-banner-sdk,.optanon-alert-box-wrapper" }],
+  "optOut": [
+    { "click": "#onetrust-pc-btn-handler,.ot-sdk-show-settings,button.js-cookie-settings" },
+    { "waitFor": "#onetrust-consent-sdk", "timeout": 2000 },
+    { "wait": 1000 },
+    {
+      "click": "#onetrust-consent-sdk input.category-switch-handler:checked,.js-editor-toggle-state:checked",
+      "all": true,
+      "optional": true
+    },
+    { "waitForThenClick": ".save-preference-btn-handler,.js-consent-save", "timeout": 1000 }
+  ],
+  "optIn": [{ "click": "onetrust-accept-btn-handler,js-accept-cookies" }],
+  "test": [
+    {
+      "eval": "window.OnetrustActiveGroups.split(',').filter(s => s.length > 0).length <= 1"
+    }
+  ]
+}

package/rules/autoconsent/osano.json

@@ -0,0 +1,11 @@
+{
+    "name": "osano",
+    "prehideSelectors": [".osano-cm-window"],
+    "isHidingRule": true,
+    "detectCmp": [{ "exists": ".osano-cm-window" }],
+    "detectPopup": [{ "visible": ".osano-cm-dialog" }],
+    "optIn": [{ "click": ".osano-cm-accept-all" }],
+    "optOut": [
+        { "hide": [".osano-cm-window"] }
+    ]
+  }

package/rules/autoconsent/quantcast.json

@@ -0,0 +1,14 @@
+{
+  "name": "quantcast",
+  "prehideSelectors": ["#qc-cmp2-main,#qc-cmp2-container"],
+  "detectCmp": [{ "exists": "#qc-cmp2-container" }],
+  "detectPopup": [{ "visible": "#qc-cmp2-ui" }],
+  "optOut": [
+    { "click": ".qc-cmp2-summary-buttons > button[mode=\"secondary\"]" },
+    { "waitFor": "#qc-cmp2-ui" },
+    { "click": ".qc-cmp2-toggle-switch > button[aria-checked=\"true\"]", "all": true, "optional": true },
+    { "click": ".qc-cmp2-main button[aria-label=\"REJECT ALL\"]", "optional": true },
+    { "waitForThenClick": ".qc-cmp2-main button[aria-label=\"SAVE & EXIT\"],.qc-cmp2-buttons-desktop > button[mode=\"primary\"]", "timeout": 5000 }
+  ],
+  "optIn": [{ "click": ".qc-cmp2-summary-buttons > button[mode=\"primary\"]" }]
+}