@duckduckgo/autoconsent 1.0.6 → 2.0.0

package/.eslintrc.cjs

@@ -0,0 +1,14 @@
+module.exports = {
+  root: true,
+  parser: '@typescript-eslint/parser',
+  plugins: [
+    '@typescript-eslint',
+  ],
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/recommended',
+  ],
+  rules: {
+    "@typescript-eslint/no-explicit-any": 0,
+  }
+}

package/.vscode/settings.json

@@ -0,0 +1,7 @@
+{
+    "eslint.format.enable": true,
+    "eslint.validate": [
+        "typescript"
+    ],
+    "editor.tabSize": 2,
+}

package/Jenkinsfile

@@ -1,50 +1,79 @@
-properties([
-    parameters([
-        choice(name: 'CHANNEL', defaultValue: 'staging', choices: 'staging\nproduction')
-    ]),
-])
-
-node('docker && !gpu') {
-
-    def img
-    def commitHash
+def runPlaywrightTests(resultDir) {
+    sh 'mkdir -p ./test-results'
+    sh """
+        PLAYWRIGHT_JUNIT_OUTPUT_NAME=results.xml npx playwright test --project webkit --reporter=junit || true
+    """
+    junit 'results.xml'
+    sh """
+        mkdir -p ${resultDir}/results/${BRANCH_NAME}/${BUILD_NUMBER}/$REGION/
+        mkdir -p ./test-results
+        mv ./test-results/ ${resultDir}/results/${BRANCH_NAME}/${BUILD_NUMBER}/$REGION/
+    """
+}
 
-    stage('Checkout') {
-        checkout scm
-        commitHash = sh(returnStdout: true, script: 'git log --pretty=format:\'%h\' -n 1').trim()
-        currentBuild.description = "${commitHash}-${params.CHANNEL}"
+def withEnvFile(envfile, Closure cb) {
+    def props = readProperties(file: envfile)    
+    withEnv(props.collect{ entry -> "${entry.key}=${entry.value}" }) {
+        cb()
     }
+}
 
-    stage('Build Docker Image') {
-        img = docker.build('autoconsent/build')
+pipeline {
+    agent { label 'crawler-worker' }
+    parameters {
+        string(name: 'TEST_RESULT_ROOT', defaultValue: '/mnt/efs/users/smacbeth/autoconsent/ci', description: 'Where test results and configuration are stored')
     }
-
-    img.inside() {
-
+    environment {
+        NODENV_VERSION = "14.15.4"
+        NODENV_ROOT = "/opt/nodeenv"
+        PATH = "/opt/nodenv/shims:/opt/nodenv/bin:$PATH"
+    }
+    stages {
+        stage('Checkout') {
+            steps {
+                checkout scm
+            }
+        }
+        
         stage('Build') {
-            sh 'cp -r /app/node_modules ./'
-            sh 'npm run bundle'
+            steps {
+                sh '''
+                npm ci
+                npx playwright install webkit
+                '''
+            }
         }
-
-        stage('Build rules') {
-            sh 'node rules/build.js'
-            sh 'rm -f rules/rules.min.*'
-            sh "jq -c '. + { version: \"${commitHash}\" }' rules/rules.json > rules/rules.min.json"
-            sh 'brotli --input rules/rules.min.json --output rules/rules.min.json.br'
-            sh "gzip -9 rules/rules.min.json"
+        
+        stage('Test: DE') {
+            steps {
+                withEnvFile("${params.TEST_RESULT_ROOT}/de.env") {
+                    runPlaywrightTests(params.TEST_RESULT_ROOT)
+                }
+            }
         }
-    }
-
-    if (env.BRANCH_NAME == 'master') {
-        stage('Publish') {
-            sh "aws s3 cp rules/rules.min.json.br s3://cdn.cliqz.com/autoconsent/rules/${commitHash}.json.br --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers --content-encoding br --content-type application/json --cache-control \"immutable\""
-            sh "aws s3 cp rules/rules.min.json.gz s3://cdn.cliqz.com/autoconsent/rules/${commitHash}.json.gz --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers --content-encoding gzip --content-type application/json --cache-control \"immutable\""
-            sh "echo '{\"ruleVersion\":\"${commitHash}\",\"disabled\":[]}' > config.json"
-            def fileName = 'config.json'
-            if (params.CHANNEL == 'staging') {
-                fileName = 'staging-config.json'
+        
+        stage('Test: US') {
+            steps {
+                withEnvFile("${params.TEST_RESULT_ROOT}/us.env") {
+                    runPlaywrightTests(params.TEST_RESULT_ROOT)
+                }
+            }
+        }
+        
+        stage('Test: GB') {
+            steps {
+                withEnvFile("${params.TEST_RESULT_ROOT}/gb.env") {
+                    runPlaywrightTests(params.TEST_RESULT_ROOT)
+                }
+            }
+        }
+        
+        stage('Test: FR') {
+            steps {
+                withEnvFile("${params.TEST_RESULT_ROOT}/fr.env") {
+                    runPlaywrightTests(params.TEST_RESULT_ROOT)
+                }
             }
-            sh "aws s3 cp config.json s3://cdn.cliqz.com/autoconsent/${fileName} --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers --content-type application/json --cache-control \"max-age=3600\""
         }
     }
 }

package/api.md

@@ -0,0 +1,104 @@
+# API
+
+## Content script
+Most of autoconsent logic is contained in a content script that should be injected into every page. However, the high-level orchestration is controlled externally by a background service worker (in case of a web extension), or a native browser integration. Autoconsent sends and receives messages to the background worker using provided callables which mimic postMessage API:
+
+```typescript
+const consent = new AutoConsent( // make sure not to leak anything to the page globals
+    chrome.runtime.sendMessage, // pass a function to send messages to the background worker
+    { // optionally, pass a config object
+        enabled: true,
+        autoAction: 'optOut',
+        disabledCmps: [],
+        enablePrehide: true,
+        detectRetries: 20,
+    },
+    { // optionally, pass JSON rules
+        autoconsent: [ ... ],
+        consentomatic: [ ... ],
+    }
+);
+
+// connect .receiveMessageCallback() to a platform-specific message receiver
+chrome.runtime.onMessage.addListener((message: BackgroundMessage) => {
+    return Promise.resolve(
+        consent.receiveMessageCallback(message)
+    );
+});
+```
+
+## Messaging API
+
+For concrete message format, refer to [messages.ts](/lib/messages.ts).
+
+The overall workflow is described below:
+
+```mermaid
+
+sequenceDiagram
+    participant BG as Browser (e.g. background service worker)
+    participant CS as Autoconsent content script in page (isolated) world
+
+    Note over BG: Fetch declarative JSON rules and user settings
+    Note over CS: Autoconsent class is instantiated.<br/>Optionally, initConfig is passed to constructor
+
+    CS -->>+ BG: (if initConfig is not passed already)<br/>init
+    BG -->>- CS: initResp<br/>(contains user settings and JSON rules)
+
+    activate CS
+    Note right of CS: Parse rules and initialize autoconsent code
+    Note right of CS: apply prehideSelectors
+    Note right of CS: wait for DOMContentLoaded
+    Note right of CS: detect a CMP presence (not necessarily visible)
+
+    CS ->> BG: cmpDetected
+    Note right of CS: detect a visible cookie popup
+    CS ->> BG: popupFound
+    deactivate CS
+    activate BG
+    alt if config.autoAction is NOT defined
+        Note left of BG: decide when to trigger opt-in / opt-out
+        BG -->> CS: optIn / optOut
+    else if config.autoAction IS defined
+        Note right of CS: <br/>proceed immediately
+    end
+
+    deactivate BG
+
+    Note right of CS: execute opt-in / opt-out rules
+    CS ->> BG: optOutResult / optInResult
+
+    opt if not intermediate ruleset
+        CS ->> BG: autoconsentDone
+    end
+
+    opt optional
+        BG -->>+ CS: (optional) selfTest
+        Note right of CS: execute self-test rules
+        CS -->>- BG: selfTestResult
+    end
+```
+
+### Asynchronous eval rules
+
+Some rulesets rely on [eval rules](/readme.md#eval) executed in page main context. Since the content script lives in an isolated world, it passes those calls to the more powerful background worker.
+
+**Warning**: eval rules are potentially dangerous and should be avoided when possible.
+
+```mermaid
+sequenceDiagram
+    participant P as Page main world
+    participant BG as Browser (e.g. background service worker)
+    participant CS as Autoconsent content script in page (isolated) world
+
+    Note over CS: Needs to evaluate <PAYLOAD> in the main world (e.g. interact with API)
+
+    CS ->> BG: { type: "eval", id: <UUID>, code: <PAYLOAD> }
+    activate BG
+    BG ->>+ P: evaluate <PAYLOAD> using platform-specific APIs<br/>e.g. chrome.scripting.executeScript()
+    P ->>- BG: evaluation <RESULT>
+    BG ->> CS: { type: "evalResp", id: <UUID>, result: <RESULT> }
+    deactivate BG
+
+    Note over CS: Continue rule execution (or fail with timeout)
+```