@browserless.io/browserless 2.0.0-beta-5 → 2.0.0-beta-7

package/extensions/ublock/js/static-net-filtering.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2014-present Raymond Hill
 
     This program is free software: you can redistribute it and/or modify
@@ -71,7 +71,7 @@ const keyvalStore = typeof vAPI !== 'undefined'
 // ||||+---------------- bit   12: removeparam filters
 // |||+----------------- bit   13: csp filters
 // ||+------------------ bit   14: permissions filters
-// |+------------------- bit   15: urltransform filters
+// |+------------------- bit   15: uritransform filters
 // +-------------------- bit   16: replace filters
 // TODO: bit 11-16 can be converted into 3-bit value, as these options are not
 //       meant to be combined.
@@ -217,7 +217,7 @@ const modifierTypeFromName = new Map([
     [ 'removeparam', MODIFIER_TYPE_REMOVEPARAM ],
     [ 'csp', MODIFIER_TYPE_CSP ],
     [ 'permissions', MODIFIER_TYPE_PERMISSIONS ],
-    [ 'urltransform', MODIFIER_TYPE_URLTRANSFORM ],
+    [ 'uritransform', MODIFIER_TYPE_URLTRANSFORM ],
     [ 'replace', MODIFIER_TYPE_REPLACE ],
 ]);
 
@@ -227,7 +227,7 @@ const modifierNameFromType = new Map([
     [ MODIFIER_TYPE_REMOVEPARAM, 'removeparam' ],
     [ MODIFIER_TYPE_CSP, 'csp' ],
     [ MODIFIER_TYPE_PERMISSIONS, 'permissions' ],
-    [ MODIFIER_TYPE_URLTRANSFORM, 'urltransform' ],
+    [ MODIFIER_TYPE_URLTRANSFORM, 'uritransform' ],
     [ MODIFIER_TYPE_REPLACE, 'replace' ],
 ]);
 
@@ -1925,13 +1925,13 @@ class FilterFromDomainMissSet extends FilterFromDomainHitSet {
         return super.match(idata) === false;
     }
 
-    static logData(idata, details) {
-        details.fromDomains.push('~' + this.getDomainOpt(idata).replace(/\|/g, '|~'));
-    }
-
     static get dnrConditionName() {
         return 'excludedInitiatorDomains';
     }
+
+    static logData(idata, details) {
+        details.fromDomains.push('~' + this.getDomainOpt(idata).replace(/\|/g, '|~'));
+    }
 }
 
 class FilterFromRegexHit extends FilterDomainRegexHit {
@@ -1939,6 +1939,10 @@ class FilterFromRegexHit extends FilterDomainRegexHit {
         return $docHostname;
     }
 
+    static get dnrConditionName() {
+        return 'initiatorDomains';
+    }
+
     static logData(idata, details) {
         details.fromDomains.push(`${this.getDomainOpt(idata)}`);
     }
@@ -1949,6 +1953,10 @@ class FilterFromRegexMiss extends FilterFromRegexHit {
         return super.match(idata) === false;
     }
 
+    static get dnrConditionName() {
+        return 'excludedInitiatorDomains';
+    }
+
     static logData(idata, details) {
         details.fromDomains.push(`~${this.getDomainOpt(idata)}`);
     }
@@ -2064,6 +2072,10 @@ class FilterToRegexHit extends FilterDomainRegexHit {
         return $requestHostname;
     }
 
+    static get dnrConditionName() {
+        return 'requestDomains';
+    }
+
     static logData(idata, details) {
         details.toDomains.push(`${this.getDomainOpt(idata)}`);
     }
@@ -2074,6 +2086,10 @@ class FilterToRegexMiss extends FilterToRegexHit {
         return super.match(idata) === false;
     }
 
+    static get dnrConditionName() {
+        return 'excludedRequestDomains';
+    }
+
     static logData(idata, details) {
         details.toDomains.push(`~${this.getDomainOpt(idata)}`);
     }
@@ -4362,7 +4378,7 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) {
         [ REMOVEPARAM_REALM, 'removeparam' ],
         [ CSP_REALM, 'csp' ],
         [ PERMISSIONS_REALM, 'permissions' ],
-        [ URLTRANSFORM_REALM, 'urltransform' ],
+        [ URLTRANSFORM_REALM, 'uritransform' ],
     ]);
     const partyness = new Map([
         [ ANYPARTY_REALM, '' ],
@@ -4430,34 +4446,38 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) {
         }
     }
 
-    // Detect and attempt salvage of rules with entity-based hostnames.
+    // Detect and attempt salvage of rules with entity-based hostnames and/or
+    // regex-based domains.
+    const isUnsupportedDomain = hn => hn.endsWith('.*') || hn.startsWith('/');
     for ( const rule of ruleset ) {
         if ( rule.condition === undefined ) { continue; }
-        if (
-            Array.isArray(rule.condition.initiatorDomains) &&
-            rule.condition.initiatorDomains.some(hn => hn.endsWith('.*'))
-        ) {
-            const domains = rule.condition.initiatorDomains.filter(
-                hn => hn.endsWith('.*') === false
-            );
-            if ( domains.length === 0 ) {
-                dnrAddRuleError(rule, `Can't salvage rule with only entity-based domain= option: ${rule.condition.initiatorDomains.join('|')}`);
-            } else {
-                dnrAddRuleWarning(rule, `Salvaged rule by ignoring ${rule.condition.initiatorDomains.length - domains.length} entity-based domain= option: ${rule.condition.initiatorDomains.join('|')}`);
-                rule.condition.initiatorDomains = domains;
+        for ( const prop of [ 'Initiator', 'Request' ] ) {
+            const hitProp = `${prop.toLowerCase()}Domains`;
+            if ( Array.isArray(rule.condition[hitProp]) ) {
+                if ( rule.condition[hitProp].some(hn => isUnsupportedDomain(hn)) ) {
+                    const domains = rule.condition[hitProp].filter(
+                        hn => isUnsupportedDomain(hn) === false 
+                    );
+                    if ( domains.length === 0 ) {
+                        dnrAddRuleError(rule, `Can't salvage rule with unsupported domain= option: ${rule.condition[hitProp].join('|')}`);
+                    } else {
+                        dnrAddRuleWarning(rule, `Salvaged rule by ignoring ${rule.condition[hitProp].length - domains.length} unsupported domain= option: ${rule.condition[hitProp].join('|')}`);
+                        rule.condition[hitProp] = domains;
+                    }
+                }
+            }
+            const missProp = `excluded${prop}Domains`;
+            if ( Array.isArray(rule.condition[missProp]) ) {
+                if ( rule.condition[missProp].some(hn => isUnsupportedDomain(hn)) ) {
+                    const domains = rule.condition[missProp].filter(
+                        hn => isUnsupportedDomain(hn) === false
+                    );
+                    rule.condition[missProp] =
+                        domains.length !== 0
+                            ? domains
+                            : undefined;
+                }
             }
-        }
-        if (
-            Array.isArray(rule.condition.excludedInitiatorDomains) &&
-            rule.condition.excludedInitiatorDomains.some(hn => hn.endsWith('.*'))
-        ) {
-            const domains = rule.condition.excludedInitiatorDomains.filter(
-                hn => hn.endsWith('.*') === false
-            );
-            rule.condition.excludedInitiatorDomains =
-                domains.length !== 0
-                    ? domains
-                    : undefined;
         }
     }
 
@@ -4550,7 +4570,7 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) {
                 dnrAddRuleError(rule, `Unsupported removeparam exception: ${rule.__modifierValue}`);
             }
             break;
-        case 'urltransform': {
+        case 'uritransform': {
             const path = rule.__modifierValue;
             let priority = rule.priority || 1;
             if ( rule.__modifierAction !== ALLOW_REALM ) {
@@ -5277,7 +5297,7 @@ FilterContainer.prototype.redirectRequest = function(redirectEngine, fctxt) {
 };
 
 FilterContainer.prototype.transformRequest = function(fctxt) {
-    const directives = this.matchAndFetchModifiers(fctxt, 'urltransform');
+    const directives = this.matchAndFetchModifiers(fctxt, 'uritransform');
     if ( directives === undefined ) { return; }
     const directive = directives[directives.length-1];
     if ( (directive.bits & ALLOW_REALM) !== 0 ) { return directives; }
@@ -5564,7 +5584,7 @@ FilterContainer.prototype.dump = function() {
         [ REMOVEPARAM_REALM, 'removeparam' ],
         [ CSP_REALM, 'csp' ],
         [ PERMISSIONS_REALM, 'permissions' ],
-        [ URLTRANSFORM_REALM, 'urltransform' ],
+        [ URLTRANSFORM_REALM, 'uritransform' ],
         [ REPLACE_REALM, 'replace' ],
     ]);
     const partyness = new Map([

package/extensions/ublock/js/storage.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2014-present Raymond Hill
 
     This program is free software: you can redistribute it and/or modify
@@ -19,8 +19,6 @@
     Home: https://github.com/gorhill/uBlock
 */
 
-/* globals WebAssembly */
-
 'use strict';
 
 /******************************************************************************/
@@ -28,8 +26,9 @@
 import publicSuffixList from '../lib/publicsuffixlist/publicsuffixlist.js';
 import punycode from '../lib/punycode.js';
 
-import cosmeticFilteringEngine from './cosmetic-filtering.js';
 import io from './assets.js';
+import { broadcast, filteringBehaviorChanged, onBroadcast } from './broadcast.js';
+import cosmeticFilteringEngine from './cosmetic-filtering.js';
 import logger from './logger.js';
 import lz4Codec from './lz4.js';
 import staticExtFilteringEngine from './static-ext-filtering.js';
@@ -109,9 +108,9 @@ import {
     };
 
     const saveTimer = vAPI.defer.create(shouldSave);
-    const saveDelay = { min: 4 };
+    const saveDelay = { sec: 23 };
 
-    saveTimer.on(saveDelay);
+    saveTimer.onidle(saveDelay);
 
     µb.saveLocalSettings = function() {
         localSettingsLastSaved = Date.now();
@@ -243,7 +242,7 @@ import {
         if ( typeof hs[key] !== typeof hsDefault[key] ) { continue; }
         this.hiddenSettings[key] = hs[key];
     }
-    this.fireEvent('hiddenSettingsChanged');
+    broadcast({ what: 'hiddenSettingsChanged' });
 };
 
 // Note: Save only the settings which values differ from the default ones.
@@ -259,7 +258,8 @@ import {
     });
 };
 
-µb.onEvent('hiddenSettingsChanged', ( ) => {
+onBroadcast(msg => {
+    if ( msg.what !== 'hiddenSettingsChanged' ) { return; }
     const µbhs = µb.hiddenSettings;
     ubologSet(µbhs.consoleLogLevel === 'info');
     vAPI.net.setOptions({
@@ -391,7 +391,8 @@ import {
     return false;
 };
 
-µb.onEvent('hiddenSettingsChanged', ( ) => {
+onBroadcast(msg => {
+    if ( msg.what !== 'hiddenSettingsChanged' ) { return; }
     µb.parsedTrustedListPrefixes = [];
 });
 
@@ -614,9 +615,8 @@ import {
 
     // https://www.reddit.com/r/uBlockOrigin/comments/cj7g7m/
     // https://www.reddit.com/r/uBlockOrigin/comments/cnq0bi/
-    µb.filteringBehaviorChanged();
-
-    vAPI.messaging.broadcast({ what: 'userFiltersUpdated' });
+    filteringBehaviorChanged();
+    broadcast({ what: 'userFiltersUpdated' });
 };
 
 µb.createUserFilters = function(details) {
@@ -635,6 +635,7 @@ import {
     for ( const key in lists ) {
         if ( lists.hasOwnProperty(key) === false ) { continue; }
         const list = lists[key];
+        if ( list.content !== 'filters' ) { continue; }
         if ( list.off !== true ) {
             selectedListKeys.push(key);
             continue;
@@ -851,7 +852,7 @@ import {
         staticExtFilteringEngine.freeze();
         redirectEngine.freeze();
         vAPI.net.unsuspend();
-        µb.filteringBehaviorChanged();
+        filteringBehaviorChanged();
 
         vAPI.storage.set({ 'availableFilterLists': µb.availableFilterLists });
 
@@ -861,7 +862,7 @@ import {
             text: 'Reloading all filter lists: done'
         });
 
-        vAPI.messaging.broadcast({
+        broadcast({
             what: 'staticFilteringDataChanged',
             parseCosmeticFilters: µb.userSettings.parseAllABPHideFilters,
             ignoreGenericCosmeticFilters: µb.userSettings.ignoreGenericCosmeticFilters,
@@ -1240,6 +1241,8 @@ import {
     //   memory usage at selfie-load time. For some reasons.
 
     const create = async function() {
+        vAPI.alarms.clear('createSelfie');
+        createTimer.off();
         if ( µb.inMemoryFilters.length !== 0 ) { return; }
         if ( Object.keys(µb.availableFilterLists).length === 0 ) { return; }
         await Promise.all([
@@ -1313,11 +1316,23 @@ import {
             io.remove(/^selfie\//);
             µb.selfieIsInvalid = true;
         }
+        if ( µb.wakeupReason === 'createSelfie' ) {
+            µb.wakeupReason = '';
+            return createTimer.offon({ sec: 27 });
+        }
+        vAPI.alarms.create('createSelfie', {
+            delayInMinutes: µb.hiddenSettings.selfieAfter
+        });
         createTimer.offon({ min: µb.hiddenSettings.selfieAfter });
     };
 
     const createTimer = vAPI.defer.create(create);
 
+    vAPI.alarms.onAlarm.addListener(alarm => {
+        if ( alarm.name !== 'createSelfie') { return; }
+        µb.wakeupReason = 'createSelfie';
+    });
+
     µb.selfieManager = { load, destroy };
 }
 
@@ -1482,17 +1497,28 @@ import {
 
     const launchTimer = vAPI.defer.create(fetchDelay => {
         next = 0;
-        io.updateStart({ delay: fetchDelay, auto: true });
+        io.updateStart({ fetchDelay, auto: true });
     });
 
-    µb.scheduleAssetUpdater = async function(updateDelay) {
+    µb.scheduleAssetUpdater = async function(details = {}) {
         launchTimer.off();
 
-        if ( updateDelay === 0 ) {
+        if ( details.now ) {
             next = 0;
+            io.updateStart(details);
             return;
         }
 
+        if ( µb.userSettings.autoUpdate === false ) {
+            if ( Boolean(details.updateDelay) === false ) {
+                next = 0;
+                return;
+            }
+        }
+
+        let updateDelay = details.updateDelay ||
+            this.hiddenSettings.autoUpdatePeriod * 3600000;
+
         const now = Date.now();
         let needEmergencyUpdate = false;
 
@@ -1583,10 +1609,10 @@ import {
         } else if ( details.assetKey === 'ublock-badlists' ) {
             this.badLists = new Map();
         }
-        vAPI.messaging.broadcast({
+        broadcast({
             what: 'assetUpdated',
             key: details.assetKey,
-            cached: cached
+            cached,
         });
         // https://github.com/gorhill/uBlock/issues/2585
         //   Whenever an asset is overwritten, the current selfie is quite
@@ -1597,10 +1623,10 @@ import {
 
     // Update failed.
     if ( topic === 'asset-update-failed' ) {
-        vAPI.messaging.broadcast({
+        broadcast({
             what: 'assetUpdated',
             key: details.assetKey,
-            failed: true
+            failed: true,
         });
         return;
     }
@@ -1617,14 +1643,8 @@ import {
             }
             this.loadFilterLists();
         }
-        if ( this.userSettings.autoUpdate ) {
-            this.scheduleAssetUpdater(
-                this.hiddenSettings.autoUpdatePeriod * 3600000 || 25200000
-            );
-        } else {
-            this.scheduleAssetUpdater(0);
-        }
-        vAPI.messaging.broadcast({
+        this.scheduleAssetUpdater();
+        broadcast({
             what: 'assetsUpdated',
             assetKeys: details.assetKeys
         });

package/extensions/ublock/js/support.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2014-present Raymond Hill
 
     This program is free software: you can redistribute it and/or modify
@@ -23,6 +23,7 @@
 
 'use strict';
 
+import { onBroadcast } from './broadcast.js';
 import { dom, qs$ } from './dom.js';
 
 /******************************************************************************/
@@ -254,8 +255,7 @@ async function updateFilterLists() {
     if ( dom.body.dataset.shouldUpdateLists === undefined ) { return false; }
     dom.cl.add(dom.body, 'updating');
     const assetKeys = JSON.parse(dom.body.dataset.shouldUpdateLists);
-    vAPI.messaging.send('dashboard', { what: 'purgeCaches', assetKeys });
-    vAPI.messaging.send('dashboard', { what: 'forceUpdateAssets' });
+    vAPI.messaging.send('dashboard', { what: 'supportUpdateNow', assetKeys });
     return true;
 }
 
@@ -316,7 +316,7 @@ uBlockDashboard.patchCodeMirrorEditor(cmEditor);
         });
     }
 
-    vAPI.broadcastListener.add(msg => {
+    onBroadcast(msg => {
         if ( msg.what === 'assetsUpdated' ) {
             dom.cl.remove(dom.body, 'updating');
             dom.cl.add(dom.body, 'updated');

package/extensions/ublock/js/tab.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2014-present Raymond Hill
 
     This program is free software: you can redistribute it and/or modify

package/extensions/ublock/js/tasks.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2014-present Raymond Hill
 
     This program is free software: you can redistribute it and/or modify

package/extensions/ublock/js/text-encode.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2018 Raymond Hill
 
     This program is free software: you can redistribute it and/or modify

package/extensions/ublock/js/text-utils.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2014-present Raymond Hill
 
     This program is free software: you can redistribute it and/or modify

package/extensions/ublock/js/theme.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2014-present Raymond Hill
 
     This program is free software: you can redistribute it and/or modify

package/extensions/ublock/js/traffic.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2014-present Raymond Hill
 
     This program is free software: you can redistribute it and/or modify
@@ -687,6 +687,7 @@ const bodyFilterer = (( ) => {
         'application/vnd.api+json',
         'application/vnd.apple.mpegurl',
         'application/vnd.apple.mpegurl.audio',
+        'application/x-javascript',
         'application/x-mpegurl',
         'application/xhtml+xml',
         'application/xml',

package/extensions/ublock/js/ublock.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2014-present Raymond Hill
 
     This program is free software: you can redistribute it and/or modify
@@ -23,12 +23,13 @@
 
 /******************************************************************************/
 
-import contextMenu from './contextmenu.js';
-import cosmeticFilteringEngine from './cosmetic-filtering.js';
 import io from './assets.js';
 import µb from './background.js';
-import { hostnameFromURI } from './uri-utils.js';
+import { broadcast, filteringBehaviorChanged, onBroadcast } from './broadcast.js';
+import contextMenu from './contextmenu.js';
+import cosmeticFilteringEngine from './cosmetic-filtering.js';
 import { redirectEngine } from './redirect-engine.js';
+import { hostnameFromURI } from './uri-utils.js';
 
 import {
     permanentFirewall,
@@ -147,7 +148,7 @@ const matchBucket = function(url, hostname, bucket, start) {
         }
         bucket.push(directive);
         this.saveWhitelist();
-        µb.filteringBehaviorChanged({ hostname: targetHostname });
+        filteringBehaviorChanged({ hostname: targetHostname });
         return true;
     }
 
@@ -188,7 +189,7 @@ const matchBucket = function(url, hostname, bucket, start) {
         }
     }
     this.saveWhitelist();
-    µb.filteringBehaviorChanged({ direction: 1 });
+    filteringBehaviorChanged({ direction: 1 });
     return true;
 };
 
@@ -344,7 +345,7 @@ const matchBucket = function(url, hostname, bucket, start) {
         }
         break;
     case 'autoUpdate':
-        this.scheduleAssetUpdater(value ? 7 * 60 * 1000 : 0);
+        this.scheduleAssetUpdater({ updateDelay: value ? 2000 : 0 });
         break;
     case 'cnameUncloakEnabled':
         if ( vAPI.net.canUncloakCnames === true ) {
@@ -423,7 +424,7 @@ const matchBucket = function(url, hostname, bucket, start) {
         redirectEngine.invalidateResourcesSelfie(io);
         this.loadRedirectResources();
     }
-    this.fireEvent('hiddenSettingsChanged');
+    broadcast({ what: 'hiddenSettingsChanged' });
 };
 
 /******************************************************************************/
@@ -524,7 +525,7 @@ const matchBucket = function(url, hostname, bucket, start) {
     cosmeticFilteringEngine.removeFromSelectorCache(srcHostname, 'net');
 
     // Flush caches
-    µb.filteringBehaviorChanged({
+    filteringBehaviorChanged({
         direction: action === 1 ? 1 : 0,
         hostname: srcHostname,
     });
@@ -613,7 +614,7 @@ const matchBucket = function(url, hostname, bucket, start) {
         switch ( details.name ) {
             case 'no-scripting':
             case 'no-remote-fonts':
-                µb.filteringBehaviorChanged({
+                filteringBehaviorChanged({
                     direction: details.state ? 1 : 0,
                     hostname: details.hostname,
                 });
@@ -677,7 +678,10 @@ const matchBucket = function(url, hostname, bucket, start) {
 
     parse();
 
-    µb.onEvent('hiddenSettingsChanged', ( ) => { parse(); });
+    onBroadcast(msg => {
+        if ( msg.what !== 'hiddenSettingsChanged' ) { return; }
+        parse();
+    });
 }
 
 /******************************************************************************/

package/extensions/ublock/js/uri-utils.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2014-present Raymond Hill
 
     This program is free software: you can redistribute it and/or modify

package/extensions/ublock/js/url-net-filtering.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to black/white list requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2015-present Raymond Hill
 
     This program is free software: you can redistribute it and/or modify