@browserless.io/browserless 2.20.1 → 2.21.0

package/extensions/ublock/js/messaging.js

@@ -19,53 +19,49 @@
     Home: https://github.com/gorhill/uBlock
 */
 
-/* globals browser */
-
-'use strict';
+import * as s14e from './s14e-serializer.js';
+import * as sfp from './static-filtering-parser.js';
 
-/******************************************************************************/
+import {
+    domainFromHostname,
+    domainFromURI,
+    entityFromDomain,
+    hostnameFromURI,
+    isNetworkURI,
+} from './uri-utils.js';
 
-import publicSuffixList from '../lib/publicsuffixlist/publicsuffixlist.js';
-import punycode from '../lib/punycode.js';
+import {
+    permanentFirewall,
+    permanentSwitches,
+    permanentURLFiltering,
+    sessionFirewall,
+    sessionSwitches,
+    sessionURLFiltering,
+} from './filtering-engines.js';
 
-import { filteringBehaviorChanged } from './broadcast.js';
 import cacheStorage from './cachestorage.js';
 import cosmeticFilteringEngine from './cosmetic-filtering.js';
+import { denseBase64 } from './base64-custom.js';
+import { filteringBehaviorChanged } from './broadcast.js';
 import htmlFilteringEngine from './html-filtering.js';
+import { i18n$ } from './i18n.js';
+import io from './assets.js';
 import logger from './logger.js';
 import lz4Codec from './lz4.js';
-import io from './assets.js';
+import publicSuffixList from '../lib/publicsuffixlist/publicsuffixlist.js';
+import punycode from '../lib/punycode.js';
+import { redirectEngine } from './redirect-engine.js';
 import scriptletFilteringEngine from './scriptlet-filtering.js';
 import staticFilteringReverseLookup from './reverselookup.js';
 import staticNetFilteringEngine from './static-net-filtering.js';
-import µb from './background.js';
 import webRequest from './traffic.js';
-import { denseBase64 } from './base64-custom.js';
-import { dnrRulesetFromRawLists } from './static-dnr-filtering.js';
-import { i18n$ } from './i18n.js';
-import { redirectEngine } from './redirect-engine.js';
-import * as sfp from './static-filtering-parser.js';
-import * as s14e from './s14e-serializer.js';
-
-import {
-    permanentFirewall,
-    sessionFirewall,
-    permanentSwitches,
-    sessionSwitches,
-    permanentURLFiltering,
-    sessionURLFiltering,
-} from './filtering-engines.js';
-
-import {
-    domainFromHostname,
-    domainFromURI,
-    entityFromDomain,
-    hostnameFromURI,
-    isNetworkURI,
-} from './uri-utils.js';
+import µb from './background.js';
 
 /******************************************************************************/
 
+const hasOwnProperty = (o, p) =>
+    Object.prototype.hasOwnProperty.call(o, p);
+
 // https://github.com/uBlockOrigin/uBlock-issues/issues/710
 //   Listeners have a name and a "privileged" status.
 //   The nameless default handler is always deemed "privileged".
@@ -291,9 +287,8 @@ const getHostnameDict = function(hostnameDetailsMap, out) {
     const cnMap = [];
 
     const createDictEntry = (domain, hostname, details) => {
-        const cname = vAPI.net.canonicalNameFromHostname(hostname);
-        if ( cname !== undefined ) {
-            cnMap.push([ cname, hostname ]);
+        if ( details.cname ) {
+            cnMap.push([ details.cname, hostname ]);
         }
         hnDict[hostname] = { domain, counts: details.counts };
     };
@@ -807,7 +802,7 @@ const onMessage = function(request, sender, callback) {
         });
         break;
 
-    case 'shouldRenderNoscriptTags':
+    case 'shouldRenderNoscriptTags': {
         if ( pageStore === null ) { break; }
         const fctxt = µb.filteringContext.fromTabId(sender.tabId);
         if ( pageStore.filterScripting(fctxt, undefined) ) {
@@ -818,7 +813,7 @@ const onMessage = function(request, sender, callback) {
             });
         }
         break;
-
+    }
     case 'retrieveGenericCosmeticSelectors':
         request.tabId = sender.tabId;
         request.frameId = sender.frameId;
@@ -1098,7 +1093,7 @@ const restoreUserData = async function(request) {
     // Discard unknown setting or setting with default value.
     for ( const key in hiddenSettings ) {
         if (
-            µb.hiddenSettingsDefault.hasOwnProperty(key) === false ||
+            hasOwnProperty(µb.hiddenSettingsDefault, key) === false ||
             hiddenSettings[key] === µb.hiddenSettingsDefault[key]
         ) {
             delete hiddenSettings[key];
@@ -1128,7 +1123,7 @@ const restoreUserData = async function(request) {
     });
     µb.saveUserFilters(userData.userFilters);
     if ( Array.isArray(userData.selectedFilterLists) ) {
-         await µb.saveSelectedFilterLists(userData.selectedFilterLists);
+        await µb.saveSelectedFilterLists(userData.selectedFilterLists);
     }
 
     vAPI.app.restart();
@@ -1150,7 +1145,7 @@ const resetUserData = async function() {
 // Filter lists
 const prepListEntries = function(entries) {
     for ( const k in entries ) {
-        if ( entries.hasOwnProperty(k) === false ) { continue; }
+        if ( hasOwnProperty(entries, k) === false ) { continue; }
         const entry = entries[k];
         if ( typeof entry.supportURL === 'string' && entry.supportURL !== '' ) {
             entry.supportName = hostnameFromURI(entry.supportURL);
@@ -1338,7 +1333,7 @@ const getSupportData = async function() {
     let addedListset = {};
     let removedListset = {};
     for ( const listKey in lists ) {
-        if ( lists.hasOwnProperty(listKey) === false ) { continue; }
+        if ( hasOwnProperty(lists, listKey) === false ) { continue; }
         const list = lists[listKey];
         if ( list.content !== 'filters' ) { continue; }
         const used = µb.selectedFilterLists.includes(listKey);
@@ -1755,7 +1750,7 @@ const onMessage = (request, sender, callback) => {
     // Sync
     let response;
     switch ( request.what ) {
-    case 'getInspectorArgs':
+    case 'getInspectorArgs': {
         const bc = new globalThis.BroadcastChannel('contentInspectorChannel');
         bc.postMessage({
             what: 'contentInspectorChannel',
@@ -1768,6 +1763,7 @@ const onMessage = (request, sender, callback) => {
             ),
         };
         break;
+    }
     default:
         return vAPI.messaging.UNHANDLED;
     }
@@ -1900,95 +1896,15 @@ const onMessage = function(request, sender, callback) {
             ),
             env: vAPI.webextFlavor.env,
         };
-        const t0 = Date.now();
-        dnrRulesetFromRawLists(listPromises, options).then(result => {
-            const { network } = result;
-            const replacer = (k, v) => {
-                if ( k.startsWith('__') ) { return; }
-                if ( Array.isArray(v) ) {
-                    return v.sort();
-                }
-                if ( v instanceof Object ) {
-                    const sorted = {};
-                    for ( const kk of Object.keys(v).sort() ) {
-                        sorted[kk] = v[kk];
-                    }
-                    return sorted;
-                }
-                return v;
-            };
-            const isUnsupported = rule =>
-                rule._error !== undefined;
-            const isRegex = rule =>
-                rule.condition !== undefined &&
-                rule.condition.regexFilter !== undefined;
-            const isRedirect = rule =>
-                rule.action !== undefined &&
-                rule.action.type === 'redirect' &&
-                rule.action.redirect.extensionPath !== undefined;
-            const isCsp = rule =>
-                rule.action !== undefined &&
-                rule.action.type === 'modifyHeaders';
-            const isRemoveparam = rule =>
-                rule.action !== undefined &&
-                rule.action.type === 'redirect' &&
-                rule.action.redirect.transform !== undefined;
-            const runtime = Date.now() - t0;
-            const { ruleset } = network;
-            const good = ruleset.filter(rule =>
-                isUnsupported(rule) === false &&
-                isRegex(rule) === false &&
-                isRedirect(rule) === false &&
-                isCsp(rule) === false &&
-                isRemoveparam(rule) === false
-            );
-            const unsupported = ruleset.filter(rule =>
-                isUnsupported(rule)
-            );
-            const regexes = ruleset.filter(rule =>
-                isUnsupported(rule) === false &&
-                isRegex(rule) &&
-                isRedirect(rule) === false &&
-                isCsp(rule) === false &&
-                isRemoveparam(rule) === false
-            );
-            const redirects = ruleset.filter(rule =>
-                isUnsupported(rule) === false &&
-                isRedirect(rule)
-            );
-            const headers = ruleset.filter(rule =>
-                isUnsupported(rule) === false &&
-                isCsp(rule)
-            );
-            const removeparams = ruleset.filter(rule =>
-                isUnsupported(rule) === false &&
-                isRemoveparam(rule)
-            );
-            const out = [
-                `dnrRulesetFromRawLists(${JSON.stringify(listNames, null, 2)})`,
-                `Run time: ${runtime} ms`,
-                `Filters count: ${network.filterCount}`,
-                `Accepted filter count: ${network.acceptedFilterCount}`,
-                `Rejected filter count: ${network.rejectedFilterCount}`,
-                `Un-DNR-able filter count: ${unsupported.length}`,
-                `Resulting DNR rule count: ${ruleset.length}`,
-            ];
-            out.push(`+ Good filters (${good.length}): ${JSON.stringify(good, replacer, 2)}`);
-            out.push(`+ Regex-based filters (${regexes.length}): ${JSON.stringify(regexes, replacer, 2)}`);
-            out.push(`+ 'redirect=' filters (${redirects.length}): ${JSON.stringify(redirects, replacer, 2)}`);
-            out.push(`+ 'csp=' filters (${headers.length}): ${JSON.stringify(headers, replacer, 2)}`);
-            out.push(`+ 'removeparam=' filters (${removeparams.length}): ${JSON.stringify(removeparams, replacer, 2)}`);
-            out.push(`+ Unsupported filters (${unsupported.length}): ${JSON.stringify(unsupported, replacer, 2)}`);
-            out.push(`+ generichide exclusions (${network.generichideExclusions.length}): ${JSON.stringify(network.generichideExclusions, replacer, 2)}`);
-            if ( result.specificCosmetic ) {
-                out.push(`+ Cosmetic filters: ${result.specificCosmetic.size}`);
-                for ( const details of result.specificCosmetic ) {
-                    out.push(`    ${JSON.stringify(details)}`);
-                }
-            } else {
-                out.push('  Cosmetic filters: 0');
-            }
-            callback(out.join('\n'));
+        import('./static-dnr-filtering.js').then(module => {
+            const t0 = Date.now();
+            module.dnrRulesetFromRawLists(listPromises, options).then(dnrdata => {
+                dnrdata.listNames = listNames;
+                dnrdata.runtime = Date.now() - t0;
+                callback(s14e.serialize(dnrdata));
+            })
+        }).catch(reason => {
+            callback(reason);
         });
         return;
     }
@@ -2004,6 +1920,12 @@ const onMessage = function(request, sender, callback) {
         response = staticNetFilteringEngine.dump();
         break;
 
+    case 'snfeQuery':
+        response = staticNetFilteringEngine.test(
+            Object.assign({ redirectEngine }, request.query)
+        );
+        break;
+
     case 'cfeDump':
         response = cosmeticFilteringEngine.dump();
         break;
@@ -2181,7 +2103,7 @@ const onMessage = function(request, sender, callback) {
         }
         break;
 
-    case 'subscribeTo':
+    case 'subscribeTo': {
         // https://github.com/uBlockOrigin/uBlock-issues/issues/1797
         if ( /^(file|https?):\/\//.test(request.location) === false ) { break; }
         const url = encodeURIComponent(request.location);
@@ -2194,8 +2116,8 @@ const onMessage = function(request, sender, callback) {
             select: true,
         });
         break;
-
-    case 'updateLists':
+    }
+    case 'updateLists': {
         const listkeys = request.listkeys.split(',').filter(s => s !== '');
         if ( listkeys.length === 0 ) { return; }
         if ( listkeys.includes('all') ) {
@@ -2211,7 +2133,7 @@ const onMessage = function(request, sender, callback) {
         });
         µb.scheduleAssetUpdater({ now: true, fetchDelay: 100, auto: request.auto });
         break;
-
+    }
     default:
         return vAPI.messaging.UNHANDLED;
     }

package/extensions/ublock/js/pagestore.js

@@ -53,6 +53,9 @@ To create a log of net requests
 /******************************************************************************/
 
 const NetFilteringResultCache = class {
+    shelfLife = 15000;
+    extensionOriginURL = vAPI.getURL('/');
+
     constructor() {
         this.pruneTimer = vAPI.defer.create(( ) => {
             this.prune();
@@ -172,9 +175,6 @@ const NetFilteringResultCache = class {
     }
 };
 
-NetFilteringResultCache.prototype.shelfLife = 15000;
-NetFilteringResultCache.prototype.extensionOriginURL = vAPI.getURL('/');
-
 /******************************************************************************/
 
 // Frame stores are used solely to associate a URL with a frame id.
@@ -274,18 +274,16 @@ const FrameStore = class {
     }
 
     static factory(frameURL, parentId = -1) {
-        const entry = FrameStore.junkyard.pop();
-        if ( entry === undefined ) {
-            return new FrameStore(frameURL, parentId);
+        const FS = FrameStore;
+        if ( FS.junkyard.length !== 0 ) {
+            return FS.junkyard.pop().init(frameURL, parentId);
         }
-        return entry.init(frameURL, parentId);
+        return new FS(frameURL, parentId);
     }
+    static junkyard = [];
+    static junkyardMax = 50;
 };
 
-// To mitigate memory churning
-FrameStore.junkyard = [];
-FrameStore.junkyardMax = 50;
-
 /******************************************************************************/
 
 const CountDetails = class {
@@ -312,19 +310,26 @@ const HostnameDetails = class {
     }
     init(hostname) {
         this.hostname = hostname;
+        this.cname = vAPI.net.canonicalNameFromHostname(hostname);
         this.counts.reset();
+        return this;
     }
     dispose() {
-        this.hostname = '';
-        if ( HostnameDetails.junkyard.length < HostnameDetails.junkyardMax ) {
-            HostnameDetails.junkyard.push(this);
+        const HD = HostnameDetails;
+        if ( HD.junkyard.length >= HD.junkyardMax ) { return; }
+        HD.junkyard.push(this);
+    }
+    static factory(hostname) {
+        const HD = HostnameDetails;
+        if ( HD.junkyard.length !== 0 ) {
+            return HD.junkyard.pop().init(hostname);
         }
+        return new HD(hostname);
     }
+    static junkyard = [];
+    static junkyardMax = 100;
 };
 
-HostnameDetails.junkyard = [];
-HostnameDetails.junkyardMax = 100;
-
 const HostnameDetailsMap = class extends Map {
     reset() {
         this.clear();
@@ -622,7 +627,7 @@ const PageStore = class {
         ) {
             this.hostnameDetailsMap.set(
                 this.tabHostname,
-                new HostnameDetails(this.tabHostname)
+                HostnameDetails.factory(this.tabHostname)
             );
         }
         return this.hostnameDetailsMap;
@@ -700,7 +705,7 @@ const PageStore = class {
             const hostname = journal[i+0];
             let hnDetails = this.hostnameDetailsMap.get(hostname);
             if ( hnDetails === undefined ) {
-                hnDetails = new HostnameDetails(hostname);
+                hnDetails = HostnameDetails.factory(hostname);
                 this.hostnameDetailsMap.set(hostname, hnDetails);
                 this.contentLastModified = now;
             }
@@ -932,19 +937,14 @@ const PageStore = class {
     }
 
     redirectNonBlockedRequest(fctxt) {
-        const transformDirectives = staticNetFilteringEngine.transformRequest(fctxt);
-        const pruneDirectives = fctxt.redirectURL === undefined &&
-            staticNetFilteringEngine.hasQuery(fctxt) &&
-            staticNetFilteringEngine.filterQuery(fctxt) ||
-            undefined;
-        if ( transformDirectives === undefined && pruneDirectives === undefined ) { return; }
-        if ( logger.enabled !== true ) { return; }
-        if ( transformDirectives !== undefined ) {
-            fctxt.pushFilters(transformDirectives.map(a => a.logData()));
-        }
-        if ( pruneDirectives !== undefined ) {
-            fctxt.pushFilters(pruneDirectives.map(a => a.logData()));
+        const directives = [];
+        staticNetFilteringEngine.transformRequest(fctxt, directives);
+        if ( staticNetFilteringEngine.hasQuery(fctxt) ) {
+            staticNetFilteringEngine.filterQuery(fctxt, directives);
         }
+        if ( directives.length === 0 ) { return; }
+        if ( logger.enabled !== true ) { return; }
+        fctxt.pushFilters(directives.map(a => a.logData()));
         if ( fctxt.redirectURL === undefined ) { return; }
         fctxt.pushFilter({
             source: 'redirect',
@@ -952,6 +952,19 @@ const PageStore = class {
         });
     }
 
+    skipMainDocument(fctxt) {
+        const directives = staticNetFilteringEngine.urlSkip(fctxt);
+        if ( directives === undefined ) { return; }
+        if ( logger.enabled !== true ) { return; }
+        fctxt.pushFilters(directives.map(a => a.logData()));
+        if ( fctxt.redirectURL !== undefined ) {
+            fctxt.pushFilter({
+                source: 'redirect',
+                raw: fctxt.redirectURL
+            });
+        }
+    }
+
     filterCSPReport(fctxt) {
         if (
             sessionSwitches.evaluateZ(

package/extensions/ublock/js/redirect-resources.js

@@ -19,8 +19,6 @@
     Home: https://github.com/gorhill/uBlock
 */
 
-'use strict';
-
 /******************************************************************************/
 
 // The resources referenced below are found in ./web_accessible_resources/
@@ -159,8 +157,20 @@ export default new Map([
         alias: 'nooptext',
         data: 'text',
     } ],
-    [ 'noop-vmap1.0.xml', {
-        alias: 'noopvmap-1.0',
+    [ 'noop-vast2.xml', {
+        alias: 'noopvast-2.0',
+        data: 'text',
+    } ],
+    [ 'noop-vast3.xml', {
+        alias: 'noopvast-3.0',
+        data: 'text',
+    } ],
+    [ 'noop-vast4.xml', {
+        alias: 'noopvast-4.0',
+        data: 'text',
+    } ],
+    [ 'noop-vmap1.xml', {
+        alias: [ 'noop-vmap1.0.xml', 'noopvmap-1.0' ],
         data: 'text',
     } ],
     [ 'outbrain-widget.js', {

package/extensions/ublock/js/scriptlet-filtering.js

@@ -61,6 +61,7 @@ const contentScriptRegisterer = new (class {
             runAt: 'document_start',
         }).then(handle => {
             this.hostnameToDetails.set(hostname, { handle, code });
+            return handle;
         }).catch(( ) => {
             this.hostnameToDetails.delete(hostname);
         });
@@ -94,7 +95,9 @@ const contentScriptRegisterer = new (class {
     }
     unregisterHandle(handle) {
         if ( handle instanceof Promise ) {
-            handle.then(handle => { handle.unregister(); });
+            handle.then(handle => {
+                if ( handle ) { handle.unregister(); }
+            });
         } else {
             handle.unregister();
         }