@browserless.io/browserless 2.34.1 → 2.36.0

package/extensions/ublocklite/js/ubo-parser.js

@@ -63,6 +63,16 @@ const validRedirectResources = (( ) => {
     return out;
 })();
 
+const isNotEntity = hn => hn.endsWith('.*') === false;
+
+/******************************************************************************/
+
+function toSuperDomain(hn) {
+    const pos = hn.indexOf('.');
+    if ( pos === -1 ) { return; }
+    return hn.slice(pos+1);
+}
+
 /******************************************************************************/
 
 function parseHostnameList(iter) {
@@ -93,64 +103,145 @@ function parseHostnameList(iter) {
 
 /******************************************************************************/
 
-export function mergeIncludeExclude(rules) {
-    const includeExcludes = [
-        { includeName: 'requestDomains', excludeName: 'excludedRequestDomains' },
-        { includeName: 'initiatorDomains', excludeName: 'excludedInitiatorDomains' },
-        { includeName: 'resourceTypes', excludeName: 'excludedResourceTypes' },
-        { includeName: 'requestMethods', excludeName: 'excludedRequestMethods' },
+function ownerFromPropertyPath(root, path) {
+    let owner = root;
+    let prop = path;
+    for (;;) {
+        if ( owner instanceof Object === false ) { break; }
+        const pos = prop.indexOf('.');
+        if ( pos === -1 ) { break; }
+        owner = owner[prop.slice(0, pos)];
+        prop = prop.slice(pos+1)
+    }
+    return { owner: owner ?? undefined, prop };
+}
+
+/******************************************************************************/
+
+function mergeArrays(rules, propertyPath) {
+    const out = [];
+    const distinctRules = new Map();
+    for ( const rule of rules ) {
+        const { id } = rule;
+        const { owner, prop } = ownerFromPropertyPath(rule, propertyPath);
+        if ( owner === undefined || Array.isArray(owner[prop]) === false ) {
+            out.push(rule);
+            continue;
+        }
+        const collection = owner[prop] || [];
+        owner[prop] = undefined;
+        rule.id = undefined;
+        const hash = JSON.stringify(rule);
+        const details = distinctRules.get(hash) ||
+            { id, collection: new Set() };
+        if ( details.collection.size === 0 ) {
+            distinctRules.set(hash, details);
+        }
+        for ( const hn of collection ) {
+            details.collection.add(hn);
+        }
+    }
+    for ( const [ hash, { id, collection } ] of distinctRules ) {
+        const rule = JSON.parse(hash);
+        if ( id ) {
+            rule.id = id;
+        }
+        if ( collection.size !== 0 ) {
+            const { owner, prop } = ownerFromPropertyPath(rule, propertyPath);
+            owner[prop] = Array.from(collection).sort();
+        }
+        out.push(rule);
+    }
+    return out;
+}
+
+/******************************************************************************/
+
+function minimizeRuleset(rules) {
+    rules = mergeArrays(rules, 'condition.requestDomains');
+    rules = mergeArrays(rules, 'condition.excludedRequestDomains');
+    rules = mergeArrays(rules, 'condition.initiatorDomains');
+    rules = mergeArrays(rules, 'condition.excludedInitiatorDomains');
+    rules = mergeArrays(rules, 'condition.resourceTypes');
+    rules = mergeArrays(rules, 'condition.excludedRequestMethods');
+    rules = mergeArrays(rules, 'condition.requestMethods');
+    rules = mergeArrays(rules, 'condition.excludedResourceTypes');
+    rules = mergeArrays(rules, 'action.redirect.transform.queryTransform.removeParams');
+    return rules;
+}
+
+/******************************************************************************/
+
+function minimizeRules(rules) {
+    const hostnameListProp = [
+        'requestDomains',
+        'excludedRequestDomains',
+        'initiatorDomains',
+        'excludedInitiatorDomains',
     ];
-    for ( const { includeName, excludeName } of includeExcludes ) {
-        const out = [];
-        const distinctRules = new Map();
-        for ( const rule of rules ) {
-            const { id, condition } = rule;
-            if ( Boolean(condition[includeName]?.length) === false ) {
-                if ( Boolean(condition[excludeName]?.length) === false ) {
-                    out.push(rule);
-                    continue;
+    for ( const rule of rules ) {
+        for ( const prop of hostnameListProp ) {
+            const hostnames = rule.condition[prop];
+            if ( hostnames === undefined ) { continue; }
+            if ( hostnames.length === 1 ) { continue; }
+            const hnSet = new Set(hostnames);
+            for ( let hn of hnSet ) {
+                for (;;) {
+                    const hnup = toSuperDomain(hn);
+                    if ( hnup === undefined ) { break; }
+                    if ( hnSet.has(hnup) ) { hnSet.delete(hn); }
+                    hn = hnup;
                 }
             }
-            const included = condition[includeName] || [];
-            condition[includeName] = undefined;
-            const excluded = condition[excludeName] || [];
-            condition[excludeName] = undefined;
-            rule.id = undefined;
-            const hash = JSON.stringify(rule);
-            const details = distinctRules.get(hash) ||
-                { id, included: new Set(), excluded: new Set() };
-            if ( details.included.size === 0 && details.excluded.size === 0 ) {
-                distinctRules.set(hash, details);
-            }
-            for ( const hn of included ) {
-                details.included.add(hn);
-            }
-            for ( const hn of excluded ) {
-                if ( details.included.has(hn) ) { continue; }
-                details.excluded.add(hn);
-            }
-        }
-        for ( const [ hash, { id, included, excluded } ] of distinctRules ) {
-            const rule = JSON.parse(hash);
-            if ( id ) {
-                rule.id = id;
-            }
-            if ( included.size !== 0 ) {
-                rule.condition[includeName] = Array.from(included);
-            }
-            if ( excluded.size !== 0 ) {
-                rule.condition[excludeName] = Array.from(excluded);
-            }
-            out.push(rule);
+            if ( hnSet.size === hostnames.length ) { continue; }
+            rule.condition[prop] = Array.from(hnSet).sort();
         }
-        rules = out;
     }
     return rules;
 }
 
 /******************************************************************************/
 
-function parseNetworkFilter(parser) {
+function dropEntities(rule, prop) {
+    const { condition } = rule;
+    if ( condition[prop] === undefined ) { return; }
+    const sanitized = condition[prop].filter(a => isNotEntity(a));
+    if ( sanitized.length === condition[prop].length ) { return; }
+    if ( sanitized.length === 0 ) { return 0; }
+    condition.requestDomains = sanitized;
+}
+
+/******************************************************************************/
+
+function validateRules(rules) {
+    const out = [];
+    for ( const rule of rules ) {
+        const { condition } = rule;
+        // "Only one of resourceTypes and excludedResourceTypes should be specified"
+        if ( condition.resourceTypes ) {
+            if ( condition.excludedResourceTypes ) { continue; }
+        }
+        // "Only one of requestMethods and excludedRequestMethods should be specified"
+        if ( condition.requestMethods ) {
+            if ( condition.excludedRequestMethods ) { continue; }
+        }
+        // Drop entity-based hostnames
+        if ( dropEntities(rule, 'requestDomains') === 0 ) { continue; }
+        if ( dropEntities(rule, 'excludedRequestDomains') === 0 ) { continue; }
+        if ( dropEntities(rule, 'initiatorDomains') === 0 ) { continue; }
+        if ( dropEntities(rule, 'excludedInitiatorDomains') === 0 ) { continue; }
+        // regexSubstitution requires regexFilter
+        if ( rule.action?.redirect?.regexSubstitution ) {
+            if ( rule.condition.regexFilter === undefined ) { continue; }
+        }
+        out.push(rule);
+    }
+    return out;
+}
+
+/******************************************************************************/
+
+export function parseNetworkFilter(parser) {
     if ( parser.isNetworkFilter() === false ) { return; }
     if ( parser.hasError() ) { return; }
 
@@ -181,6 +272,9 @@ function parseNetworkFilter(parser) {
         rule.condition.urlFilter = pattern;
     }
 
+    const defaultResourceTypes = new Set();
+    let defaultUrlFilter = '';
+
     const initiatorDomains = new Set();
     const excludedInitiatorDomains = new Set();
     const requestDomains = new Set();
@@ -247,6 +341,9 @@ function parseNetworkFilter(parser) {
                 operation: 'append',
                 value: parser.getNetOptionValue(type),
             } ];
+            defaultResourceTypes.add('main_frame');
+            defaultResourceTypes.add('sub_frame');
+            defaultResourceTypes.add('object');
             break;
         case sfp.NODE_TYPE_NET_OPTION_NAME_CSS:
             processResourceType('stylesheet', type);
@@ -338,6 +435,9 @@ function parseNetworkFilter(parser) {
                 operation: 'append',
                 value: parser.getNetOptionValue(type),
             } ];
+            defaultResourceTypes.add('main_frame');
+            defaultResourceTypes.add('sub_frame');
+            defaultResourceTypes.add('object');
             break;
         case sfp.NODE_TYPE_NET_OPTION_NAME_PING:
             processResourceType('ping', type);
@@ -374,6 +474,10 @@ function parseNetworkFilter(parser) {
             rule.action.redirect = {
                 transform: { queryTransform: { removeParams } }
             };
+            defaultResourceTypes.add('main_frame');
+            defaultResourceTypes.add('sub_frame');
+            defaultResourceTypes.add('xmlhttprequest');
+            defaultUrlFilter = '?';
             break;
         }
         case sfp.NODE_TYPE_NET_OPTION_NAME_SCRIPT:
@@ -395,11 +499,16 @@ function parseNetworkFilter(parser) {
             }
             break;
         }
-        case sfp.NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM:
-            if ( this.processOptionWithValue(parser, type) === false ) {
-                return this.FILTER_INVALID;
-            }
+        case sfp.NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM: {
+            const parsed = sfp.parseReplaceByRegexValue(parser.getNetOptionValue(type));
+            if ( parsed === undefined ) { return; }
+            if ( parsed.re ) { return; }
+            rule.action.type = 'redirect';
+            rule.action.redirect = {
+                regexSubstitution: parsed.replacement.replace(/\$(\d+)/g, '\\$1'),
+            };
             break;
+        }
         case sfp.NODE_TYPE_NET_OPTION_NAME_XHR:
             processResourceType('xmlhttprequest', type);
             break;
@@ -410,34 +519,38 @@ function parseNetworkFilter(parser) {
             break;
         }
     }
+    if ( pattern === '*' && defaultUrlFilter !== '' ) {
+        rule.condition.urlFilter = defaultUrlFilter;
+    }
     if ( initiatorDomains.size !== 0 ) {
-        rule.condition.initiatorDomains = Array.from(initiatorDomains);
+        rule.condition.initiatorDomains = Array.from(initiatorDomains).sort();
     }
     if ( excludedInitiatorDomains.size !== 0 ) {
-        rule.condition.excludedInitiatorDomains = Array.from(excludedInitiatorDomains);
+        rule.condition.excludedInitiatorDomains = Array.from(excludedInitiatorDomains).sort();
     }
     if ( requestDomains.size !== 0 ) {
-        rule.condition.requestDomains = Array.from(requestDomains);
+        rule.condition.requestDomains = Array.from(requestDomains).sort();
     }
     if ( excludedRequestDomains.size !== 0 ) {
-        rule.condition.excludedRequestDomains = Array.from(excludedRequestDomains);
+        rule.condition.excludedRequestDomains = Array.from(excludedRequestDomains).sort();
     }
     if ( requestMethods.size !== 0 ) {
-        rule.condition.requestMethods = Array.from(requestMethods);
+        rule.condition.requestMethods = Array.from(requestMethods).sort();
     }
     if ( excludedRequestMethods.size !== 0 ) {
-        rule.condition.excludedRequestMethods = Array.from(excludedRequestMethods);
+        rule.condition.excludedRequestMethods = Array.from(excludedRequestMethods).sort();
+    }
+    if ( resourceTypes.size === 0 && excludedResourceTypes.size === 0 ) {
+        defaultResourceTypes.forEach(a => resourceTypes.add(a));
     }
     if ( resourceTypes.size !== 0 ) {
-        const types = Array.from(resourceTypes).filter(a => a !== '');
+        const types = Array.from(resourceTypes).filter(a => a !== '').sort();
         if ( types.length === 0 ) { return; }
         rule.condition.resourceTypes = types;
     }
     if ( excludedResourceTypes.size !== 0 ) {
-        if ( resourceTypes.size !== 0 ) {
-            if ( excludedResourceTypes.size !== 0 ) { return; }
-        }
-        rule.condition.excludedResourceTypes = Array.from(excludedResourceTypes);
+        if ( resourceTypes.size !== 0 ) { return; }
+        rule.condition.excludedResourceTypes = Array.from(excludedResourceTypes).sort();
     }
     if ( priority !== 0 ) {
         rule.priority = priority;
@@ -452,7 +565,7 @@ export function parseFilters(text) {
     if ( text.endsWith('---') ) { return; }
     const lines = text.split(/\n/);
     if ( lines.some(a => a.startsWith(' ')) ) { return; }
-    const rules = [];
+    let rules = [];
     const parser = new sfp.AstFilterParser({ trustedSource: true });
     for ( const line of lines ) {
         parser.parse(line);
@@ -461,5 +574,8 @@ export function parseFilters(text) {
         if ( rule === undefined ) { continue; }
         rules.push(rule);
     }
-    return mergeIncludeExclude(rules);
+    rules = minimizeRuleset(rules);
+    rules = minimizeRules(rules);
+    rules = validateRules(rules);
+    return rules;
 }

package/extensions/ublocklite/manifest.json

@@ -324,7 +324,7 @@
   "storage": {
     "managed_schema": "managed_storage.json"
   },
-  "version": "2025.818.1918",
+  "version": "2025.911.1335",
   "web_accessible_resources": [
     {
       "resources": [
@@ -396,8 +396,8 @@
         "web_accessible_resources/nobab2.js",
         "web_accessible_resources/chartbeat.js",
         "web_accessible_resources/noop.json",
-        "web_accessible_resources/click2load.html",
-        "web_accessible_resources/noeval-silent.js"
+        "web_accessible_resources/noeval-silent.js",
+        "web_accessible_resources/click2load.html"
       ],
       "matches": [
         "<all_urls>"

package/extensions/ublocklite/rulesets/generic-details.json

@@ -60,7 +60,6 @@
     "anime-loads.org",
     "animedevil.com",
     "animefire.plus",
-    "animesaria.com",
     "animevietsub.tv",
     "antenadosnaskyecia.com",
     "antonimos.de",
@@ -124,7 +123,6 @@
     "blackwoodacademy.org",
     "blog.itijobalert.in",
     "blog.potterworld.co",
-    "blog.yurasu.xyz",
     "blogshinobijawi.blogspot.com",
     "blogspot.com",
     "boardgamesonline.net",
@@ -136,7 +134,6 @@
     "bookmystrip.com",
     "bookshared.org",
     "bootdey.com",
-    "brazzers3x.org",
     "british-birdsongs.uk",
     "broncos.com.au",
     "brownsboys.com",
@@ -395,8 +392,6 @@
     "freex2line.online",
     "frkn64modding.com",
     "fruitlab.com",
-    "fssquad.com",
-    "fulltip.net",
     "fun--seiten-de.translate.goog",
     "fun-seiten.de",
     "funkeypagali.com",
@@ -417,11 +412,9 @@
     "gastroepato.it",
     "gatevidyalay.com",
     "gats.io",
-    "gdrivelinks.me",
     "gecmisi.com.tr",
     "geekdrop.com",
     "geekprank.com",
-    "genelpara.com",
     "genialne.pl",
     "gestyy.com",
     "get-click2.blogspot.com",
@@ -455,7 +448,6 @@
     "h-flash.com",
     "hacknetfl1x.net",
     "hackshort.me",
-    "hacoscripts.com",
     "handelsblatt.com",
     "hanime.tv",
     "hardware--entwicklung-de.translate.goog",
@@ -520,7 +512,6 @@
     "is2btc.com",
     "isekaipalace.com",
     "isthischannelmonetized.com",
-    "itopmusic.org",
     "itsguider.com",
     "jastrzabpost.pl",
     "javaguides.net",
@@ -546,7 +537,6 @@
     "knightlobby.com",
     "kojodan.jp",
     "komorkomania.pl",
-    "konstantinova.net",
     "koyso.com",
     "kryptografie-de.translate.goog",
     "kryptografie.de",
@@ -605,13 +595,13 @@
     "makeincomeinfo.com",
     "mangfall24.de",
     "manhwa18.com",
+    "manwa.me",
     "maqal360.com",
     "matchendirect.fr",
     "mathportal.org",
     "matzoo.pl",
     "mcqmall.com",
     "me-encantas.com",
-    "megadescarga.net",
     "meinyouporn.com",
     "melbournestorm.com.au",
     "memexcomputer.it",
@@ -642,7 +632,6 @@
     "money.it",
     "money.pl",
     "mopo.de",
-    "morbidthought.com",
     "motoroids.com",
     "mpsconline.gov.in",
     "mrfreemium.blogspot.com",
@@ -710,7 +699,6 @@
     "ontechhindi.com",
     "operationharshdoorstop.com",
     "orangeobserver.com",
-    "orangeptc.com",
     "osuskinner.com",
     "osuskins.net",
     "otakukan.com",
@@ -769,7 +757,6 @@
     "premid.app",
     "preu-pdf-paste.blogspot.com",
     "printablebricks.com",
-    "printerprojects.com",
     "prinxy.app",
     "proboards.com",
     "procinehub.com",
@@ -780,7 +767,9 @@
     "promods.net",
     "properhacks.weebly.com",
     "psprices.com",
+    "pubgaimassist.com",
     "pudelek.pl",
+    "pvpoke-re.com",
     "pysznosci.pl",
     "qctimes.com",
     "qrl.com.au",
@@ -864,7 +853,6 @@
     "shirainime.com",
     "shoop.de",
     "shoot-yalla.to",
-    "shoot-yalla.tv",
     "shorterall.com",
     "shortit.pw",
     "shorttrick.in",
@@ -885,7 +873,6 @@
     "sms-anonyme.net",
     "smscodeonline.com",
     "smsonline.cloud",
-    "soccerinhd.com",
     "software-on.com",
     "softwaredetail.com",
     "softwareengineer-de.translate.goog",
@@ -1464,6 +1451,7 @@
    "unhide": [
     "10010.com",
     "10086.cn",
+    "1991421.cn",
     "199it.com",
     "1keydata.com",
     "3dpchip.com",
@@ -1548,6 +1536,7 @@
     "haichaobiao.com",
     "haiwai.com",
     "hh010.com",
+    "hk.on.cc",
     "hopto.org",
     "ibf.tw",
     "imydl.com",
@@ -2192,6 +2181,7 @@
     "duan24h.net",
     "gamehayvl.com",
     "gocmod.com",
+    "hackbloxfruit.pro",
     "idpvn.com",
     "itudong.com",
     "laptrinhcanban.com",