@browserless.io/browserless 2.25.0-beta-2 → 2.25.1

package/extensions/ublock/js/static-dnr-filtering.js

@@ -90,6 +90,39 @@ const keyFromSelector = selector => {
 
 /******************************************************************************/
 
+function addGenericCosmeticFilter(context, selector, isException) {
+    if ( selector === undefined ) { return; }
+    if ( selector.length <= 1 ) { return; }
+    if ( isException ) {
+        if ( context.genericCosmeticExceptions === undefined ) {
+            context.genericCosmeticExceptions = new Set();
+        }
+        context.genericCosmeticExceptions.add(selector);
+        return;
+    }
+    if ( selector.charCodeAt(0) === 0x7B /* '{' */ ) { return; }
+    const key = keyFromSelector(selector);
+    if ( key === undefined ) {
+        if ( context.genericHighCosmeticFilters === undefined ) {
+            context.genericHighCosmeticFilters = new Set();
+        }
+        context.genericHighCosmeticFilters.add(selector);
+        return;
+    }
+    const type = key.charCodeAt(0);
+    const hash = hashFromStr(type, key.slice(1));
+    if ( context.genericCosmeticFilters === undefined ) {
+        context.genericCosmeticFilters = new Map();
+    }
+    let bucket = context.genericCosmeticFilters.get(hash);
+    if ( bucket === undefined ) {
+        context.genericCosmeticFilters.set(hash, bucket = []);
+    }
+    bucket.push(selector);
+}
+
+/******************************************************************************/
+
 function addExtendedToDNR(context, parser) {
     if ( parser.isExtendedFilter() === false ) { return false; }
 
@@ -195,35 +228,8 @@ function addExtendedToDNR(context, parser) {
 
     // Generic cosmetic filtering
     if ( parser.hasOptions() === false ) {
-        const { compiled } = parser.result;
-        if ( compiled === undefined ) { return; }
-        if ( compiled.length <= 1 ) { return; }
-        if ( parser.isException() ) {
-            if ( context.genericCosmeticExceptions === undefined ) {
-                context.genericCosmeticExceptions = new Set();
-            }
-            context.genericCosmeticExceptions.add(compiled);
-            return;
-        }
-        if ( compiled.charCodeAt(0) === 0x7B /* '{' */ ) { return; }
-        const key = keyFromSelector(compiled);
-        if ( key === undefined ) {
-            if ( context.genericHighCosmeticFilters === undefined ) {
-                context.genericHighCosmeticFilters = new Set();
-            }
-            context.genericHighCosmeticFilters.add(compiled);
-            return;
-        }
-        const type = key.charCodeAt(0);
-        const hash = hashFromStr(type, key.slice(1));
-        if ( context.genericCosmeticFilters === undefined ) {
-            context.genericCosmeticFilters = new Map();
-        }
-        let bucket = context.genericCosmeticFilters.get(hash);
-        if ( bucket === undefined ) {
-            context.genericCosmeticFilters.set(hash, bucket = []);
-        }
-        bucket.push(compiled);
+        const { compiled, exception } = parser.result;
+        addGenericCosmeticFilter(context, compiled, exception);
         return;
     }
 
@@ -234,26 +240,22 @@ function addExtendedToDNR(context, parser) {
     if ( context.specificCosmeticFilters === undefined ) {
         context.specificCosmeticFilters = new Map();
     }
+    const { compiled, exception, raw } = parser.result;
+    if ( compiled === undefined ) {
+        context.specificCosmeticFilters.set(`Invalid filter: ...##${raw}`, {
+            rejected: true
+        });
+        return;
+    }
+    let details = context.specificCosmeticFilters.get(compiled);
     for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) {
         if ( bad ) { continue; }
+        if ( not && exception ) { continue; }
         if ( isRegex(hn) ) { continue; }
-        let { compiled, exception, raw } = parser.result;
-        if ( exception ) { continue; }
-        let rejected;
-        if ( compiled === undefined ) {
-            rejected = `Invalid filter: ${hn}##${raw}`;
-        }
-        if ( rejected ) {
-            compiled = rejected;
-        }
-        let details = context.specificCosmeticFilters.get(compiled);
         if ( details === undefined ) {
-            details = {};
-            if ( rejected ) { details.rejected = true; }
-            context.specificCosmeticFilters.set(compiled, details);
+            context.specificCosmeticFilters.set(compiled, details = {});
         }
-        if ( rejected ) { continue; }
-        if ( not ) {
+        if ( exception ) {
             if ( details.excludeMatches === undefined ) {
                 details.excludeMatches = [];
             }
@@ -270,6 +272,13 @@ function addExtendedToDNR(context, parser) {
         }
         details.matches.push(hn);
     }
+    if ( details === undefined ) { return; }
+    if ( exception ) { return; }
+    if ( compiled.startsWith('{') ) { return; }
+    if ( details.matches === undefined || details.matches.includes('*') ) {
+        addGenericCosmeticFilter(context, compiled, false);
+        details.matches = undefined;
+    }
 }
 
 /******************************************************************************/
@@ -284,6 +293,7 @@ function addToDNR(context, list) {
         toDNR: true,
         nativeCssHas: env.includes('native_css_has'),
         badTypes: [ sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE ],
+        trustedSource: list.trustedSource || undefined,
     });
     const compiler = staticNetFilteringEngine.createCompiler();
 
@@ -344,105 +354,109 @@ function addToDNR(context, list) {
 
 /******************************************************************************/
 
-function finalizeRuleset(context, network) {
-    const ruleset = network.ruleset;
+// Merge rules where possible by merging arrays of a specific property.
+//
+// https://github.com/uBlockOrigin/uBOL-home/issues/10#issuecomment-1304822579
+//   Do not merge rules which have errors.
 
-    // Assign rule ids
-    const rulesetMap = new Map();
-    {
-        let ruleId = 1;
-        for ( const rule of ruleset ) {
-            rulesetMap.set(ruleId++, rule);
+function mergeRules(rulesetMap, mergeTarget) {
+    const sorter = (_, v) => {
+        if ( Array.isArray(v) ) {
+            return typeof v[0] === 'string' ? v.sort() : v;
         }
-    }
-    // Merge rules where possible by merging arrays of a specific property.
-    //
-    // https://github.com/uBlockOrigin/uBOL-home/issues/10#issuecomment-1304822579
-    //   Do not merge rules which have errors.
-    const mergeRules = (rulesetMap, mergeTarget) => {
-        const mergeMap = new Map();
-        const sorter = (_, v) => {
-            if ( Array.isArray(v) ) {
-                return typeof v[0] === 'string' ? v.sort() : v;
+        if ( v instanceof Object ) {
+            const sorted = {};
+            for ( const kk of Object.keys(v).sort() ) {
+                sorted[kk] = v[kk];
             }
+            return sorted;
+        }
+        return v;
+    };
+    const ruleHasher = (rule, target) => {
+        return JSON.stringify(rule, (k, v) => {
+            if ( k.startsWith('_') ) { return; }
+            if ( k === target ) { return; }
+            return sorter(k, v);
+        });
+    };
+    const extractTargetValue = (obj, target) => {
+        for ( const [ k, v ] of Object.entries(obj) ) {
+            if ( Array.isArray(v) && k === target ) { return v; }
             if ( v instanceof Object ) {
-                const sorted = {};
-                for ( const kk of Object.keys(v).sort() ) {
-                    sorted[kk] = v[kk];
-                }
-                return sorted;
+                const r = extractTargetValue(v, target);
+                if ( r !== undefined ) { return r; }
             }
-            return v;
-        };
-        const ruleHasher = (rule, target) => {
-            return JSON.stringify(rule, (k, v) => {
-                if ( k.startsWith('_') ) { return; }
-                if ( k === target ) { return; }
-                return sorter(k, v);
-            });
-        };
-        const extractTargetValue = (obj, target) => {
-            for ( const [ k, v ] of Object.entries(obj) ) {
-                if ( Array.isArray(v) && k === target ) { return v; }
-                if ( v instanceof Object ) {
-                    const r = extractTargetValue(v, target);
-                    if ( r !== undefined ) { return r; }
-                }
-            }
-        };
-        const extractTargetOwner = (obj, target) => {
-            for ( const [ k, v ] of Object.entries(obj) ) {
-                if ( Array.isArray(v) && k === target ) { return obj; }
-                if ( v instanceof Object ) {
-                    const r = extractTargetOwner(v, target);
-                    if ( r !== undefined ) { return r; }
-                }
-            }
-        };
-        for ( const [ id, rule ] of rulesetMap ) {
-            if ( rule._error !== undefined ) { continue; }
-            const hash = ruleHasher(rule, mergeTarget);
-            if ( mergeMap.has(hash) === false ) {
-                mergeMap.set(hash, []);
+        }
+    };
+    const extractTargetOwner = (obj, target) => {
+        for ( const [ k, v ] of Object.entries(obj) ) {
+            if ( Array.isArray(v) && k === target ) { return obj; }
+            if ( v instanceof Object ) {
+                const r = extractTargetOwner(v, target);
+                if ( r !== undefined ) { return r; }
             }
-            mergeMap.get(hash).push(id);
         }
-        for ( const ids of mergeMap.values() ) {
-            if ( ids.length === 1 ) { continue; }
-            const leftHand = rulesetMap.get(ids[0]);
-            const leftHandSet = new Set(
-                extractTargetValue(leftHand, mergeTarget) || []
-            );
-            for ( let i = 1; i < ids.length; i++ ) {
-                const rightHandId = ids[i];
-                const rightHand = rulesetMap.get(rightHandId);
-                const rightHandArray =  extractTargetValue(rightHand, mergeTarget);
-                if ( rightHandArray !== undefined ) {
-                    if ( leftHandSet.size !== 0 ) {
-                        for ( const item of rightHandArray ) {
-                            leftHandSet.add(item);
-                        }
+    };
+    const mergeMap = new Map();
+    for ( const [ id, rule ] of rulesetMap ) {
+        if ( rule._error !== undefined ) { continue; }
+        const hash = ruleHasher(rule, mergeTarget);
+        if ( mergeMap.has(hash) === false ) {
+            mergeMap.set(hash, []);
+        }
+        mergeMap.get(hash).push(id);
+    }
+    for ( const ids of mergeMap.values() ) {
+        if ( ids.length === 1 ) { continue; }
+        const leftHand = rulesetMap.get(ids[0]);
+        const leftHandSet = new Set(
+            extractTargetValue(leftHand, mergeTarget) || []
+        );
+        for ( let i = 1; i < ids.length; i++ ) {
+            const rightHandId = ids[i];
+            const rightHand = rulesetMap.get(rightHandId);
+            const rightHandArray =  extractTargetValue(rightHand, mergeTarget);
+            if ( rightHandArray !== undefined ) {
+                if ( leftHandSet.size !== 0 ) {
+                    for ( const item of rightHandArray ) {
+                        leftHandSet.add(item);
                     }
-                } else {
-                    leftHandSet.clear();
                 }
-                rulesetMap.delete(rightHandId);
+            } else {
+                leftHandSet.clear();
             }
-            const leftHandOwner = extractTargetOwner(leftHand, mergeTarget);
-            if ( leftHandSet.size > 1 ) {
-                //if ( leftHandOwner === undefined ) { debugger; }
-                leftHandOwner[mergeTarget] = Array.from(leftHandSet).sort();
-            } else if ( leftHandSet.size === 0 ) {
-                if ( leftHandOwner !== undefined ) {
-                    leftHandOwner[mergeTarget] = undefined;
-                }
+            rulesetMap.delete(rightHandId);
+        }
+        const leftHandOwner = extractTargetOwner(leftHand, mergeTarget);
+        if ( leftHandSet.size > 1 ) {
+            //if ( leftHandOwner === undefined ) { debugger; }
+            leftHandOwner[mergeTarget] = Array.from(leftHandSet).sort();
+        } else if ( leftHandSet.size === 0 ) {
+            if ( leftHandOwner !== undefined ) {
+                leftHandOwner[mergeTarget] = undefined;
             }
         }
-    };
+    }
+}
+
+/******************************************************************************/
+
+function finalizeRuleset(context, network) {
+    const ruleset = network.ruleset;
+
+    // Assign rule ids
+    const rulesetMap = new Map();
+    {
+        let ruleId = 1;
+        for ( const rule of ruleset ) {
+            rulesetMap.set(ruleId++, rule);
+        }
+    }
     mergeRules(rulesetMap, 'resourceTypes');
+    mergeRules(rulesetMap, 'removeParams');
     mergeRules(rulesetMap, 'initiatorDomains');
     mergeRules(rulesetMap, 'requestDomains');
-    mergeRules(rulesetMap, 'removeParams');
     mergeRules(rulesetMap, 'responseHeaders');
 
     // Patch id
@@ -498,4 +512,4 @@ async function dnrRulesetFromRawLists(lists, options = {}) {
 
 /******************************************************************************/
 
-export { dnrRulesetFromRawLists };
+export { dnrRulesetFromRawLists, mergeRules };

package/extensions/ublock/js/static-filtering-parser.js

@@ -22,6 +22,7 @@
 /******************************************************************************/
 
 import * as cssTree from '../lib/csstree/css-tree.js';
+import { ArglistParser } from './arglist-parser.js';
 import Regex from '../lib/regexanalyzer/regex.js';
 
 /*******************************************************************************
@@ -606,102 +607,6 @@ const exCharCodeAt = (s, i) => {
 
 /******************************************************************************/
 
-class ArgListParser {
-    constructor(separatorChar = ',', mustQuote = false) {
-        this.separatorChar = this.actualSeparatorChar = separatorChar;
-        this.separatorCode = this.actualSeparatorCode = separatorChar.charCodeAt(0);
-        this.mustQuote = mustQuote;
-        this.quoteBeg = 0; this.quoteEnd = 0;
-        this.argBeg = 0; this.argEnd = 0;
-        this.separatorBeg = 0; this.separatorEnd = 0;
-        this.transform = false;
-        this.failed = false;
-        this.reWhitespaceStart = /^\s+/;
-        this.reWhitespaceEnd = /\s+$/;
-        this.reOddTrailingEscape = /(?:^|[^\\])(?:\\\\)*\\$/;
-        this.reTrailingEscapeChars = /\\+$/;
-    }
-    nextArg(pattern, beg = 0) {
-        const len = pattern.length;
-        this.quoteBeg = beg + this.leftWhitespaceCount(pattern.slice(beg));
-        this.failed = false;
-        const qc = pattern.charCodeAt(this.quoteBeg);
-        if ( qc === 0x22 /* " */ || qc === 0x27 /* ' */ || qc === 0x60 /* ` */ ) {
-            this.indexOfNextArgSeparator(pattern, qc);
-            if ( this.argEnd !== len ) {
-                this.quoteEnd = this.argEnd + 1;
-                this.separatorBeg = this.separatorEnd = this.quoteEnd;
-                this.separatorEnd += this.leftWhitespaceCount(pattern.slice(this.quoteEnd));
-                if ( this.separatorEnd === len ) { return this; }
-                if ( pattern.charCodeAt(this.separatorEnd) === this.separatorCode ) {
-                    this.separatorEnd += 1;
-                    return this;
-                }
-            }
-        }
-        this.indexOfNextArgSeparator(pattern, this.separatorCode);
-        this.separatorBeg = this.separatorEnd = this.argEnd;
-        if ( this.separatorBeg < len ) {
-            this.separatorEnd += 1;
-        }
-        this.argEnd -= this.rightWhitespaceCount(pattern.slice(0, this.separatorBeg));
-        this.quoteEnd = this.argEnd;
-        if ( this.mustQuote ) {
-            this.failed = true;
-        }
-        return this;
-    }
-    normalizeArg(s, char = '') {
-        if ( char === '' ) { char = this.actualSeparatorChar; }
-        let out = '';
-        let pos = 0;
-        while ( (pos = s.lastIndexOf(char)) !== -1 ) {
-            out = s.slice(pos) + out;
-            s = s.slice(0, pos);
-            const match = this.reTrailingEscapeChars.exec(s);
-            if ( match === null ) { continue; }
-            const tail = (match[0].length & 1) !== 0
-                ? match[0].slice(0, -1)
-                : match[0];
-            out = tail + out;
-            s = s.slice(0, -match[0].length);
-        }
-        if ( out === '' ) { return s; }
-        return s + out;
-    }
-    leftWhitespaceCount(s) {
-        const match = this.reWhitespaceStart.exec(s);
-        return match === null ? 0 : match[0].length;
-    }
-    rightWhitespaceCount(s) {
-        const match = this.reWhitespaceEnd.exec(s);
-        return match === null ? 0 : match[0].length;
-    }
-    indexOfNextArgSeparator(pattern, separatorCode) {
-        this.argBeg = this.argEnd = separatorCode !== this.separatorCode
-            ? this.quoteBeg + 1
-            : this.quoteBeg;
-        this.transform = false;
-        if ( separatorCode !== this.actualSeparatorCode ) {
-            this.actualSeparatorCode = separatorCode;
-            this.actualSeparatorChar = String.fromCharCode(separatorCode);
-        }
-        while ( this.argEnd < pattern.length ) {
-            const pos = pattern.indexOf(this.actualSeparatorChar, this.argEnd);
-            if ( pos === -1 ) {
-                return (this.argEnd = pattern.length);
-            }
-            if ( this.reOddTrailingEscape.test(pattern.slice(0, pos)) === false ) {
-                return (this.argEnd = pos);
-            }
-            this.transform = true;
-            this.argEnd = pos + 1;
-        }
-    }
-}
-
-/******************************************************************************/
-
 class AstWalker {
     constructor(parser, from = 0) {
         this.parser = parser;
@@ -904,8 +809,8 @@ export class AstFilterParser {
         this.reBadPP = /(?:^|[;,])\s*report-to\b/i;
         this.reNetOption = /^(~?)([134a-z_-]+)(=?)/;
         this.reNoopOption = /^_+$/;
-        this.netOptionValueParser = new ArgListParser(',');
-        this.scriptletArgListParser = new ArgListParser(',');
+        this.netOptionValueParser = new ArglistParser(',');
+        this.scriptletArgListParser = new ArglistParser(',');
     }
 
     finish() {
@@ -1572,18 +1477,15 @@ export class AstFilterParser {
         if ( j === -1 ) { return end; }
         if ( (j+1) === end ) { return end; }
         for (;;) {
-            const before = s.charCodeAt(j-1);
-            if ( j !== start && before === 0x24 /* $ */ ) { return -1; }
-            const after = s.charCodeAt(j+1);
-            if (
-                after !== 0x29 /* ) */ &&
-                after !== 0x2F /* / */ &&
-                after !== 0x7C /* | */ &&
-                before !== 0x5C /* \ */
-            ) {
-                return j;
+            const before = s.charAt(j-1);
+            if ( before === '$' ) { return -1; }
+            const after = s.charAt(j+1);
+            if ( ')/|'.includes(after) === false ) {
+                if ( before === '' || '"\'\\`'.includes(before) === false ) {
+                    return j;
+                }
             }
-            if ( j <= start ) { break; }
+            if ( j === start ) { break; }
             j = s.lastIndexOf('$', j-1);
             if ( j === -1 ) { break; }
         }
@@ -3100,7 +3002,7 @@ export function parseHeaderValue(arg) {
 
 export function parseReplaceValue(s) {
     if ( s.charCodeAt(0) !== 0x2F /* / */ ) { return; }
-    const parser = new ArgListParser('/');
+    const parser = new ArglistParser('/');
     parser.nextArg(s, 1);
     let pattern = s.slice(parser.argBeg, parser.argEnd);
     if ( parser.transform ) {
@@ -3345,10 +3247,13 @@ class ExtSelectorCompiler {
         //   We have an Adguard/ABP cosmetic filter if and only if the
         //   character is `$`, `%` or `?`, otherwise it's not a cosmetic
         //   filter.
-        // Adguard's style injection: translate to uBO's format.
-        if ( compileOptions.adgStyleSyntax === true ) {
-            raw = this.translateAdguardCSSInjectionFilter(raw);
-            if ( raw === '' ) { return false; }
+        // Adguard/EasyList style injection: translate to uBO's format.
+        if ( this.isStyleInjectionFilter(raw) ) {
+            const translated = this.translateStyleInjectionFilter(raw);
+            if ( translated === undefined ) { return false; }
+            raw = translated;
+        } else if ( compileOptions.adgStyleSyntax === true ) {
+            return false;
         }
 
         // Normalize AdGuard's attribute-based procedural operators.
@@ -3884,9 +3789,14 @@ class ExtSelectorCompiler {
         return true;
     }
 
-    translateAdguardCSSInjectionFilter(suffix) {
-        const matches = /^(.*)\s*\{([^}]+)\}\s*$/.exec(suffix);
-        if ( matches === null ) { return ''; }
+    isStyleInjectionFilter(selector) {
+        const len = selector.length;
+        return len !== 0 && selector.charCodeAt(len-1) === 0x7D /* } */;
+    }
+
+    translateStyleInjectionFilter(raw) {
+        const matches = /^(.+)\s*\{([^}]+)\}$/.exec(raw);
+        if ( matches === null ) { return; }
         const selector = matches[1].trim();
         const style = matches[2].trim();
         // Special style directive `remove: true` is converted into a