@browserless.io/browserless 2.23.0 → 2.24.0-beta-2

package/extensions/ublock/js/static-filtering-parser.js

@@ -1531,7 +1531,7 @@ export class AstFilterParser {
                 break;
             }
             const value = this.getNetOptionValue(NODE_TYPE_NET_OPTION_NAME_URLSKIP);
-            if ( value.startsWith('?') === false || value.length < 2 ) {
+            if ( value.length < 2 ) {
                 this.astError = AST_ERROR_OPTION_BADVALUE;
                 realBad = true;
             }
@@ -1988,6 +1988,7 @@ export class AstFilterParser {
         if ( parentEnd === parentBeg ) { return 0; }
         const s = this.getNodeString(parent);
         const optionsEnd = s.length;
+        const parseDetails = { node: 0, len: 0 };
         const head = this.allocHeadNode();
         let prev = head, next = 0;
         let optionBeg = 0, optionEnd = 0;
@@ -1997,11 +1998,11 @@ export class AstFilterParser {
                 parentBeg + optionBeg,
                 parentBeg + optionsEnd // open ended
             );
-            const { node: down, len: optionLen } = this.parseNetOption(next);
+            this.parseNetOption(next, parseDetails);
             // set next's end to down's end
-            optionEnd += optionLen;
+            optionEnd += parseDetails.len;
             this.nodes[next+NODE_END_INDEX] = parentBeg + optionEnd;
-            this.linkDown(next, down);
+            this.linkDown(next, parseDetails.node);
             prev = this.linkRight(prev, next);
             if ( optionEnd === optionsEnd ) { break; }
             optionBeg = optionEnd + 1;
@@ -2010,7 +2011,7 @@ export class AstFilterParser {
                 parentBeg + optionEnd,
                 parentBeg + optionBeg
             );
-            if ( optionLen === 0 || optionBeg === optionsEnd ) {
+            if ( parseDetails.len === 0 || optionBeg === optionsEnd ) {
                 this.addNodeFlags(next, NODE_FLAG_ERROR);
                 this.addFlags(AST_FLAG_HAS_ERROR);
             }
@@ -2023,7 +2024,7 @@ export class AstFilterParser {
         return this.throwHeadNode(head);
     }
 
-    parseNetOption(parent) {
+    parseNetOption(parent, parseDetails) {
         const parentBeg = this.nodes[parent+NODE_BEG_INDEX];
         const s = this.getNodeString(parent);
         const match = this.reNetOption.exec(s) || [];
@@ -2031,7 +2032,9 @@ export class AstFilterParser {
             this.addNodeFlags(parent, NODE_FLAG_ERROR);
             this.addFlags(AST_FLAG_HAS_ERROR);
             this.astError = AST_ERROR_OPTION_UNKNOWN;
-            return { node: 0, len: s.length };
+            parseDetails.node = 0;
+            parseDetails.len = s.length;
+            return;
         }
         const head = this.allocHeadNode();
         let prev = head, next = 0;
@@ -2073,7 +2076,9 @@ export class AstFilterParser {
         }
         prev = this.linkRight(prev, next);
         if ( assigned === false ) {
-            return { node: this.throwHeadNode(head), len: matchEnd };
+            parseDetails.node = this.throwHeadNode(head);
+            parseDetails.len = matchEnd;
+            return;
         }
         next = this.allocTypedNode(
             NODE_TYPE_NET_OPTION_ASSIGN,
@@ -2129,7 +2134,8 @@ export class AstFilterParser {
             );
             this.linkRight(prev, next);
         }
-        return { node: this.throwHeadNode(head), len: details.quoteEnd };
+        parseDetails.node = this.throwHeadNode(head);
+        parseDetails.len = details.quoteEnd;
     }
 
     endOfNetOption(s, beg) {
@@ -2156,32 +2162,26 @@ export class AstFilterParser {
         );
         if ( parentEnd === parentBeg ) { return containerNode; }
         const separatorCode = separator.charCodeAt(0);
+        const parseDetails = { separator, mode, node: 0, len: 0 };
         const listNode = this.allocHeadNode();
         let prev = listNode;
         let domainNode = 0;
         let separatorNode = 0;
         const s = this.getNodeString(parent);
         const listEnd = s.length;
-        let beg = 0, end = 0, c = 0;
+        let beg = 0, end = 0;
         while ( beg < listEnd ) {
-            c = s.charCodeAt(beg);
-            if ( c === 0x7E /* ~ */ ) {
-                c = s.charCodeAt(beg+1) || 0;
-            }
-            if ( c !== 0x2F /* / */ ) {
-                end = s.indexOf(separator, beg);
-            } else {
-                end = s.indexOf('/', beg+1);
-                end = s.indexOf(separator, end !== -1 ? end+1 : beg);
-            }
-            if ( end === -1 ) { end = listEnd; }
+            const next = this.allocTypedNode(
+                NODE_TYPE_OPTION_VALUE_DOMAIN_RAW,
+                parentBeg + beg,
+                parentBeg + listEnd // open ended
+            );
+            this.parseDomain(next, parseDetails);
+            end = beg + parseDetails.len;
+            this.nodes[next+NODE_END_INDEX] = parentBeg + end;
             if ( end !== beg ) {
-                domainNode = this.allocTypedNode(
-                    NODE_TYPE_OPTION_VALUE_DOMAIN_RAW,
-                    parentBeg + beg,
-                    parentBeg + end
-                );
-                this.linkDown(domainNode, this.parseDomain(domainNode, mode));
+                domainNode = next;
+                this.linkDown(domainNode, parseDetails.node);
                 prev = this.linkRight(prev, domainNode);
             } else {
                 domainNode = 0;
@@ -2217,23 +2217,38 @@ export class AstFilterParser {
         return containerNode;
     }
 
-    parseDomain(parent, mode = 0b0000) {
+    parseDomain(parent, parseDetails) {
         const parentBeg = this.nodes[parent+NODE_BEG_INDEX];
         const parentEnd = this.nodes[parent+NODE_END_INDEX];
+        const not = this.charCodeAt(parentBeg) === 0x7E /* ~ */;
         let head = 0, next = 0;
         let beg = parentBeg;
-        const c = this.charCodeAt(beg);
-        if ( c === 0x7E /* ~ */ ) {
+        if ( not ) {
             this.addNodeFlags(parent, NODE_FLAG_IS_NEGATED);
             head = this.allocTypedNode(NODE_TYPE_OPTION_VALUE_NOT, beg, beg + 1);
-            if ( (mode & 0b1000) === 0 ) {
+            if ( (parseDetails.mode & 0b1000) === 0 ) {
                 this.addNodeFlags(parent, NODE_FLAG_ERROR);
             }
             beg += 1;
         }
-        if ( beg !== parentEnd ) {
-            next = this.allocTypedNode(NODE_TYPE_OPTION_VALUE_DOMAIN, beg, parentEnd);
-            const hn = this.normalizeDomainValue(this.getNodeString(next), mode);
+        const c0 = this.charCodeAt(beg);
+        let end = beg;
+        let type = 0;
+        if ( c0 === 0x2F /* / */ ) {
+            end = this.indexOf('/', beg + 1, parentEnd);
+            if ( end !== -1 ) { end += 1; }
+            type = 1;
+        } else if ( c0 === 0x5B /* [ */ && this.startsWith('[$domain=/', beg) ) {
+            end = this.indexOf('/]', beg + 10, parentEnd);
+            if ( end !== -1 ) { end += 2; }
+            type = 2;
+        } else {
+            end = this.indexOf(parseDetails.separator, end, parentEnd);
+        }
+        if ( end === -1 ) { end = parentEnd; }
+        if ( beg !== end ) {
+            next = this.allocTypedNode(NODE_TYPE_OPTION_VALUE_DOMAIN, beg, end);
+            const hn = this.normalizeDomainValue(next, type, parseDetails.mode);
             if ( hn !== undefined ) {
                 if ( hn !== '' ) {
                     this.setNodeTransform(next, hn);
@@ -2252,7 +2267,8 @@ export class AstFilterParser {
             this.addNodeFlags(parent, NODE_FLAG_ERROR);
             this.addFlags(AST_FLAG_HAS_ERROR);
         }
-        return head;
+        parseDetails.node = head;
+        parseDetails.len = end - parentBeg;
     }
 
     // mode bits:
@@ -2261,16 +2277,16 @@ export class AstFilterParser {
     //   0b00100: can use single wildcard
     //   0b01000: can be negated
     //   0b10000: can be a regex
-    normalizeDomainValue(s, modeBits) {
-        if ( (modeBits & 0b10000) === 0 ||
-            s.length <= 2 ||
-            s.charCodeAt(0) !== 0x2F /* / */ ||
-            exCharCodeAt(s, -1) !== 0x2F /* / */
-        ) {
+    normalizeDomainValue(node, type, modeBits) {
+        const s = this.getNodeString(node);
+        if ( type === 0 ) {
             return this.normalizeHostnameValue(s, modeBits);
         }
-        const source = this.normalizeRegexPattern(s);
+        if ( (modeBits & 0b10000) === 0 ) { return ''; }
+        const regex = type === 1 ? s : `/${s.slice(10, -2)}/`;
+        const source = this.normalizeRegexPattern(regex);
         if ( source === '' ) { return ''; }
+        if ( type === 1 && source === regex ) { return; }
         return `/${source}/`;
     }
 
@@ -2857,6 +2873,15 @@ export class AstFilterParser {
         return pos < this.rawEnd ? this.raw.charCodeAt(pos) : -1;
     }
 
+    indexOf(needle, beg, end = 0) {
+        const haystack = end === 0 ? this.raw : this.raw.slice(0, end);
+        return haystack.indexOf(needle, beg);
+    }
+
+    startsWith(s, pos) {
+        return pos < this.rawEnd && this.raw.startsWith(s, pos);
+    }
+
     isTokenCharCode(c) {
         return c === 0x25 ||
             c >= 0x30 && c <= 0x39 ||
@@ -4116,9 +4141,11 @@ class ExtSelectorCompiler {
     compileXpathExpression(s) {
         const r = this.unquoteString(s);
         if ( r.i !== s.length ) { return; }
-        if ( globalThis.document instanceof Object === false ) { return r.s; }
+        const doc = globalThis.document;
+        if ( doc instanceof Object === false ) { return r.s; }
         try {
-            globalThis.document.createExpression(r.s, null);
+            const expr = doc.createExpression(r.s, null);
+            expr.evaluate(doc, XPathResult.ANY_UNORDERED_NODE_TYPE);
         } catch (e) {
             return;
         }
@@ -4362,7 +4389,7 @@ export const utils = (( ) => {
         // only ABP.
         [ 'ext_abp', 'false' ],
         // Compatibility with other blockers
-        // https://kb.adguard.com/en/general/how-to-create-your-own-ad-filters#adguard-specific
+        // https://adguard.com/kb/general/ad-filtering/create-own-filters/#conditions-directive
         [ 'adguard', 'adguard' ],
         [ 'adguard_app_android', 'false' ],
         [ 'adguard_app_ios', 'false' ],
@@ -4430,14 +4457,14 @@ export const utils = (( ) => {
             const parts = [ 0 ];
             let discard = false;
 
-            const shouldDiscard = ( ) => stack.some(v => v);
+            const shouldDiscard = ( ) => stack.some(v => v.known && v.discard);
 
-            const begif = (startDiscard, match) => {
-                if ( discard === false && startDiscard ) {
-                    parts.push(match.index);
+            const begif = details => {
+                if ( discard === false && details.known && details.discard ) {
+                    parts.push(details.pos);
                     discard = true;
                 }
-                stack.push(startDiscard);
+                stack.push(details);
             };
 
             const endif = match => {
@@ -4455,15 +4482,21 @@ export const utils = (( ) => {
 
                 switch ( match[1] ) {
                 case 'if': {
-                    const startDiscard = this.evaluateExpr(match[2].trim(), env) === false;
-                    begif(startDiscard, match);
+                    const result = this.evaluateExpr(match[2].trim(), env);
+                    begif({
+                        known: result !== undefined,
+                        discard: result === false,
+                        pos: match.index,
+                    });
                     break;
                 }
                 case 'else': {
                     if ( stack.length === 0 ) { break; }
-                    const startDiscard = stack[stack.length-1] === false;
+                    const details = stack[stack.length-1];
                     endif(match);
-                    begif(startDiscard, match);
+                    details.discard = details.discard === false;
+                    details.pos = match.index;
+                    begif(details);
                     break;
                 }
                 case 'endif': {

package/extensions/ublock/js/static-net-filtering.js

@@ -3048,9 +3048,6 @@ class FilterIPAddress {
             if ( ipaddr.startsWith('::ffff:') === false ) { return false; }
             return this.reIPv6IPv4lan.test(ipaddr);
         }
-        if ( c0 === 0x36 /* 6 */ ) {
-            return ipaddr.startsWith('64:ff9b:');
-        }
         if ( c0 === 0x66 /* f */ ) {
             return this.reIPv6local.test(ipaddr);
         }
@@ -4789,20 +4786,13 @@ StaticNetFilteringEngine.prototype.toSelfie = function() {
     };
 };
 
-StaticNetFilteringEngine.prototype.serialize = async function() {
-    const selfie = [];
-    const storage = {
-        put(name, data) {
-            selfie.push([ name, data ]);
-        }
-    };
-    await this.toSelfie(storage, '');
-    return JSON.stringify(selfie);
+StaticNetFilteringEngine.prototype.serialize = function() {
+    return this.toSelfie();
 };
 
 /******************************************************************************/
 
-StaticNetFilteringEngine.prototype.fromSelfie = async function(selfie) {
+StaticNetFilteringEngine.prototype.fromSelfie = function(selfie) {
     if ( typeof selfie !== 'object' || selfie === null ) { return; }
 
     this.reset();
@@ -4835,14 +4825,8 @@ StaticNetFilteringEngine.prototype.fromSelfie = async function(selfie) {
     return true;
 };
 
-StaticNetFilteringEngine.prototype.unserialize = async function(s) {
-    const selfie = new Map(JSON.parse(s));
-    const storage = {
-        async get(name) {
-            return { content: selfie.get(name) };
-        }
-    };
-    return this.fromSelfie(storage, '');
+StaticNetFilteringEngine.prototype.unserialize = function(selfie) {
+    return this.fromSelfie(selfie);
 };
 
 /******************************************************************************/
@@ -5369,12 +5353,10 @@ StaticNetFilteringEngine.prototype.transformRequest = function(fctxt, out = [])
             out.push(directive);
             continue;
         }
-        const { refs } = directive;
-        if ( refs instanceof Object === false ) { continue; }
-        if ( refs.$cache === null ) {
-            refs.$cache = sfp.parseReplaceValue(refs.value);
+        if ( directive.cache === null ) {
+            directive.cache = sfp.parseReplaceValue(directive.value);
         }
-        const cache = refs.$cache;
+        const cache = directive.cache;
         if ( cache === undefined ) { continue; }
         const before = `${redirectURL.pathname}${redirectURL.search}${redirectURL.hash}`;
         if ( cache.re.test(before) !== true ) { continue; }
@@ -5395,9 +5377,60 @@ StaticNetFilteringEngine.prototype.transformRequest = function(fctxt, out = [])
     return out;
 };
 
-/******************************************************************************/
-
-StaticNetFilteringEngine.prototype.urlSkip = function(fctxt, out = []) {
+/**
+ * @trustedOption urlskip
+ * 
+ * @description
+ * Extract a URL from another URL according to one or more transformation steps,
+ * thereby skipping over intermediate network request(s) to remote servers.
+ * Requires a trusted source.
+ * 
+ * @param steps
+ * A serie of space-separated directives representing the transformation steps
+ * to perform to extract the final URL to which a network request should be
+ * redirected.
+ * 
+ * Supported directives:
+ * 
+ * `?name`: extract the value of parameter `name` as the current string.
+ * 
+ * `&i`: extract the name of the parameter at position `i` as the current
+ *   string. The position is 1-based.
+ * 
+ * `/.../`: extract the first capture group of a regex as the current string.
+ * 
+ * `+https`: prepend the current string with `https://`.
+ * 
+ * `-base64`: decode the current string as a base64-encoded string.
+ * 
+ * `-uricomponent`: decode the current string as a URI encoded string.
+ * 
+ * `-blocked`: allow the redirection of blocked requests. By default, blocked
+ *  requests can't by urlskip'ed.
+ * 
+ * At any given step, the currently extracted string may not necessarily be
+ * a valid URL, and more transformation steps may be needed to obtain a valid
+ * URL once all the steps are applied.
+ * 
+ * An unsupported step or a failed step will abort the transformation and no
+ * redirection will be performed.
+ * 
+ * The final step is expected to yield a valid URL. If the result is not a
+ * valid URL, no redirection will be performed.
+ * 
+ * @example
+ * ||example.com/path/to/tracker$urlskip=?url
+ * ||example.com/path/to/tracker$urlskip=?url ?to
+ * ||pixiv.net/jump.php?$urlskip=&1
+ * ||podtrac.com/pts/redirect.mp3/$urlskip=/\/redirect\.mp3\/(.*?\.mp3\b)/ +https
+ * 
+ * */
+
+StaticNetFilteringEngine.prototype.urlSkip = function(
+    fctxt,
+    blocked,
+    out = []
+) {
     if ( fctxt.redirectURL !== undefined ) { return; }
     const directives = this.matchAndFetchModifiers(fctxt, 'urlskip');
     if ( directives === undefined ) { return; }
@@ -5409,7 +5442,7 @@ StaticNetFilteringEngine.prototype.urlSkip = function(fctxt, out = []) {
         const urlin = fctxt.url;
         const value = directive.value;
         const steps = value.includes(' ') && value.split(/ +/) || [ value ];
-        const urlout = urlSkip(urlin, steps);
+        const urlout = urlSkip(directive, urlin, blocked, steps);
         if ( urlout === undefined ) { continue; }
         if ( urlout === urlin ) { continue; }
         fctxt.redirectURL = urlout;
@@ -5420,31 +5453,74 @@ StaticNetFilteringEngine.prototype.urlSkip = function(fctxt, out = []) {
     return out;
 };
 
-function urlSkip(urlin, steps) {
+function urlSkip(directive, url, blocked, steps) {
     try {
-        let urlout;
+        let redirectBlocked = false;
+        let urlout = url;
         for ( const step of steps ) {
+            const urlin = urlout;
+            const c0 = step.charCodeAt(0);
+            // Extract from URL parameter name at position i
+            if ( c0 === 0x26 ) { // &
+                const i = (parseInt(step.slice(1)) || 0) - 1;
+                if ( i < 0 ) { return; }
+                const url = new URL(urlin);
+                if ( i >= url.searchParams.size ) { return; }
+                const params = Array.from(url.searchParams.keys());
+                urlout = decodeURIComponent(params[i]);
+                continue;
+            }
+            // Enforce https
+            if ( c0 === 0x2B && step === '+https' ) {
+                const s = urlin.replace(/^https?:\/\//, '');
+                if ( /^[\w-]:\/\//.test(s) ) { return; }
+                urlout = `https://${s}`;
+                continue;
+            }
+            // Decode
+            if ( c0 === 0x2D ) {
+                // Base64
+                if ( step === '-base64' ) {
+                    urlout = self.atob(urlin);
+                    continue;
+                }
+                // URI component
+                if ( step === '-uricomponent' ) {
+                    urlout = self.decodeURIComponent(urlin);
+                    continue;
+                }
+                // Enable skip of blocked requests
+                if ( step === '-blocked' ) {
+                    redirectBlocked = true;
+                    continue;
+                }
+            }
+            // Regex extraction from first capture group
+            if ( c0 === 0x2F ) { // /
+                if ( directive.cache === null ) {
+                    directive.cache = new RegExp(step.slice(1, -1));
+                }
+                const match = directive.cache.exec(urlin);
+                if ( match === null ) { return; }
+                if ( match.length <= 1 ) { return; }
+                urlout = match[1];
+                continue;
+            }
             // Extract from URL parameter
-            if ( step.startsWith('?') ) {
+            if ( c0 === 0x3F ) { // ?
                 urlout = (new URL(urlin)).searchParams.get(step.slice(1));
                 if ( urlout === null ) { return; }
                 if ( urlout.includes(' ') ) {
                     urlout = urlout.replace(/ /g, '%20');
                 }
-                urlin = urlout;
-                continue;
-            }
-            // Enforce https
-            if ( step === '+https' ) {
-                const s = urlin.replace(/^https?:\/\//, '');
-                if ( /^[\w-]:\/\//.test(s) ) { return; }
-                urlin = urlout = `https://${s}`;
                 continue;
             }
             // Unknown directive
             return;
         }
-        void new URL(urlout);
+        const urlfinal = new URL(urlout);
+        if ( urlfinal.protocol !== 'https:' ) { return; }
+        if ( blocked && redirectBlocked !== true ) { return; }
         return urlout;
     } catch(x) {
     }

package/extensions/ublock/js/traffic.js

@@ -196,8 +196,9 @@ const onBeforeRootFrameRequest = function(fctxt) {
     if ( trusted === false && pageStore !== null ) {
         if ( result !== 1 ) {
             pageStore.redirectNonBlockedRequest(fctxt);
+        } else {
+            pageStore.skipMainDocument(fctxt, true);
         }
-        pageStore.skipMainDocument(fctxt);
     }
 
     if ( logger.enabled ) {
@@ -215,16 +216,20 @@ const onBeforeRootFrameRequest = function(fctxt) {
     if ( result !== 1 ) { return; }
 
     // No log data means no strict blocking (because we need to report why
-    // the blocking occurs.
+    // the blocking occurs
     if ( logData === undefined  ) { return; }
 
     // Blocked
 
+    // Find out the URL navigated to should the document not be strict-blocked
+    pageStore.skipMainDocument(fctxt, false);
+
     const query = encodeURIComponent(JSON.stringify({
         url: requestURL,
-        hn: requestHostname,
         dn: fctxt.getDomain() || requestHostname,
-        fs: logData.raw
+        fs: logData.raw,
+        hn: requestHostname,
+        to: fctxt.redirectURL || '',
     }));
 
     vAPI.tabs.replace(
@@ -488,7 +493,7 @@ const onHeadersReceived = function(details) {
     }
     if ( pageStore.getNetFilteringSwitch(fctxt) === false ) { return; }
 
-    if ( fctxt.itype === fctxt.IMAGE || fctxt.itype === fctxt.MEDIA ) {
+    if ( (fctxt.itype & foilLargeMediaElement.TYPE_BITS) !== 0 ) {
         const result = foilLargeMediaElement(details, fctxt, pageStore);
         if ( result !== undefined ) { return result; }
     }
@@ -1124,15 +1129,12 @@ const injectPP = function(fctxt, pageStore, responseHeaders) {
 const foilLargeMediaElement = function(details, fctxt, pageStore) {
     if ( details.fromCache === true ) { return; }
 
-    let size = 0;
-    if ( µb.userSettings.largeMediaSize !== 0 ) {
-        const headers = details.responseHeaders;
-        const i = headerIndexFromName('content-length', headers);
-        if ( i === -1 ) { return; }
-        size = parseInt(headers[i].value, 10) || 0;
-    }
+    onDemandHeaders.setHeaders(details.responseHeaders);
+
+    const result = pageStore.filterLargeMediaElement(fctxt, onDemandHeaders);
+
+    onDemandHeaders.reset();
 
-    const result = pageStore.filterLargeMediaElement(fctxt, size);
     if ( result === 0 ) { return; }
 
     if ( logger.enabled ) {
@@ -1142,16 +1144,15 @@ const foilLargeMediaElement = function(details, fctxt, pageStore) {
     return { cancel: true };
 };
 
+foilLargeMediaElement.TYPE_BITS = fc.IMAGE | fc.MEDIA | fc.XMLHTTPREQUEST;
+
 /******************************************************************************/
 
 // Caller must ensure headerName is normalized to lower case.
 
 const headerIndexFromName = function(headerName, headers) {
-    let i = headers.length;
-    while ( i-- ) {
-        if ( headers[i].name.toLowerCase() === headerName ) {
-            return i;
-        }
+    for ( let i = 0, n = headers.length; i < n; i++ ) {
+        if ( headers[i].name.toLowerCase() === headerName ) { return i; }
     }
     return -1;
 };
@@ -1161,6 +1162,24 @@ const headerValueFromName = function(headerName, headers) {
     return i !== -1 ? headers[i].value : '';
 };
 
+const onDemandHeaders = {
+    headers: [],
+    get contentLength() {
+        const contentLength = headerValueFromName('content-length', this.headers);
+        if ( contentLength === '' ) { return Number.NaN; }
+        return parseInt(contentLength, 10) || 0;
+    },
+    get contentType() {
+        return headerValueFromName('content-type', this.headers);
+    },
+    setHeaders(headers) {
+        this.headers = headers;
+    },
+    reset() {
+        this.headers = [];
+    }
+};
+
 /******************************************************************************/
 
 const strictBlockBypasser = {