@browserless.io/browserless 2.0.0-beta-5 → 2.0.0-beta-7

package/extensions/ublock/js/assets.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2014-present Raymond Hill
 
     This program is free software: you can redistribute it and/or modify
@@ -23,11 +23,12 @@
 
 /******************************************************************************/
 
-import cacheStorage from './cachestorage.js';
-import logger from './logger.js';
 import µb from './background.js';
+import { broadcast } from './broadcast.js';
+import cacheStorage from './cachestorage.js';
 import { ubolog } from './console.js';
 import { i18n$ } from './i18n.js';
+import logger from './logger.js';
 import * as sfp from './static-filtering-parser.js';
 import { orphanizeString, } from './text-utils.js';
 
@@ -38,6 +39,7 @@ const reIsUserAsset = /^user-/;
 const errorCantConnectTo = i18n$('errorCantConnectTo');
 const MS_PER_HOUR = 60 * 60 * 1000;
 const MS_PER_DAY = 24 * MS_PER_HOUR;
+const MINUTES_PER_DAY = 24 * 60;
 const EXPIRES_DEFAULT = 7;
 
 const assets = {};
@@ -52,7 +54,7 @@ const stringIsNotEmpty = s => typeof s === 'string' && s !== '';
 
 const parseExpires = s => {
     const matches = s.match(/(\d+)\s*([dhm]?)/i);
-    if ( matches === null ) { return 0; }
+    if ( matches === null ) { return; }
     let updateAfter = parseInt(matches[1], 10);
     if ( matches[2] === 'h' ) {
         updateAfter = Math.max(updateAfter, 4) / 24;
@@ -93,31 +95,11 @@ const extractMetadataFromList = (content, fields) => {
 assets.extractMetadataFromList = extractMetadataFromList;
 
 const resourceTimeFromXhr = xhr => {
-    try {
-        // First lookup timestamp from content
-        let assetTime = 0;
-        if ( typeof xhr.response === 'string' ) {
-            const metadata = extractMetadataFromList(xhr.response, [
-                'Last-Modified'
-            ]);
-            assetTime = metadata.lastModified || 0;
-        }
-        // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Age
-        // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Date
-        let networkTime = 0;
-        if ( assetTime === 0 ) {
-            const age = parseInt(xhr.getResponseHeader('Age'), 10);
-            if ( isNaN(age) === false ) {
-                const time = (new Date(xhr.getResponseHeader('Date'))).getTime();
-                if ( isNaN(time) === false ) {
-                    networkTime = time - age * 1000;
-                }
-            }
-        }
-        return Math.max(assetTime, networkTime, 0);
-    } catch(_) {
-    }
-    return 0;
+    if ( typeof xhr.response !== 'string' ) {  return 0; }
+    const metadata = extractMetadataFromList(xhr.response, [
+        'Last-Modified'
+    ]);
+    return metadata.lastModified || 0;
 };
 
 const resourceTimeFromParts = (parts, time) => {
@@ -174,6 +156,23 @@ const isDiffUpdatableAsset = content => {
         data.diffPath.startsWith('%') === false;
 };
 
+const computedPatchUpdateTime = assetKey => {
+    const entry = assetCacheRegistry[assetKey];
+    if ( entry === undefined ) { return 0; }
+    if ( typeof entry.diffPath !== 'string' ) { return 0; }
+    if ( typeof entry.diffExpires !== 'number' ) { return 0; }
+    const match = /(\d+)\.(\d+)\.(\d+)\.(\d+)/.exec(entry.diffPath);
+    if ( match === null ) { return getWriteTime(); }
+    const date = new Date();
+    date.setUTCFullYear(
+        parseInt(match[1], 10),
+        parseInt(match[2], 10) - 1,
+        parseInt(match[3], 10)
+    );
+    date.setUTCHours(0, parseInt(match[4], 10) + entry.diffExpires * MINUTES_PER_DAY, 0, 0);
+    return date.getTime();
+};
+
 /******************************************************************************/
 
 // favorLocal: avoid making network requests whenever possible
@@ -197,14 +196,14 @@ const getContentURLs = (assetKey, options = {}) => {
             return 0;
         });
     }
-    if ( Array.isArray(entry.cdnURLs) ) {
+    if ( options.favorOrigin !== true && Array.isArray(entry.cdnURLs) ) {
         const cdnURLs = entry.cdnURLs.slice();
         for ( let i = 0, n = cdnURLs.length; i < n; i++ ) {
             const j = Math.floor(Math.random() * n);
             if ( j === i ) { continue; }
             [ cdnURLs[j], cdnURLs[i] ] = [ cdnURLs[i], cdnURLs[j] ];
         }
-        if ( options.favorLocal || options.favorOrigin ) {
+        if ( options.favorLocal ) {
             contentURLs.push(...cdnURLs);
         } else {
             contentURLs.unshift(...cdnURLs);
@@ -984,6 +983,17 @@ assets.get = async function(assetKey, options = {}) {
                 silent: options.silent === true,
             });
             registerAssetSource(assetKey, { error: undefined });
+            if ( assetDetails.content === 'filters' ) {
+                const metadata = extractMetadataFromList(details.content, [
+                    'Last-Modified',
+                    'Expires',
+                    'Diff-Name',
+                    'Diff-Path',
+                    'Diff-Expires',
+                ]);
+                metadata.diffUpdated = undefined;
+                assetCacheSetDetails(assetKey, metadata);
+            }
         }
         return reportBack(details.content, contentURL);
     }
@@ -1038,8 +1048,10 @@ async function getRemote(assetKey, options = {}) {
         error = undefined;
 
         // If fetched resource is older than cached one, ignore
-        stale = resourceIsStale(result, cacheDetails);
-        if ( stale ) { continue; }
+        if ( options.favorOrigin !== true ) {
+            stale = resourceIsStale(result, cacheDetails);
+            if ( stale ) { continue; }
+        }
 
         // Success
         assetCacheWrite(assetKey, {
@@ -1056,6 +1068,7 @@ async function getRemote(assetKey, options = {}) {
                 'Diff-Path',
                 'Diff-Expires',
             ]);
+            metadata.diffUpdated = undefined;
             assetCacheSetDetails(assetKey, metadata);
         }
 
@@ -1112,6 +1125,9 @@ assets.metadata = async function() {
             const obsoleteAfter = cacheEntry.writeTime + getUpdateAfterTime(assetKey);
             assetEntry.obsolete = obsoleteAfter < now;
             assetEntry.remoteURL = cacheEntry.remoteURL;
+            if ( cacheEntry.diffUpdated ) {
+                assetEntry.diffUpdated = cacheEntry.diffUpdated;
+            }
         } else if (
             assetEntry.contentURL &&
             assetEntry.contentURL.length !== 0
@@ -1153,7 +1169,7 @@ assets.getUpdateAges = async function(conditions = {}) {
         out.push({
             assetKey,
             age,
-            ageNormalized: age / getUpdateAfterTime(assetKey),
+            ageNormalized: age / Math.max(1, getUpdateAfterTime(assetKey)),
         });
     }
     return out;
@@ -1188,8 +1204,20 @@ const getAssetDiffDetails = assetKey => {
     out.writeTime = cacheEntry.writeTime;
     const assetEntry = assetSourceRegistry[assetKey];
     if ( assetEntry === undefined ) { return; }
-    if ( Array.isArray(assetEntry.cdnURLs) === false ) { return; }
-    out.cdnURLs = assetEntry.cdnURLs.slice();
+    if ( assetEntry.content !== 'filters' ) { return; }
+    if ( Array.isArray(assetEntry.cdnURLs) ) {
+        out.cdnURLs = assetEntry.cdnURLs.slice();
+    } else if ( reIsExternalPath.test(assetKey) ) {
+        out.cdnURLs = [ assetKey ];
+    } else if ( typeof assetEntry.contentURL === 'string' ) {
+        out.cdnURLs = [ assetEntry.contentURL ];
+    } else if ( Array.isArray(assetEntry.contentURL) ) {
+        out.cdnURLs = assetEntry.contentURL.slice(0).filter(url =>
+            reIsExternalPath.test(url)
+        );
+    }
+    if ( Array.isArray(out.cdnURLs) === false ) { return; }
+    if ( out.cdnURLs.length === 0 ) { return; }
     return out;
 };
 
@@ -1205,12 +1233,13 @@ async function diffUpdater() {
         const assetDetails = getAssetDiffDetails(assetKey);
         if ( assetDetails === undefined ) { continue; }
         assetDetails.what = 'update';
-        if ( (getWriteTime(assetKey) + assetDetails.diffExpires) > now ) {
-            assetDetails.fetch = false;
-            toSoftUpdate.push(assetDetails);
-        } else {
+        const computedUpdateTime = computedPatchUpdateTime(assetKey);
+        if ( computedUpdateTime !== 0 && computedUpdateTime <= now ) {
             assetDetails.fetch = true;
             toHardUpdate.push(assetDetails);
+        } else {
+            assetDetails.fetch = false;
+            toSoftUpdate.push(assetDetails);
         }
     }
     if ( toHardUpdate.length === 0 ) { return; }
@@ -1234,7 +1263,7 @@ async function diffUpdater() {
             assetCacheSetDetails(data.assetKey, metadata);
         };
         bc.onmessage = ev => {
-            const data = ev.data;
+            const data = ev.data || {};
             if ( data.what === 'ready' ) {
                 ubolog('Diff updater: hard updating', toHardUpdate.map(v => v.assetKey).join());
                 while ( toHardUpdate.length !== 0 ) {
@@ -1272,6 +1301,7 @@ async function diffUpdater() {
                     content: data.text,
                     resourceTime: metadata.lastModified || 0,
                 });
+                metadata.diffUpdated = true;
                 assetCacheSetDetails(data.assetKey, metadata);
                 updaterUpdated.push(data.assetKey);
             } else if ( data.error ) {
@@ -1279,7 +1309,7 @@ async function diffUpdater() {
             } else if ( data.status === 'nopatch-yet' || data.status === 'nodiff' ) {
                 ubolog(`Diff updater: skip update of ${data.assetKey} using ${data.patchPath}\n\treason: ${data.status}`);
                 assetCacheSetDetails(data.assetKey, { writeTime: data.writeTime });
-                vAPI.messaging.broadcast({
+                broadcast({
                     what: 'assetUpdated',
                     key: data.assetKey,
                     cached: true,
@@ -1415,8 +1445,8 @@ function updateDone() {
 
 assets.updateStart = function(details) {
     const oldUpdateDelay = updaterAssetDelay;
-    const newUpdateDelay = typeof details.delay === 'number'
-        ? details.delay
+    const newUpdateDelay = typeof details.fetchDelay === 'number'
+        ? details.fetchDelay
         : updaterAssetDelayDefault;
     updaterAssetDelay = Math.min(oldUpdateDelay, newUpdateDelay);
     updaterAuto = details.auto === true;

package/extensions/ublock/js/background.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2014-present Raymond Hill
 
     This program is free software: you can redistribute it and/or modify
@@ -144,10 +144,12 @@ if ( vAPI.webextFlavor.soup.has('firefox') ) {
 }
 
 const µBlock = {  // jshint ignore:line
-    userSettingsDefault: userSettingsDefault,
+    wakeupReason: '',
+
+    userSettingsDefault,
     userSettings: Object.assign({}, userSettingsDefault),
 
-    hiddenSettingsDefault: hiddenSettingsDefault,
+    hiddenSettingsDefault,
     hiddenSettingsAdmin: {},
     hiddenSettings: Object.assign({}, hiddenSettingsDefault),
 
@@ -268,6 +270,10 @@ const µBlock = {  // jshint ignore:line
     uiAccentStylesheet: '',
 };
 
+µBlock.isReadyPromise = new Promise(resolve => {
+    µBlock.isReadyResolve = resolve;
+});
+
 µBlock.domainFromHostname = domainFromHostname;
 µBlock.hostnameFromURI = hostnameFromURI;
 

package/extensions/ublock/js/base64-custom.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2014-present Raymond Hill
 
     This program is free software: you can redistribute it and/or modify

package/extensions/ublock/js/benchmarks.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2014-present Raymond Hill
 
     This program is free software: you can redistribute it and/or modify

package/extensions/ublock/js/biditrie.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2019-present Raymond Hill
 
     This program is free software: you can redistribute it and/or modify

package/extensions/ublock/js/broadcast.js

@@ -0,0 +1,75 @@
+/*******************************************************************************
+
+    uBlock Origin - a comprehensive, efficient content blocker
+    Copyright (C) 2014-present Raymond Hill
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see {http://www.gnu.org/licenses/}.
+
+    Home: https://github.com/gorhill/uBlock
+*/
+
+/* globals browser */
+
+'use strict';
+
+/******************************************************************************/
+
+// Broadcast a message to all uBO contexts
+
+let broadcastChannel;
+
+export function broadcast(message) {
+    if ( broadcastChannel === undefined ) {
+        broadcastChannel = new self.BroadcastChannel('uBO');
+    }
+    broadcastChannel.postMessage(message);
+}
+
+/******************************************************************************/
+
+// Broadcast a message to all uBO contexts and all uBO's content scripts
+
+export async function broadcastToAll(message) {
+    broadcast(message);
+    const tabs = await vAPI.tabs.query({
+        discarded: false,
+    });
+    const bcmessage = Object.assign({ broadcast: true }, message);
+    for ( const tab of tabs ) {
+        browser.tabs.sendMessage(tab.id, bcmessage);
+    }
+}
+
+/******************************************************************************/
+
+export function onBroadcast(listener) {
+    const bc = new self.BroadcastChannel('uBO');
+    bc.onmessage = ev => listener(ev.data || {});
+    return bc;
+}
+
+/******************************************************************************/
+
+export function filteringBehaviorChanged(details = {}) {
+    if ( typeof details.direction !== 'number' || details.direction >= 0 ) {
+        filteringBehaviorChanged.throttle.offon(727);
+    }
+    broadcast(Object.assign({ what: 'filteringBehaviorChanged' }, details));
+}
+
+filteringBehaviorChanged.throttle = vAPI.defer.create(( ) => {
+    vAPI.net.handlerBehaviorChanged();
+});
+
+/******************************************************************************/

package/extensions/ublock/js/cachestorage.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2016-present The uBlock Origin authors
 
     This program is free software: you can redistribute it and/or modify
@@ -61,27 +61,40 @@ const STORAGE_NAME = 'uBlock0CacheStorage';
 // Default to webext storage.
 const storageLocal = webext.storage.local;
 
+let storageReadyResolve;
+const storageReadyPromise = new Promise(resolve => {
+    storageReadyResolve = resolve;
+});
+
 const cacheStorage = {
     name: 'browser.storage.local',
     get(...args) {
-        return storageLocal.get(...args).catch(reason => {
-            console.log(reason);
-        });
+        return storageReadyPromise.then(( ) =>
+            storageLocal.get(...args).catch(reason => {
+                console.log(reason);
+            })
+        );
     },
     set(...args) {
-        return storageLocal.set(...args).catch(reason => {
-            console.log(reason);
-        });
+        return storageReadyPromise.then(( ) =>
+            storageLocal.set(...args).catch(reason => {
+                console.log(reason);
+            })
+        );
     },
     remove(...args) {
-        return storageLocal.remove(...args).catch(reason => {
-            console.log(reason);
-        });
+        return storageReadyPromise.then(( ) =>
+            storageLocal.remove(...args).catch(reason => {
+                console.log(reason);
+            })
+        );
     },
     clear(...args) {
-        return storageLocal.clear(...args).catch(reason => {
-            console.log(reason);
-        });
+        return storageReadyPromise.then(( ) =>
+            storageLocal.clear(...args).catch(reason => {
+                console.log(reason);
+            })
+        );
     },
     select: function(selectedBackend) {
         let actualBackend = selectedBackend;
@@ -94,15 +107,18 @@ const cacheStorage = {
             return selectIDB().then(success => {
                 if ( success || selectedBackend === 'indexedDB' ) {
                     clearWebext();
+                    storageReadyResolve();
                     return 'indexedDB';
                 }
                 clearIDB();
+                storageReadyResolve();
                 return 'browser.storage.local';
             });
         }
         if ( actualBackend === 'browser.storage.local' ) {
             clearIDB();
         }
+        storageReadyResolve();
         return Promise.resolve('browser.storage.local');
         
     },
@@ -435,36 +451,44 @@ const selectIDB = async function() {
 
     cacheStorage.name = 'indexedDB';
     cacheStorage.get = function get(keys) {
-        return new Promise(resolve => {
-            if ( keys === null ) {
-                return getAllFromDb(bin => resolve(bin));
-            }
-            let toRead, output = {};
-            if ( typeof keys === 'string' ) {
-                toRead = [ keys ];
-            } else if ( Array.isArray(keys) ) {
-                toRead = keys;
-            } else /* if ( typeof keys === 'object' ) */ {
-                toRead = Object.keys(keys);
-                output = keys;
-            }
-            getFromDb(toRead, output, bin => resolve(bin));
-        });
+        return storageReadyPromise.then(( ) =>
+            new Promise(resolve => {
+                if ( keys === null ) {
+                    return getAllFromDb(bin => resolve(bin));
+                }
+                let toRead, output = {};
+                if ( typeof keys === 'string' ) {
+                    toRead = [ keys ];
+                } else if ( Array.isArray(keys) ) {
+                    toRead = keys;
+                } else /* if ( typeof keys === 'object' ) */ {
+                    toRead = Object.keys(keys);
+                    output = keys;
+                }
+                getFromDb(toRead, output, bin => resolve(bin));
+            })
+        );
     };
     cacheStorage.set = function set(keys) {
-        return new Promise(resolve => {
-            putToDb(keys, details => resolve(details));
-        });
+        return storageReadyPromise.then(( ) =>
+            new Promise(resolve => {
+                putToDb(keys, details => resolve(details));
+            })
+        );
     };
     cacheStorage.remove = function remove(keys) {
-        return new Promise(resolve => {
-            deleteFromDb(keys, ( ) => resolve());
-        });
+        return storageReadyPromise.then(( ) =>
+            new Promise(resolve => {
+                deleteFromDb(keys, ( ) => resolve());
+            })
+        );
     };
     cacheStorage.clear = function clear() {
-        return new Promise(resolve => {
-            clearDb(( ) => resolve());
-        });
+        return storageReadyPromise.then(( ) =>
+            new Promise(resolve => {
+                clearDb(( ) => resolve());
+            })
+        );
     };
     cacheStorage.getBytesInUse = function getBytesInUse() {
         return Promise.resolve(0);
@@ -475,18 +499,17 @@ const selectIDB = async function() {
 // https://github.com/uBlockOrigin/uBlock-issues/issues/328
 //   Delete cache-related entries from webext storage.
 const clearWebext = async function() {
-    const bin = await webext.storage.local.get('assetCacheRegistry');
-    if (
-        bin instanceof Object === false ||
-        bin.assetCacheRegistry instanceof Object === false
-    ) {
-        return;
+    let bin;
+    try {
+        bin = await webext.storage.local.get('assetCacheRegistry');
+    } catch(ex) {
+        console.error(ex);
     }
+    if ( bin instanceof Object === false ) { return; }
+    if ( bin.assetCacheRegistry instanceof Object === false ) { return; }
     const toRemove = [
         'assetCacheRegistry',
         'assetSourceRegistry',
-        'resourcesSelfie',
-        'selfie'
     ];
     for ( const key in bin.assetCacheRegistry ) {
         if ( bin.assetCacheRegistry.hasOwnProperty(key) ) {

package/extensions/ublock/js/click2load.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2014-present Raymond Hill
 
     This program is free software: you can redistribute it and/or modify

package/extensions/ublock/js/cloud-ui.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2015-2018 Raymond Hill
 
     This program is free software: you can redistribute it and/or modify

package/extensions/ublock/js/code-viewer.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2023-present Raymond Hill
 
     This program is free software: you can redistribute it and/or modify

package/extensions/ublock/js/codemirror/search-thread.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2020-present Raymond Hill
 
     This program is free software: you can redistribute it and/or modify

package/extensions/ublock/js/codemirror/search.js

@@ -377,7 +377,7 @@ import { i18n$ } from '../i18n.js';
             if ( markers === null ) { return; }
             const marker = markers['CodeMirror-lintgutter'];
             if ( marker === undefined ) { return; }
-            if ( marker.dataset.lint !== 'error' )  { return; }
+            if ( marker.dataset.error !== 'y' ) { return; }
             const line = lineHandle.lineNo();
             if ( dir < 0 ) {
                 found = line;

package/extensions/ublock/js/codemirror/ubo-dynamic-filtering.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2019-present Raymond Hill
 
     This program is free software: you can redistribute it and/or modify