@browserless.io/browserless 2.5.0-beta-4 → 2.6.0

package/extensions/ublock/js/3p-filters.js

@@ -19,11 +19,9 @@
     Home: https://github.com/gorhill/uBlock
 */
 
-'use strict';
-
-import { onBroadcast } from './broadcast.js';
 import { dom, qs$, qsa$ } from './dom.js';
 import { i18n, i18n$ } from './i18n.js';
+import { onBroadcast } from './broadcast.js';
 
 /******************************************************************************/
 
@@ -32,6 +30,10 @@ const obsoleteTemplateString = i18n$('3pExternalListObsolete');
 const reValidExternalList = /^[a-z-]+:\/\/(?:\S+\/\S*|\/\S+)/m;
 const recentlyUpdated = 1 * 60 * 60 * 1000; // 1 hour
 
+// https://eslint.org/docs/latest/rules/no-prototype-builtins
+const hasOwnProperty = (o, p) =>
+    Object.prototype.hasOwnProperty.call(o, p);
+
 let listsetDetails = {};
 
 /******************************************************************************/
@@ -74,7 +76,9 @@ const renderNodeStats = (used, total) => {
 };
 
 const i18nGroupName = name => {
-    return i18n$('3pGroup' + name.charAt(0).toUpperCase() + name.slice(1));
+    const groupname = i18n$('3pGroup' + name.charAt(0).toUpperCase() + name.slice(1));
+    if ( groupname !== '' ) { return groupname; }
+    return `${name.charAt(0).toLocaleUpperCase}${name.slice(1)}`;
 };
 
 /******************************************************************************/
@@ -90,8 +94,9 @@ const renderFilterLists = ( ) => {
 
     const initializeListEntry = (listDetails, listEntry) => {
         const listkey = listEntry.dataset.key;
+        const groupkey = listDetails.group2 || listDetails.group;
         const listEntryPrevious =
-            qs$(`[data-key="${listDetails.group}"] [data-key="${listkey}"]`);
+            qs$(`[data-key="${groupkey}"] [data-key="${listkey}"]`);
         if ( listEntryPrevious !== null ) {
             if ( dom.cl.has(listEntryPrevious, 'checked') ) {
                 dom.cl.add(listEntry, 'checked');
@@ -179,6 +184,9 @@ const renderFilterLists = ( ) => {
         if ( depth !== 0 ) {
             const reEmojis = /\p{Emoji}+/gu;
             treeEntries.sort((a ,b) => {
+                const ap = a[1].preferred === true;
+                const bp = b[1].preferred === true;
+                if ( ap !== bp ) { return ap ? -1 : 1; }
                 const as = (a[1].title || a[0]).replace(reEmojis, '');
                 const bs = (b[1].title || b[0]).replace(reEmojis, '');
                 return as.localeCompare(bs);
@@ -223,8 +231,11 @@ const renderFilterLists = ( ) => {
             'privacy',
             'malware',
             'multipurpose',
+            'cookies',
+            'social',
             'annoyances',
             'regions',
+            'unknown',
             'custom'
         ];
         for ( const key of groupKeys ) {
@@ -234,17 +245,20 @@ const renderFilterLists = ( ) => {
             };
         }
         for ( const [ listkey, listDetails ] of Object.entries(response.available) ) {
-            let groupKey = listDetails.group;
-            if ( groupKey === 'social' ) {
-                groupKey = 'annoyances';
+            let groupkey = listDetails.group2 || listDetails.group;
+            if ( hasOwnProperty(listTree, groupkey) === false ) {
+                groupkey = 'unknown';
             }
-            const groupDetails = listTree[groupKey];
+            const groupDetails = listTree[groupkey];
             if ( listDetails.parent !== undefined ) {
                 let lists = groupDetails.lists;
                 for ( const parent of listDetails.parent.split('|') ) {
                     if ( lists[parent] === undefined ) {
                         lists[parent] = { title: parent, lists: {} };
                     }
+                    if ( listDetails.preferred === true ) {
+                        lists[parent].preferred = true;
+                    }
                     lists = lists[parent].lists;
                 }
                 lists[listkey] = listDetails;
@@ -253,6 +267,15 @@ const renderFilterLists = ( ) => {
                 groupDetails.lists[listkey] = listDetails;
             }
         }
+        // https://github.com/uBlockOrigin/uBlock-issues/issues/3154#issuecomment-1975413427
+        //   Remove empty sections
+        for ( const groupkey of groupKeys ) {
+            const groupDetails = listTree[groupkey];
+            if ( groupDetails === undefined ) { continue; }
+            if ( Object.keys(groupDetails.lists).length !== 0 ) { continue; }
+            delete listTree[groupkey];
+        }
+
         const listEntries = createListEntries('root', listTree);
         qs$('#lists .listEntries').replaceWith(listEntries);
 
@@ -530,6 +553,35 @@ dom.on('#lists', 'click', 'span.cache', onPurgeClicked);
 /******************************************************************************/
 
 const selectFilterLists = async ( ) => {
+    // External filter lists to import
+    // Find stock list matching entries in lists to import
+    const toImport = (( ) => {
+        const textarea = qs$('#lists .listEntry[data-role="import"].expanded textarea');
+        if ( textarea === null ) { return ''; }
+        const lists = listsetDetails.available;
+        const lines = textarea.value.split(/\s+\n|\s+/);
+        const after = [];
+        for ( const line of lines ) {
+            if ( /^https?:\/\//.test(line) === false ) { continue; }
+            for ( const [ listkey, list ] of Object.entries(lists) ) {
+                if ( list.content !== 'filters' ) { continue; }
+                if ( list.contentURL === undefined ) { continue; }
+                if ( list.contentURL.includes(line) === false ) {
+                    after.push(line);
+                    continue;
+                }
+                const groupkey = list.group2 || list.group;
+                const listEntry = qs$(`[data-key="${groupkey}"] [data-key="${listkey}"]`);
+                if ( listEntry === null ) { break; }
+                toggleFilterList(listEntry, true);
+                break;
+            }
+        }
+        dom.cl.remove(textarea.closest('.expandable'), 'expanded');
+        textarea.value = '';
+        return after.join('\n');
+    })();
+
     // Cosmetic filtering switch
     let checked = qs$('#parseCosmeticFilters').checked;
     vAPI.messaging.send('dashboard', {
@@ -552,7 +604,7 @@ const selectFilterLists = async ( ) => {
     const toRemove = [];
     for ( const liEntry of qsa$('#lists .listEntry[data-role="leaf"]') ) {
         const listkey = liEntry.dataset.key;
-        if ( listsetDetails.available.hasOwnProperty(listkey) === false ) {
+        if ( hasOwnProperty(listsetDetails.available, listkey) === false ) {
             continue;
         }
         const listDetails = listsetDetails.available[listkey];
@@ -569,14 +621,6 @@ const selectFilterLists = async ( ) => {
         }
     }
 
-    // External filter lists to import
-    const textarea = qs$('#lists .listEntry[data-role="import"].expanded textarea');
-    const toImport = textarea !== null && textarea.value.trim() || '';
-    if ( textarea !== null ) {
-        dom.cl.remove(textarea.closest('.expandable'), 'expanded');
-        textarea.value = '';
-    }
-
     hashFromListsetDetails();
 
     await vAPI.messaging.send('dashboard', {
@@ -630,7 +674,7 @@ dom.on('#suspendUntilListsAreLoaded', 'change', userSettingCheckboxChanged);
 /******************************************************************************/
 
 const searchFilterLists = ( ) => {
-    const pattern = dom.prop('.searchbar input', 'value') || '';
+    const pattern = dom.prop('.searchfield input', 'value') || '';
     dom.cl.toggle('#lists', 'searchMode', pattern !== '');
     if ( pattern === '' ) { return; }
     const reflectSearchMatches = listEntry => {
@@ -657,10 +701,11 @@ const searchFilterLists = ( ) => {
         if ( listDetails === undefined ) { continue; }
         let haystack = perListHaystack.get(listDetails);
         if ( haystack === undefined ) {
+            const groupkey = listDetails.group2 || listDetails.group || '';
             haystack = [
                 listDetails.title,
-                listDetails.group || '',
-                i18nGroupName(listDetails.group || ''),
+                groupkey,
+                i18nGroupName(groupkey),
                 listDetails.tags || '',
                 toI18n(listDetails.tags || ''),
             ].join(' ').trim();
@@ -673,14 +718,13 @@ const searchFilterLists = ( ) => {
 
 const perListHaystack = new WeakMap();
 
-dom.on('.searchbar input', 'input', searchFilterLists);
+dom.on('.searchfield input', 'input', searchFilterLists);
 
 /******************************************************************************/
 
 const expandedListSet = new Set([
-    'uBlock filters',
-    'AdGuard – Annoyances',
-    'EasyList – Annoyances',
+    'cookies',
+    'social',
 ]);
 
 const listIsExpanded = which => {
@@ -844,6 +888,8 @@ self.cloud.onPull = function fromCloudData(data, append) {
 
 /******************************************************************************/
 
+self.wikilink = 'https://github.com/gorhill/uBlock/wiki/Dashboard:-Filter-lists';
+
 self.hasUnsavedData = function() {
     return hashFromCurrentFromSettings() !== filteringSettingsHash;
 };

package/extensions/ublock/js/asset-viewer.js

@@ -60,6 +60,7 @@ import './codemirror/ubo-static-filtering.js';
         lineWrapping: true,
         matchBrackets: true,
         maxScanLines: 1,
+        maximizable: false,
         readOnly: true,
         styleActiveLine: {
             nonEmpty: true,

package/extensions/ublock/js/assets.js

@@ -528,12 +528,12 @@ function getAssetSourceRegistry() {
         assetSourceRegistryPromise = cacheStorage.get(
             'assetSourceRegistry'
         ).then(bin => {
-            if (
-                bin instanceof Object &&
-                bin.assetSourceRegistry instanceof Object
-            ) {
-                assetSourceRegistry = bin.assetSourceRegistry;
-                return assetSourceRegistry;
+            if ( bin instanceof Object ) {
+                if ( bin.assetSourceRegistry instanceof Object ) {
+                    assetSourceRegistry = bin.assetSourceRegistry;
+                    ubolog('Loaded assetSourceRegistry');
+                    return assetSourceRegistry;
+                }
             }
             return assets.fetchText(
                 µb.assetsBootstrapLocation || µb.assetsJsonPath
@@ -543,6 +543,7 @@ function getAssetSourceRegistry() {
                     : assets.fetchText(µb.assetsJsonPath);
             }).then(details => {
                 updateAssetSourceRegistry(details.content, true);
+                ubolog('Loaded assetSourceRegistry');
                 return assetSourceRegistry;
             });
         });
@@ -673,49 +674,36 @@ let assetCacheRegistryPromise;
 let assetCacheRegistry = {};
 
 function getAssetCacheRegistry() {
-    if ( assetCacheRegistryPromise === undefined ) {
-        assetCacheRegistryPromise = cacheStorage.get(
-            'assetCacheRegistry'
-        ).then(bin => {
-            if (
-                bin instanceof Object &&
-                bin.assetCacheRegistry instanceof Object
-            ) {
-                if ( Object.keys(assetCacheRegistry).length === 0 ) {
-                    assetCacheRegistry = bin.assetCacheRegistry;
-                } else {
-                    console.error(
-                        'getAssetCacheRegistry(): assetCacheRegistry reassigned!'
-                    );
-                    if (
-                        Object.keys(bin.assetCacheRegistry).sort().join() !==
-                        Object.keys(assetCacheRegistry).sort().join()
-                    ) {
-                        console.error(
-                            'getAssetCacheRegistry(): assetCacheRegistry changes overwritten!'
-                        );
-                    }
-                }
-            }
-            return assetCacheRegistry;
-        });
+    if ( assetCacheRegistryPromise !== undefined ) {
+        return assetCacheRegistryPromise;
     }
-
+    assetCacheRegistryPromise = cacheStorage.get(
+        'assetCacheRegistry'
+    ).then(bin => {
+        if ( bin instanceof Object === false ) { return; }
+        if ( bin.assetCacheRegistry instanceof Object === false ) { return; }
+        if ( Object.keys(assetCacheRegistry).length !== 0 ) {
+            return console.error('getAssetCacheRegistry(): assetCacheRegistry reassigned!');
+        }
+        ubolog('Loaded assetCacheRegistry');
+        assetCacheRegistry = bin.assetCacheRegistry;
+    }).then(( ) =>
+        assetCacheRegistry
+    );
     return assetCacheRegistryPromise;
 }
 
 const saveAssetCacheRegistry = (( ) => {
-    const save = function() {
+    const save = ( ) => {
         timer.off();
-        cacheStorage.set({ assetCacheRegistry });
+        return cacheStorage.set({ assetCacheRegistry });
     };
     const timer = vAPI.defer.create(save);
-    return function(lazily) {
-        if ( lazily ) {
-            timer.offon({ sec: 30 });
-        } else {
-            save();
+    return (throttle = 0) => {
+        if ( throttle === 0 ) {
+            return save();
         }
+        timer.offon({ sec: throttle });
     };
 })();
 
@@ -726,7 +714,9 @@ async function assetCacheRead(assetKey, updateReadTime = false) {
     const reportBack = function(content) {
         if ( content instanceof Blob ) { content = ''; }
         const details = { assetKey, content };
-        if ( content === '' ) { details.error = 'ENOTFOUND'; }
+        if ( content === '' || content === undefined ) {
+            details.error = 'ENOTFOUND';
+        }
         return details;
     };
 
@@ -742,55 +732,39 @@ async function assetCacheRead(assetKey, updateReadTime = false) {
         ) + ' ms';
     }
 
-    if (
-        bin instanceof Object === false ||
-        bin.hasOwnProperty(internalKey) === false
-    ) {
-        return reportBack('');
-    }
+    if ( bin instanceof Object === false ) { return reportBack(''); }
+    if ( bin.hasOwnProperty(internalKey) === false ) { return reportBack(''); }
 
     const entry = assetCacheRegistry[assetKey];
-    if ( entry === undefined ) {
-        return reportBack('');
-    }
+    if ( entry === undefined ) { return reportBack(''); }
 
     entry.readTime = Date.now();
     if ( updateReadTime ) {
-        saveAssetCacheRegistry(true);
+        saveAssetCacheRegistry(23);
     }
 
     return reportBack(bin[internalKey]);
 }
 
-async function assetCacheWrite(assetKey, details) {
-    let content = '';
-    let options = {};
-    if ( typeof details === 'string' ) {
-        content = details;
-    } else if ( details instanceof Object ) {
-        content = details.content || '';
-        options = details;
-    }
-
-    if ( content === '' ) {
+async function assetCacheWrite(assetKey, content, options = {}) {
+    if ( content === '' || content === undefined ) {
         return assetCacheRemove(assetKey);
     }
 
     const cacheDict = await getAssetCacheRegistry();
 
-    let entry = cacheDict[assetKey];
-    if ( entry === undefined ) {
-        entry = cacheDict[assetKey] = {};
-    }
+    const { resourceTime, url } = options;
+    const entry = cacheDict[assetKey] || {};
     entry.writeTime = entry.readTime = Date.now();
-    entry.resourceTime = options.resourceTime || 0;
-    if ( typeof options.url === 'string' ) {
-        entry.remoteURL = options.url;
+    entry.resourceTime = resourceTime || 0;
+    if ( typeof url === 'string' ) {
+        entry.remoteURL = url;
     }
-    cacheStorage.set({
-        assetCacheRegistry,
-        [`cache/${assetKey}`]: content
-    });
+    cacheDict[assetKey] = entry;
+
+    await cacheStorage.set({ [`cache/${assetKey}`]: content });
+
+    saveAssetCacheRegistry(3);
 
     const result = { assetKey, content };
     // https://github.com/uBlockOrigin/uBlock-issues/issues/248
@@ -800,21 +774,31 @@ async function assetCacheWrite(assetKey, details) {
     return result;
 }
 
-async function assetCacheRemove(pattern) {
+async function assetCacheRemove(pattern, options = {}) {
     const cacheDict = await getAssetCacheRegistry();
     const removedEntries = [];
     const removedContent = [];
     for ( const assetKey in cacheDict ) {
-        if ( pattern instanceof RegExp && !pattern.test(assetKey) ) {
-            continue;
-        }
-        if ( typeof pattern === 'string' && assetKey !== pattern ) {
-            continue;
+        if ( pattern instanceof RegExp ) {
+            if ( pattern.test(assetKey) === false ) { continue; }
+        } else if ( typeof pattern === 'string' ) {
+            if ( assetKey !== pattern ) { continue; }
         }
         removedEntries.push(assetKey);
-        removedContent.push('cache/' + assetKey);
+        removedContent.push(`cache/${assetKey}`);
         delete cacheDict[assetKey];
     }
+    if ( options.janitor && pattern instanceof RegExp ) {
+        const re = new RegExp(
+            pattern.source.replace(/^\^/, '^cache\\/'),
+            pattern.flags
+        );
+        const keys = await cacheStorage.keys(re);
+        for ( const key of keys ) {
+            removedContent.push(key);
+            ubolog(`Removing stray ${key}`);
+        }
+    }
     if ( removedContent.length !== 0 ) {
         await Promise.all([
             cacheStorage.remove(removedContent),
@@ -854,7 +838,7 @@ async function assetCacheSetDetails(assetKey, details) {
         }
     }
     if ( modified ) {
-        saveAssetCacheRegistry();
+        saveAssetCacheRegistry(3);
     }
 }
 
@@ -980,8 +964,7 @@ assets.get = async function(assetKey, options = {}) {
         }
         if ( details.content === '' ) { continue; }
         if ( reIsExternalPath.test(contentURL) && options.dontCache !== true ) {
-            assetCacheWrite(assetKey, {
-                content: details.content,
+            assetCacheWrite(assetKey, details.content, {
                 url: contentURL,
                 silent: options.silent === true,
             });
@@ -1057,8 +1040,7 @@ async function getRemote(assetKey, options = {}) {
         }
 
         // Success
-        assetCacheWrite(assetKey, {
-            content: result.content,
+        assetCacheWrite(assetKey, result.content, {
             url: contentURL,
             resourceTime: result.resourceTime || 0,
         });
@@ -1101,6 +1083,17 @@ assets.put = async function(assetKey, content) {
 
 /******************************************************************************/
 
+assets.toCache = async function(assetKey, content) {
+    return assetCacheWrite(assetKey, content);
+};
+
+assets.fromCache = async function(assetKey) {
+    const details = await assetCacheRead(assetKey);
+    return details && details.content;
+};
+
+/******************************************************************************/
+
 assets.metadata = async function() {
     await Promise.all([
         getAssetSourceRegistry(),
@@ -1147,8 +1140,8 @@ assets.metadata = async function() {
 
 assets.purge = assetCacheMarkAsDirty;
 
-assets.remove = function(pattern) {
-    return assetCacheRemove(pattern);
+assets.remove = function(...args) {
+    return assetCacheRemove(...args);
 };
 
 assets.rmrf = function() {
@@ -1300,8 +1293,7 @@ async function diffUpdater() {
                     'Diff-Path',
                     'Diff-Expires',
                 ]);
-                assetCacheWrite(data.assetKey, {
-                    content: data.text,
+                assetCacheWrite(data.assetKey, data.text, {
                     resourceTime: metadata.lastModified || 0,
                 });
                 metadata.diffUpdated = true;
@@ -1333,6 +1325,8 @@ async function diffUpdater() {
             terminate();
         };
         const worker = new Worker('js/diff-updater.js');
+    }).catch(reason => {
+        ubolog(`Diff updater: ${reason}`);
     });
 }
 

package/extensions/ublock/js/background.js

@@ -19,22 +19,18 @@
     Home: https://github.com/gorhill/uBlock
 */
 
-/* globals browser */
-
-'use strict';
-
 /******************************************************************************/
 
-import logger from './logger.js';
-import { FilteringContext } from './filtering-context.js';
-import { ubologSet } from './console.js';
-
 import {
     domainFromHostname,
     hostnameFromURI,
     originFromURI,
 } from './uri-utils.js';
 
+import { FilteringContext } from './filtering-context.js';
+import logger from './logger.js';
+import { ubologSet } from './console.js';
+
 /******************************************************************************/
 
 // Not all platforms may have properly declared vAPI.webextFlavor.
@@ -49,13 +45,14 @@ const hiddenSettingsDefault = {
     allowGenericProceduralFilters: false,
     assetFetchTimeout: 30,
     autoCommentFilterTemplate: '{{date}} {{origin}}',
-    autoUpdateAssetFetchPeriod: 15,
-    autoUpdateDelayAfterLaunch: 105,
+    autoUpdateAssetFetchPeriod: 5,
+    autoUpdateDelayAfterLaunch: 37,
     autoUpdatePeriod: 1,
     benchmarkDatasetURL: 'unset',
     blockingProfiles: '11111/#F00 11010/#C0F 11001/#00F 00001',
-    cacheStorageAPI: 'unset',
     cacheStorageCompression: true,
+    cacheStorageCompressionThreshold: 65536,
+    cacheStorageMultithread: 2,
     cacheControlForFirefox1376932: 'no-cache, no-store, must-revalidate',
     cloudStorageCompression: true,
     cnameIgnoreList: 'unset',
@@ -78,10 +75,12 @@ const hiddenSettingsDefault = {
     modifyWebextFlavor: 'unset',
     popupFontSize: 'unset',
     popupPanelDisabledSections: 0,
-    popupPanelLockedSections: 0,
     popupPanelHeightMode: 0,
+    popupPanelLockedSections: 0,
+    popupPanelOrientation: 'unset',
     requestJournalProcessPeriod: 1000,
-    selfieAfter: 2,
+    requestStatsDisabled: false,
+    selfieDelayInSeconds: 53,
     strictBlockingBypassDuration: 120,
     toolbarWarningTimeout: 60,
     trustedListPrefixes: 'ublock-',
@@ -93,7 +92,7 @@ const hiddenSettingsDefault = {
 
 if ( vAPI.webextFlavor.soup.has('devbuild') ) {
     hiddenSettingsDefault.consoleLogLevel = 'info';
-    hiddenSettingsDefault.trustedListPrefixes += ' user-';
+    hiddenSettingsDefault.cacheStorageAPI = 'unset';
     ubologSet(true);
 }
 
@@ -112,7 +111,7 @@ const userSettingsDefault = {
     externalLists: '',
     firewallPaneMinimized: true,
     hyperlinkAuditingDisabled: true,
-    ignoreGenericCosmeticFilters: vAPI.webextFlavor.soup.has('mobile'),
+    ignoreGenericCosmeticFilters: false,
     importedLists: [],
     largeMediaSize: 50,
     parseAllABPHideFilters: true,
@@ -122,6 +121,7 @@ const userSettingsDefault = {
     showIconBadge: true,
     suspendUntilListsAreLoaded: vAPI.Net.canSuspend(),
     tooltipsDisabled: false,
+    userFiltersTrusted: false,
     webrtcIPAddressHidden: false,
 };
 
@@ -144,7 +144,7 @@ if ( vAPI.webextFlavor.soup.has('firefox') ) {
 }
 
 const µBlock = {  // jshint ignore:line
-    wakeupReason: '',
+    alarmQueue: [],
 
     userSettingsDefault,
     userSettings: Object.assign({}, userSettingsDefault),
@@ -168,26 +168,19 @@ const µBlock = {  // jshint ignore:line
     netWhitelist: new Map(),
     netWhitelistModifyTime: 0,
     netWhitelistDefault: [
-        'about-scheme',
         'chrome-extension-scheme',
-        'chrome-scheme',
-        'edge-scheme',
         'moz-extension-scheme',
-        'opera-scheme',
-        'vivaldi-scheme',
-        'wyciwyg-scheme',   // Firefox's "What-You-Cache-Is-What-You-Get"
     ],
 
-    localSettings: {
-        blockedRequestCount: 0,
-        allowedRequestCount: 0,
+    requestStats: {
+        blockedCount: 0,
+        allowedCount: 0,
     },
-    localSettingsLastModified: 0,
 
     // Read-only
     systemSettings: {
         compiledMagic: 57,  // Increase when compiled format changes
-        selfieMagic: 57,    // Increase when selfie format changes
+        selfieMagic: 58,    // Increase when selfie format changes
     },
 
     // https://github.com/uBlockOrigin/uBlock-issues/issues/759#issuecomment-546654501