@browserless.io/browserless 2.0.0-beta-5 → 2.0.0-beta-7

package/extensions/ublock/js/logger-ui.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2015-present Raymond Hill
 
     This program is free software: you can redistribute it and/or modify
@@ -671,7 +671,7 @@ const viewPort = (( ) => {
             `  width: calc(calc(100% - ${reservedWidth}px) * ${cellWidths[COLUMN_FILTER]});`,
             '}',
             `#vwContent .logEntry > div.messageRealm > span:nth-of-type(${COLUMN_MESSAGE+1}) {`,
-            `  width: calc(100% - ${cellWidths[COLUMN_MESSAGE]}px);`,
+            `  width: calc(100% - ${cellWidths[COLUMN_TIMESTAMP]}px);`,
             '}',
             `#vwContent .logEntry > div > span:nth-of-type(${COLUMN_RESULT+1}) {`,
             `  width: ${cellWidths[COLUMN_RESULT]}px;`,
@@ -972,7 +972,7 @@ const viewPort = (( ) => {
 const updateCurrentTabTitle = (( ) => {
     const i18nCurrentTab = i18n$('loggerCurrentTab');
 
-    return function() {
+    return ( ) => {
         const select = qs$('#pageSelector');
         if ( select.value !== '_' || activeTabId === 0 ) { return; }
         const opt0 = qs$(select, '[value="_"]');
@@ -1033,8 +1033,7 @@ const synchronizeTabIds = function(newTabIds) {
         return newTabIds.get(a).localeCompare(newTabIds.get(b));
     });
     let j = 3;
-    for ( let i = 0; i < tabIds.length; i++ ) {
-        const tabId = tabIds[i];
+    for ( const tabId of tabIds ) {
         if ( tabId <= 0 ) { continue; }
         if ( j === select.options.length ) {
             select.appendChild(document.createElement('option'));
@@ -2252,6 +2251,23 @@ const rowFilterer = (( ) => {
         dom.cl.toggle(ev.target, 'on');
         builtinFilterExpression();
     });
+    dom.on('#filterInput > input', 'drop', ev => {
+        const dropItem = item => {
+            if ( item.kind !== 'string' ) { return false; }
+            if ( item.type !== 'text/plain' ) { return false; }
+            item.getAsString(s => {
+                qs$('#filterInput > input').value = s;
+                parseInput();
+                filterAll();
+            });
+            return true;
+        };
+        for ( const item of ev.dataTransfer.items ) {
+            if ( dropItem(item) === false ) { continue; }
+            ev.preventDefault();
+            break;
+        }
+    });
 
     // https://github.com/gorhill/uBlock/issues/404
     //   Ensure page state is in sync with the state of its various widgets.

package/extensions/ublock/js/logger.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2015-present Raymond Hill
 
     This program is free software: you can redistribute it and/or modify
@@ -23,6 +23,10 @@
 
 /******************************************************************************/
 
+import { broadcastToAll } from './broadcast.js';
+
+/******************************************************************************/
+
 let buffer = null;
 let lastReadTime = 0;
 let writePtr = 0;
@@ -40,7 +44,7 @@ const janitorTimer = vAPI.defer.create(( ) => {
     buffer = null;
     writePtr = 0;
     logger.ownerId = undefined;
-    vAPI.messaging.broadcast({ what: 'loggerDisabled' });
+    broadcastToAll({ what: 'loggerDisabled' });
 });
 
 const boxEntry = function(details) {

package/extensions/ublock/js/lz4.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2018-present Raymond Hill
 
     This program is free software: you can redistribute it and/or modify
@@ -44,7 +44,7 @@ let ttlCount = 0;
 let ttlDelay = 60000;
 
 const init = function() {
-    ttlDelay = µb.hiddenSettings.autoUpdateAssetFetchPeriod * 1000 + 15000;
+    ttlDelay = µb.hiddenSettings.autoUpdateAssetFetchPeriod * 2 * 1000;
     if ( promisedInstance === undefined ) {
         let flavor;
         if ( µb.hiddenSettings.disableWebAssembly === true ) {

package/extensions/ublock/js/messaging.js

@@ -1,6 +1,6 @@
 /*******************************************************************************
 
-    uBlock Origin - a browser extension to block requests.
+    uBlock Origin - a comprehensive, efficient content blocker
     Copyright (C) 2014-present Raymond Hill
 
     This program is free software: you can redistribute it and/or modify
@@ -28,6 +28,7 @@
 import publicSuffixList from '../lib/publicsuffixlist/publicsuffixlist.js';
 import punycode from '../lib/punycode.js';
 
+import { filteringBehaviorChanged } from './broadcast.js';
 import cacheStorage from './cachestorage.js';
 import cosmeticFilteringEngine from './cosmetic-filtering.js';
 import htmlFilteringEngine from './html-filtering.js';
@@ -141,120 +142,6 @@ const onMessage = function(request, sender, callback) {
         });
         return;
 
-    case 'snfeBenchmark':
-        µb.benchmarkStaticNetFiltering({ redirectEngine }).then(result => {
-            callback(result);
-        });
-        return;
-
-    case 'snfeToDNR': {
-        const listPromises = [];
-        const listNames = [];
-        for ( const assetKey of µb.selectedFilterLists ) {
-            listPromises.push(
-                io.get(assetKey, { dontCache: true }).then(details => {
-                    listNames.push(assetKey);
-                    return { name: assetKey, text: details.content };
-                })
-            );
-        }
-        const options = {
-            extensionPaths: redirectEngine.getResourceDetails(),
-            env: vAPI.webextFlavor.env,
-        };
-        const t0 = Date.now();
-        dnrRulesetFromRawLists(listPromises, options).then(result => {
-            const { network } = result;
-            const replacer = (k, v) => {
-                if ( k.startsWith('__') ) { return; }
-                if ( Array.isArray(v) ) {
-                    return v.sort();
-                }
-                if ( v instanceof Object ) {
-                    const sorted = {};
-                    for ( const kk of Object.keys(v).sort() ) {
-                        sorted[kk] = v[kk];
-                    }
-                    return sorted;
-                }
-                return v;
-            };
-            const isUnsupported = rule =>
-                rule._error !== undefined;
-            const isRegex = rule =>
-                rule.condition !== undefined &&
-                rule.condition.regexFilter !== undefined;
-            const isRedirect = rule =>
-                rule.action !== undefined &&
-                rule.action.type === 'redirect' &&
-                rule.action.redirect.extensionPath !== undefined;
-            const isCsp = rule =>
-                rule.action !== undefined &&
-                rule.action.type === 'modifyHeaders';
-            const isRemoveparam = rule =>
-                rule.action !== undefined &&
-                rule.action.type === 'redirect' &&
-                rule.action.redirect.transform !== undefined;
-            const runtime = Date.now() - t0;
-            const { ruleset } = network;
-            const good = ruleset.filter(rule =>
-                isUnsupported(rule) === false &&
-                isRegex(rule) === false &&
-                isRedirect(rule) === false &&
-                isCsp(rule) === false &&
-                isRemoveparam(rule) === false
-            );
-            const unsupported = ruleset.filter(rule =>
-                isUnsupported(rule)
-            );
-            const regexes = ruleset.filter(rule =>
-                isUnsupported(rule) === false &&
-                isRegex(rule) &&
-                isRedirect(rule) === false &&
-                isCsp(rule) === false &&
-                isRemoveparam(rule) === false
-            );
-            const redirects = ruleset.filter(rule =>
-                isUnsupported(rule) === false &&
-                isRedirect(rule)
-            );
-            const headers = ruleset.filter(rule =>
-                isUnsupported(rule) === false &&
-                isCsp(rule)
-            );
-            const removeparams = ruleset.filter(rule =>
-                isUnsupported(rule) === false &&
-                isRemoveparam(rule)
-            );
-            const out = [
-                `dnrRulesetFromRawLists(${JSON.stringify(listNames, null, 2)})`,
-                `Run time: ${runtime} ms`,
-                `Filters count: ${network.filterCount}`,
-                `Accepted filter count: ${network.acceptedFilterCount}`,
-                `Rejected filter count: ${network.rejectedFilterCount}`,
-                `Un-DNR-able filter count: ${unsupported.length}`,
-                `Resulting DNR rule count: ${ruleset.length}`,
-            ];
-            out.push(`+ Good filters (${good.length}): ${JSON.stringify(good, replacer, 2)}`);
-            out.push(`+ Regex-based filters (${regexes.length}): ${JSON.stringify(regexes, replacer, 2)}`);
-            out.push(`+ 'redirect=' filters (${redirects.length}): ${JSON.stringify(redirects, replacer, 2)}`);
-            out.push(`+ 'csp=' filters (${headers.length}): ${JSON.stringify(headers, replacer, 2)}`);
-            out.push(`+ 'removeparam=' filters (${removeparams.length}): ${JSON.stringify(removeparams, replacer, 2)}`);
-            out.push(`+ Unsupported filters (${unsupported.length}): ${JSON.stringify(unsupported, replacer, 2)}`);
-            out.push(`+ generichide exclusions (${network.generichideExclusions.length}): ${JSON.stringify(network.generichideExclusions, replacer, 2)}`);
-            if ( result.specificCosmetic ) {
-                out.push(`+ Cosmetic filters: ${result.specificCosmetic.size}`);
-                for ( const details of result.specificCosmetic ) {
-                    out.push(`    ${JSON.stringify(details)}`);
-                }
-            } else {
-                out.push('  Cosmetic filters: 0');
-            }
-            callback(out.join('\n'));
-        });
-        return;
-    }
-
     default:
         break;
     }
@@ -275,13 +162,6 @@ const onMessage = function(request, sender, callback) {
         µb.createUserFilters(request);
         break;
 
-    case 'forceUpdateAssets':
-        µb.scheduleAssetUpdater(0);
-        io.updateStart({
-            delay: µb.hiddenSettings.manualUpdateAssetFetchPeriod
-        });
-        break;
-
     case 'getAppData':
         response = {
             name: browser.runtime.getManifest().name,
@@ -346,7 +226,7 @@ const onMessage = function(request, sender, callback) {
     case 'setWhitelist':
         µb.netWhitelist = µb.whitelistFromString(request.whitelist);
         µb.saveWhitelist();
-        µb.filteringBehaviorChanged();
+        filteringBehaviorChanged();
         break;
 
     case 'toggleHostnameSwitch':
@@ -378,14 +258,6 @@ const onMessage = function(request, sender, callback) {
         }
         break;
 
-    case 'snfeDump':
-        response = staticNetFilteringEngine.dump();
-        break;
-
-    case 'cfeDump':
-        response = cosmeticFilteringEngine.dump();
-        break;
-
     default:
         return vAPI.messaging.UNHANDLED;
     }
@@ -846,13 +718,7 @@ const retrieveContentScriptParameters = async function(sender, request) {
     if ( logger.enabled || request.needScriptlets ) {
         const scriptletDetails = scriptletFilteringEngine.injectNow(request);
         if ( scriptletDetails !== undefined ) {
-            if ( logger.enabled ) {
-                scriptletFilteringEngine.logFilters(
-                    tabId,
-                    request.url,
-                    scriptletDetails.filters
-                );
-            }
+            scriptletFilteringEngine.toLogger(request, scriptletDetails);
             if ( request.needScriptlets ) {
                 response.scriptletDetails = scriptletDetails;
             }
@@ -1442,7 +1308,9 @@ const getSupportData = async function() {
 
     const now = Date.now();
 
-    const formatDelayFromNow = time => {
+    const formatDelayFromNow = list => {
+        const time = list.writeTime;
+        if ( typeof time !== 'number' || time === 0 ) { return 'never'; }
         if ( (time || 0) === 0 ) { return '?'; }
         const delayInSec = (now - time) / 1000;
         const days = (delayInSec / 86400) | 0;
@@ -1453,7 +1321,9 @@ const getSupportData = async function() {
         if ( hours > 0 ) { parts.push(`${hours}h`); }
         if ( minutes > 0 ) { parts.push(`${minutes}m`); }
         if ( parts.length === 0 ) { parts.push('now'); }
-        return parts.join('.');
+        const out = parts.join('.');
+        if ( list.diffUpdated ) { return `${out} Δ`; }
+        return out;
     };
 
     const lists = µb.availableFilterLists;
@@ -1470,11 +1340,7 @@ const getSupportData = async function() {
             if ( typeof list.entryCount === 'number' ) {
                 listDetails.push(`${list.entryCount}-${list.entryCount-list.entryUsedCount}`);
             }
-            if ( typeof list.writeTime !== 'number' || list.writeTime === 0 ) {
-                listDetails.push('never');
-            } else {
-                listDetails.push(formatDelayFromNow(list.writeTime));
-            }
+            listDetails.push(formatDelayFromNow(list));
         }
         if ( list.isDefault || listKey === µb.userFiltersPath ) {
             if ( used ) {
@@ -1560,7 +1426,9 @@ const onMessage = function(request, sender, callback) {
         });
 
     case 'getLists':
-        return getLists(callback);
+        return µb.isReadyPromise.then(( ) => {
+            getLists(callback);
+        });
 
     case 'getLocalData':
         return getLocalData().then(localData => {
@@ -1623,19 +1491,25 @@ const onMessage = function(request, sender, callback) {
         response = getRules();
         break;
 
-    case 'purgeAllCaches':
-        if ( request.hard ) {
-            io.remove(/./);
-        } else {
-            io.purge(/./, 'public_suffix_list.dat');
+    case 'supportUpdateNow': {
+        const { assetKeys } = request;
+        if ( assetKeys.length === 0 ) { return; }
+        for ( const assetKey of assetKeys ) {
+            io.purge(assetKey);
         }
+        µb.scheduleAssetUpdater({ now: true, fetchDelay: 100 });
         break;
+    }
 
-    case 'purgeCaches':
-        for ( const assetKey of request.assetKeys ) {
+    case 'listsUpdateNow': {
+        const { assetKeys, preferOrigin = false } = request;
+        if ( assetKeys.length === 0 ) { return; }
+        for ( const assetKey of assetKeys ) {
             io.purge(assetKey);
         }
+        µb.scheduleAssetUpdater({ now: true, fetchDelay: 100, auto: preferOrigin !== true });
         break;
+    }
 
     case 'readHiddenSettings':
         response = {
@@ -1653,6 +1527,10 @@ const onMessage = function(request, sender, callback) {
         resetUserData();
         break;
 
+    case 'updateNow':
+        µb.scheduleAssetUpdater({ now: true, fetchDelay: 100, auto: true });
+        break;
+
     case 'writeHiddenSettings':
         µb.changeHiddenSettings(µb.hiddenSettingsFromString(request.content));
         break;
@@ -1695,17 +1573,14 @@ const getLoggerData = async function(details, activeTabId, callback) {
         tooltips: µb.userSettings.tooltipsDisabled === false
     };
     if ( µb.pageStoresToken !== details.tabIdsToken ) {
-        const tabIds = new Map();
+        response.tabIds = [];
         for ( const [ tabId, pageStore ] of µb.pageStores ) {
-            const { rawURL } = pageStore;
-            if (
-                rawURL.startsWith(extensionOriginURL) === false ||
-                rawURL.startsWith(documentBlockedURL)
-            ) {
-                tabIds.set(tabId, pageStore.title);
+            const { rawURL, title } = pageStore;
+            if ( rawURL.startsWith(extensionOriginURL) ) {
+                if ( rawURL.startsWith(documentBlockedURL) === false ) { continue; }
             }
+            response.tabIds.push([ tabId, title ]);
         }
-        response.tabIds = Array.from(tabIds);
     }
     if ( activeTabId ) {
         const pageStore = µb.pageStoreFromTabId(activeTabId);
@@ -1845,6 +1720,54 @@ vAPI.messaging.listen({
 /******************************************************************************/
 /******************************************************************************/
 
+// Channel:
+//      domInspectorContent
+//      unprivileged
+
+{
+// >>>>> start of local scope
+
+const onMessage = (request, sender, callback) => {
+    // Async
+    switch ( request.what ) {
+    default:
+        break;
+    }
+    // Sync
+    let response;
+    switch ( request.what ) {
+    case 'getInspectorArgs':
+        const bc = new globalThis.BroadcastChannel('contentInspectorChannel');
+        bc.postMessage({
+            what: 'contentInspectorChannel',
+            tabId: sender.tabId || 0,
+            frameId: sender.frameId || 0,
+        });
+        response = {
+            inspectorURL: vAPI.getURL(
+                `/web_accessible_resources/dom-inspector.html?secret=${vAPI.warSecret.short()}`
+            ),
+        };
+        break;
+    default:
+        return vAPI.messaging.UNHANDLED;
+    }
+
+    callback(response);
+};
+
+vAPI.messaging.listen({
+    name: 'domInspectorContent',
+    listener: onMessage,
+    privileged: false,
+});
+
+// <<<<< end of local scope
+}
+
+/******************************************************************************/
+/******************************************************************************/
+
 // Channel:
 //      documentBlocked
 //      privileged
@@ -1892,6 +1815,181 @@ vAPI.messaging.listen({
 /******************************************************************************/
 /******************************************************************************/
 
+// Channel:
+//      devTools
+//      privileged
+
+{
+// >>>>> start of local scope
+
+const onMessage = function(request, sender, callback) {
+    // Async
+    switch ( request.what ) {
+    case 'purgeAllCaches':
+        µb.getBytesInUse().then(bytesInUseBefore =>
+            io.remove(/./).then(( ) =>
+                µb.getBytesInUse().then(bytesInUseAfter => {
+                    callback([
+                        `Storage used before: ${µb.formatCount(bytesInUseBefore)}B`,
+                        `Storage used after: ${µb.formatCount(bytesInUseAfter)}B`,
+                    ].join('\n'));
+                })
+            )
+        );
+        return;
+
+    case 'snfeBenchmark':
+        µb.benchmarkStaticNetFiltering({ redirectEngine }).then(result => {
+            callback(result);
+        });
+        return;
+
+    case 'snfeToDNR': {
+        const listPromises = [];
+        const listNames = [];
+        for ( const assetKey of µb.selectedFilterLists ) {
+            listPromises.push(
+                io.get(assetKey, { dontCache: true }).then(details => {
+                    listNames.push(assetKey);
+                    return { name: assetKey, text: details.content };
+                })
+            );
+        }
+        const options = {
+            extensionPaths: redirectEngine.getResourceDetails().filter(e =>
+                typeof e[1].extensionPath === 'string' && e[1].extensionPath !== ''
+            ).map(e =>
+                [ e[0], e[1].extensionPath ]
+            ),
+            env: vAPI.webextFlavor.env,
+        };
+        const t0 = Date.now();
+        dnrRulesetFromRawLists(listPromises, options).then(result => {
+            const { network } = result;
+            const replacer = (k, v) => {
+                if ( k.startsWith('__') ) { return; }
+                if ( Array.isArray(v) ) {
+                    return v.sort();
+                }
+                if ( v instanceof Object ) {
+                    const sorted = {};
+                    for ( const kk of Object.keys(v).sort() ) {
+                        sorted[kk] = v[kk];
+                    }
+                    return sorted;
+                }
+                return v;
+            };
+            const isUnsupported = rule =>
+                rule._error !== undefined;
+            const isRegex = rule =>
+                rule.condition !== undefined &&
+                rule.condition.regexFilter !== undefined;
+            const isRedirect = rule =>
+                rule.action !== undefined &&
+                rule.action.type === 'redirect' &&
+                rule.action.redirect.extensionPath !== undefined;
+            const isCsp = rule =>
+                rule.action !== undefined &&
+                rule.action.type === 'modifyHeaders';
+            const isRemoveparam = rule =>
+                rule.action !== undefined &&
+                rule.action.type === 'redirect' &&
+                rule.action.redirect.transform !== undefined;
+            const runtime = Date.now() - t0;
+            const { ruleset } = network;
+            const good = ruleset.filter(rule =>
+                isUnsupported(rule) === false &&
+                isRegex(rule) === false &&
+                isRedirect(rule) === false &&
+                isCsp(rule) === false &&
+                isRemoveparam(rule) === false
+            );
+            const unsupported = ruleset.filter(rule =>
+                isUnsupported(rule)
+            );
+            const regexes = ruleset.filter(rule =>
+                isUnsupported(rule) === false &&
+                isRegex(rule) &&
+                isRedirect(rule) === false &&
+                isCsp(rule) === false &&
+                isRemoveparam(rule) === false
+            );
+            const redirects = ruleset.filter(rule =>
+                isUnsupported(rule) === false &&
+                isRedirect(rule)
+            );
+            const headers = ruleset.filter(rule =>
+                isUnsupported(rule) === false &&
+                isCsp(rule)
+            );
+            const removeparams = ruleset.filter(rule =>
+                isUnsupported(rule) === false &&
+                isRemoveparam(rule)
+            );
+            const out = [
+                `dnrRulesetFromRawLists(${JSON.stringify(listNames, null, 2)})`,
+                `Run time: ${runtime} ms`,
+                `Filters count: ${network.filterCount}`,
+                `Accepted filter count: ${network.acceptedFilterCount}`,
+                `Rejected filter count: ${network.rejectedFilterCount}`,
+                `Un-DNR-able filter count: ${unsupported.length}`,
+                `Resulting DNR rule count: ${ruleset.length}`,
+            ];
+            out.push(`+ Good filters (${good.length}): ${JSON.stringify(good, replacer, 2)}`);
+            out.push(`+ Regex-based filters (${regexes.length}): ${JSON.stringify(regexes, replacer, 2)}`);
+            out.push(`+ 'redirect=' filters (${redirects.length}): ${JSON.stringify(redirects, replacer, 2)}`);
+            out.push(`+ 'csp=' filters (${headers.length}): ${JSON.stringify(headers, replacer, 2)}`);
+            out.push(`+ 'removeparam=' filters (${removeparams.length}): ${JSON.stringify(removeparams, replacer, 2)}`);
+            out.push(`+ Unsupported filters (${unsupported.length}): ${JSON.stringify(unsupported, replacer, 2)}`);
+            out.push(`+ generichide exclusions (${network.generichideExclusions.length}): ${JSON.stringify(network.generichideExclusions, replacer, 2)}`);
+            if ( result.specificCosmetic ) {
+                out.push(`+ Cosmetic filters: ${result.specificCosmetic.size}`);
+                for ( const details of result.specificCosmetic ) {
+                    out.push(`    ${JSON.stringify(details)}`);
+                }
+            } else {
+                out.push('  Cosmetic filters: 0');
+            }
+            callback(out.join('\n'));
+        });
+        return;
+    }
+    default:
+        break;
+    }
+
+    // Sync
+    let response;
+
+    switch ( request.what ) {
+    case 'snfeDump':
+        response = staticNetFilteringEngine.dump();
+        break;
+
+    case 'cfeDump':
+        response = cosmeticFilteringEngine.dump();
+        break;
+
+    default:
+        return vAPI.messaging.UNHANDLED;
+    }
+
+    callback(response);
+};
+
+vAPI.messaging.listen({
+    name: 'devTools',
+    listener: onMessage,
+    privileged: true,
+});
+
+// <<<<< end of local scope
+}
+
+/******************************************************************************/
+/******************************************************************************/
+
 // Channel:
 //      scriptlets
 //      unprivileged
@@ -2063,16 +2161,18 @@ const onMessage = function(request, sender, callback) {
     case 'updateLists':
         const listkeys = request.listkeys.split(',').filter(s => s !== '');
         if ( listkeys.length === 0 ) { return; }
-        for ( const listkey of listkeys ) {
-            io.purge(listkey);
-            io.remove(`compiled/${listkey}`);
+        if ( listkeys.includes('all') ) {
+            io.purge(/./, 'public_suffix_list.dat');
+        } else {
+            for ( const listkey of listkeys ) {
+                io.purge(listkey);
+            }
         }
-        µb.scheduleAssetUpdater(0);
         µb.openNewTab({
             url: 'dashboard.html#3p-filters.html',
             select: true,
         });
-        io.updateStart({ delay: 100, auto: request.manual !== true });
+        µb.scheduleAssetUpdater({ now: true, fetchDelay: 100, auto: request.auto });
         break;
 
     default: