@browserless.io/browserless 2.30.1 → 2.31.1-beta-1

package/extensions/ublocklite/js/ruleset-manager.js

@@ -35,11 +35,16 @@ import { dnr } from './ext-compat.js';
 import { fetchJSON } from './fetch.js';
 import { getAdminRulesets } from './admin.js';
 import { hasBroadHostPermissions } from './utils.js';
+import { rulesFromText } from './dnr-parser.js';
 import { ubolLog } from './debug.js';
 
 /******************************************************************************/
 
+const SPECIAL_RULES_REALM = 5000000;
+const USER_RULES_BASE_RULE_ID = 9000000;
+const USER_RULES_PRIORITY = 1000000;
 const TRUSTED_DIRECTIVE_BASE_RULE_ID = 8000000;
+const TRUSTED_DIRECTIVE_PRIORITY = USER_RULES_PRIORITY + 1000000;
 const STRICTBLOCK_PRIORITY = 29;
 
 let dynamicRegexCount = 0;
@@ -80,15 +85,12 @@ function getRulesetDetails() {
 
 /******************************************************************************/
 
-async function pruneInvalidRegexRules(realm, rulesIn) {
-    const rejectedRegexRules = [];
-
+async function pruneInvalidRegexRules(realm, rulesIn, rejected = []) {
     const validateRegex = regex => {
         return dnr.isRegexSupported({ regex, isCaseSensitive: false }).then(result => {
-            const isSupported = result?.isSupported || false;
-            pruneInvalidRegexRules.validated.set(regex, isSupported);
-            if ( isSupported ) { return true; }
-            rejectedRegexRules.push(`\t${regex}  ${result?.reason}`);
+            pruneInvalidRegexRules.validated.set(regex, result?.reason || true);
+            if ( result.isSupported ) { return true; }
+            rejected.push({ regex, reason: result?.reason });
             return false;
         });
     };
@@ -101,8 +103,11 @@ async function pruneInvalidRegexRules(realm, rulesIn) {
             continue;
         }
         const { regexFilter } = rule.condition;
-        if ( pruneInvalidRegexRules.validated.has(regexFilter) ) {
-            toCheck.push(pruneInvalidRegexRules.validated.get(regexFilter));
+        const reason = pruneInvalidRegexRules.validated.get(regexFilter);
+        if ( reason !== undefined ) {
+            toCheck.push(reason === true);
+            if ( reason === true  ) { continue; }
+            rejected.push({ regex: regexFilter, reason });
             continue;
         }
         toCheck.push(validateRegex(regexFilter));
@@ -111,9 +116,9 @@ async function pruneInvalidRegexRules(realm, rulesIn) {
     // Collate results
     const isValid = await Promise.all(toCheck);
 
-    if ( rejectedRegexRules.length !== 0 ) {
+    if ( rejected.length !== 0 ) {
         ubolLog(`${realm} realm: rejected regexes:\n`,
-            rejectedRegexRules.join('\n')
+            rejected.map(e => `${e.regex} → ${e.reason}`).join('\n')
         );
     }
 
@@ -126,6 +131,7 @@ pruneInvalidRegexRules.validated = new Map();
 async function updateRegexRules(currentRules, addRules, removeRuleIds) {
     // Remove existing regex-related block rules
     for ( const rule of currentRules ) {
+        if ( rule.id >= SPECIAL_RULES_REALM ) { continue; }
         const { type } = rule.action;
         if ( type !== 'block' && type !== 'allow' ) { continue; }
         if ( rule.condition.regexFilter === undefined ) { continue; }
@@ -164,6 +170,7 @@ async function updateRegexRules(currentRules, addRules, removeRuleIds) {
 async function updateRemoveparamRules(currentRules, addRules, removeRuleIds) {
     // Remove existing removeparam-related rules
     for ( const rule of currentRules ) {
+        if ( rule.id >= SPECIAL_RULES_REALM ) { continue; }
         if ( rule.action.type !== 'redirect' ) { continue; }
         if ( rule.action.redirect.transform === undefined ) { continue; }
         removeRuleIds.push(rule.id);
@@ -179,7 +186,6 @@ async function updateRemoveparamRules(currentRules, addRules, removeRuleIds) {
     }
     const removeparamRulesets = await Promise.all(toFetch);
 
-    // Removeparam rules can only be enforced with omnipotence
     const allRules = [];
     for ( const rules of removeparamRulesets ) {
         if ( Array.isArray(rules) === false ) { continue; }
@@ -201,6 +207,7 @@ async function updateRemoveparamRules(currentRules, addRules, removeRuleIds) {
 async function updateRedirectRules(currentRules, addRules, removeRuleIds) {
     // Remove existing redirect-related rules
     for ( const rule of currentRules ) {
+        if ( rule.id >= SPECIAL_RULES_REALM ) { continue; }
         if ( rule.action.type !== 'redirect' ) { continue; }
         if ( rule.action.redirect.extensionPath === undefined ) { continue; }
         removeRuleIds.push(rule.id);
@@ -216,7 +223,6 @@ async function updateRedirectRules(currentRules, addRules, removeRuleIds) {
     }
     const redirectRulesets = await Promise.all(toFetch);
 
-    // Redirect rules can only be enforced with omnipotence
     const allRules = [];
     for ( const rules of redirectRulesets ) {
         if ( Array.isArray(rules) === false ) { continue; }
@@ -238,6 +244,7 @@ async function updateRedirectRules(currentRules, addRules, removeRuleIds) {
 async function updateModifyHeadersRules(currentRules, addRules, removeRuleIds) {
     // Remove existing header modification-related rules
     for ( const rule of currentRules ) {
+        if ( rule.id >= SPECIAL_RULES_REALM ) { continue; }
         if ( rule.action.type !== 'modifyHeaders' ) { continue; }
         removeRuleIds.push(rule.id);
     }
@@ -252,7 +259,6 @@ async function updateModifyHeadersRules(currentRules, addRules, removeRuleIds) {
     }
     const rulesets = await Promise.all(toFetch);
 
-    // Redirect rules can only be enforced with omnipotence
     const allRules = [];
     for ( const rules of rulesets ) {
         if ( Array.isArray(rules) === false ) { continue; }
@@ -278,6 +284,7 @@ async function updateDynamicRules() {
 
     // Remove potentially left-over strict-block rules from previous version
     for ( const rule of currentRules ) {
+        if ( rule.id >= SPECIAL_RULES_REALM ) { continue; }
         if ( isStrictBlockRule(rule) === false ) { continue; }
         removeRuleIds.push(rule.id);
     }
@@ -294,7 +301,6 @@ async function updateDynamicRules() {
     let ruleId = 1;
     for ( const rule of addRules ) {
         if ( rule?.condition.regexFilter ) { dynamicRegexCount += 1; }
-        if ( (rule.id || 0) >= TRUSTED_DIRECTIVE_BASE_RULE_ID ) { continue; }
         rule.id = ruleId++;
     }
     if ( dynamicRegexCount !== 0 ) {
@@ -467,7 +473,8 @@ async function filteringModesToDNR(modes) {
         TRUSTED_DIRECTIVE_BASE_RULE_ID,
         requestDomains.sort(),
         excludedRequestDomains.sort(),
-        allowEverywhere
+        allowEverywhere,
+        TRUSTED_DIRECTIVE_PRIORITY
     ).then(modified => {
         if ( modified === false ) { return; }
         ubolLog(`${allowEverywhere ? 'Enabled' : 'Disabled'} DNR filtering for ${noneCount} sites`);
@@ -476,7 +483,7 @@ async function filteringModesToDNR(modes) {
 
 /******************************************************************************/
 
-async function defaultRulesetsFromLanguage() {
+async function defaultRulesetsFromEnv() {
     const dropCountry = lang => {
         const pos = lang.indexOf('-');
         if ( pos === -1 ) { return lang; }
@@ -494,6 +501,10 @@ async function defaultRulesetsFromLanguage() {
         `\\b(${Array.from(langSet).join('|')})\\b`
     );
 
+    const reMobile = /\bMobile\b/.test(navigator.userAgent)
+        ? /\bmobile\b/
+        : null
+
     const rulesetDetails = await getRulesetDetails();
     const out = [];
     for ( const ruleset of rulesetDetails.values() ) {
@@ -502,10 +513,20 @@ async function defaultRulesetsFromLanguage() {
             out.push(id);
             continue;
         }
-        if ( typeof ruleset.lang !== 'string' ) { continue; }
-        if ( reTargetLang.test(ruleset.lang) === false ) { continue; }
-        out.push(id);
+        if ( typeof ruleset.lang === 'string' ) {
+            if ( reTargetLang.test(ruleset.lang) ) {
+                out.push(id);
+                continue;
+            }
+        }
+        if ( typeof ruleset.tags === 'string' ) {
+            if ( reMobile?.test(ruleset.tags) ) {
+                out.push(id);
+                continue;
+            }
+        }
     }
+   
     return out;
 }
 
@@ -518,7 +539,7 @@ async function patchDefaultRulesets() {
         staticRulesetIds,
     ] = await Promise.all([
         localRead('defaultRulesetIds'),
-        defaultRulesetsFromLanguage(),
+        defaultRulesetsFromEnv(),
         getStaticRulesets().then(r => r.map(a => a.id)),
     ]);
     const toAdd = [];
@@ -648,15 +669,94 @@ async function getEnabledRulesetsDetails() {
 
 /******************************************************************************/
 
+async function getEffectiveUserRules() {
+    const allRules = await dnr.getDynamicRules();
+    const userRules = [];
+    for ( const rule of allRules ) {
+        if ( rule.id < USER_RULES_BASE_RULE_ID ) { continue; }
+        userRules.push(rule);
+    }
+    return userRules;
+}
+
+async function updateUserRules() {
+    const [
+        userRules,
+        userRulesText = '',
+    ] = await Promise.all([
+        getEffectiveUserRules(),
+        localRead('userDnrRules'),
+    ]);
+
+    const effectiveRulesText = rulesetConfig.developerMode
+        ? userRulesText
+        : '';
+
+    const parsed = rulesFromText(effectiveRulesText);
+    const { rules } = parsed;
+    const removeRuleIds = [ ...userRules.map(a => a.id) ];
+    const rejectedRegexes = [];
+    const addRules = await pruneInvalidRegexRules('user', rules, rejectedRegexes);
+    const out = { added: 0, removed: 0, errors: [] };
+
+    if ( rejectedRegexes.length !== 0 ) {
+        rejectedRegexes.forEach(e =>
+            out.errors.push(`regexFilter: ${e.regex} → ${e.reason}`)
+        );
+    }
+
+    if ( removeRuleIds.length === 0 && addRules.length === 0 ) {
+        await localRemove('userDnrRuleCount');
+        return out;
+    }
+
+    let ruleId = 0;
+    for ( const rule of addRules ) {
+        rule.id = USER_RULES_BASE_RULE_ID + ruleId++;
+        rule.priority = (rule.priority || 1) + USER_RULES_PRIORITY;
+    }
+
+    // Rules are first removed separately to ensure registered rules match
+    // user rules text. A bad rule in user rules text would prevent the
+    // rules from being removed if the removal was done at the same time as
+    // adding rules.
+    try {
+        await dnr.updateDynamicRules({ removeRuleIds });
+        await dnr.updateDynamicRules({ addRules });
+        if ( removeRuleIds.length !== 0 ) {
+            ubolLog(`updateUserRules() / Removed ${removeRuleIds.length} dynamic DNR rules`);
+        }
+        if ( addRules.length !== 0 ) {
+            ubolLog(`updateUserRules() / Added ${addRules.length} DNR rules`);
+        }
+        out.added = addRules.length;
+        out.removed = removeRuleIds.length;
+    } catch(reason) {
+        console.info(`updateUserRules() / ${reason}`);
+        out.errors.push(`${reason}`);
+    } finally {
+        const userRules = await getEffectiveUserRules();
+        if ( userRules.length === 0 ) {
+            await localRemove('userDnrRuleCount');
+        } else {
+            await localWrite('userDnrRuleCount', addRules.length);
+        }
+    }
+    return out;
+}
+
+/******************************************************************************/
+
 export {
-    defaultRulesetsFromLanguage,
     enableRulesets,
     excludeFromStrictBlock,
     filteringModesToDNR,
+    getEffectiveUserRules,
     getEnabledRulesetsDetails,
     getRulesetDetails,
     patchDefaultRulesets,
     setStrictBlockMode,
     updateDynamicRules,
     updateSessionRules,
+    updateUserRules,
 };

package/extensions/ublocklite/js/scripting-manager.js

@@ -21,7 +21,12 @@
 
 import * as ut from './utils.js';
 
-import { browser } from './ext.js';
+import {
+    browser,
+    localRemove,
+    localWrite,
+} from './ext.js';
+
 import { fetchJSON } from './fetch.js';
 import { getEnabledRulesetsDetails } from './ruleset-manager.js';
 import { getFilteringModeDetails } from './mode-manager.js';
@@ -142,11 +147,13 @@ function registerHighGeneric(context, genericDetails) {
     const directive = {
         id: 'css-generichigh',
         css,
-        allFrames: true,
         matches,
-        excludeMatches,
+        allFrames: true,
         runAt: 'document_end',
     };
+    if ( excludeMatches.length !== 0 ) {
+        directive.excludeMatches = excludeMatches;
+    }
 
     // register
     if ( registered === undefined ) {
@@ -239,9 +246,12 @@ function registerGeneric(context, genericDetails) {
         js,
         allFrames: true,
         matches: [ '<all_urls>' ],
-        excludeMatches,
         runAt: 'document_idle',
     };
+    if ( excludeMatches.length !== 0 ) {
+        directiveAll.excludeMatches = excludeMatches;
+    }
+
     if ( registeredAll === undefined ) { // register
         context.toAdd.push(directiveAll);
     } else if ( // update
@@ -316,11 +326,13 @@ function registerProcedural(context) {
     const directive = {
         id: 'css-procedural',
         js,
-        allFrames: true,
         matches,
-        excludeMatches,
+        allFrames: true,
         runAt: 'document_start',
     };
+    if ( excludeMatches.length !== 0 ) {
+        directive.excludeMatches = excludeMatches;
+    }
 
     // register
     if ( registered === undefined ) {
@@ -378,11 +390,13 @@ function registerDeclarative(context) {
     const directive = {
         id: 'css-declarative',
         js,
-        allFrames: true,
         matches,
-        excludeMatches,
+        allFrames: true,
         runAt: 'document_start',
     };
+    if ( excludeMatches.length !== 0 ) {
+        directive.excludeMatches = excludeMatches;
+    }
 
     // register
     if ( registered === undefined ) {
@@ -440,11 +454,13 @@ function registerSpecific(context) {
     const directive = {
         id: 'css-specific',
         js,
-        allFrames: true,
         matches,
-        excludeMatches,
+        allFrames: true,
         runAt: 'document_start',
     };
+    if ( excludeMatches.length !== 0 ) {
+        directive.excludeMatches = excludeMatches;
+    }
 
     // register
     if ( registered === undefined ) {
@@ -511,19 +527,22 @@ function registerScriptlet(context, scriptletDetails) {
             }
             if ( targetHostnames.length === 0 ) { continue; }
             matches.push(...ut.matchesFromHostnames(targetHostnames));
+            normalizeMatches(matches);
 
             before.delete(id); // Important!
 
             const directive = {
                 id,
                 js: [ `/rulesets/scripting/scriptlet/${id}.js` ],
-                allFrames: true,
                 matches,
-                excludeMatches,
+                allFrames: true,
                 matchOriginAsFallback: true,
                 runAt: 'document_start',
                 world: details.world,
             };
+            if ( excludeMatches.length !== 0 ) {
+                directive.excludeMatches = excludeMatches;
+            }
 
             // register
             if ( registered === undefined ) {
@@ -593,14 +612,24 @@ async function registerInjectables() {
 
     if ( toRemove.length !== 0 ) {
         ubolLog(`Unregistered ${toRemove} content (css/js)`);
-        await browser.scripting.unregisterContentScripts({ ids: toRemove })
-            .catch(reason => { console.info(reason); });
+        try {
+            await browser.scripting.unregisterContentScripts({ ids: toRemove });
+            localRemove('$scripting.unregisterContentScripts');
+        } catch(reason) {
+            localWrite('$scripting.unregisterContentScripts', `${reason}`);
+            console.info(reason);
+        }
     }
 
     if ( toAdd.length !== 0 ) {
         ubolLog(`Registered ${toAdd.map(v => v.id)} content (css/js)`);
-        await browser.scripting.registerContentScripts(toAdd)
-            .catch(reason => { console.info(reason); });
+        try {
+            await browser.scripting.registerContentScripts(toAdd);
+            localRemove('$scripting.registerContentScripts');
+        } catch(reason) {
+            localWrite('$scripting.registerContentScripts', `${reason}`);
+            console.info(reason);
+        }
     }
 
     registerInjectables.barrier = false;

package/extensions/ublocklite/js/settings.js

@@ -22,11 +22,23 @@
 import { browser, sendMessage } from './ext.js';
 import { dom, qs$ } from './dom.js';
 import { hashFromIterable } from './dashboard.js';
+import { i18n$ } from './i18n.js';
 import punycode from './punycode.js';
 import { renderFilterLists } from './filter-lists.js';
 
 /******************************************************************************/
 
+const cm6 = self.cm6;
+
+const cmTrustedSites = (( ) => {
+    const options = {};
+    if ( dom.cl.has(':root', 'dark') ) {
+        options.oneDark = true;
+    }
+    options.placeholder = i18n$('noFilteringModePlaceholder');
+    return cm6.createEditorView(options, qs$('#trustedSites'));
+})();
+
 let cachedRulesetData = {};
 
 /******************************************************************************/
@@ -70,12 +82,10 @@ function renderWidgets() {
     }
 
     {
-        dom.prop('#developerMode input[type="checkbox"]', 'checked',
-            Boolean(cachedRulesetData.developerMode)
-        );
-        if ( cachedRulesetData.isSideloaded ) {
-            dom.attr('#developerMode', 'hidden', null);
-        }
+        const state = Boolean(cachedRulesetData.developerMode) &&
+            cachedRulesetData.disabledFeatures?.includes('develop') !== true;
+        dom.body.dataset.develop = `${state}`;
+        dom.prop('#developerMode input[type="checkbox"]', 'checked', state);
     }
 }
 
@@ -158,21 +168,23 @@ dom.on('#strictBlockMode input[type="checkbox"]', 'change', ev => {
 });
 
 dom.on('#developerMode input[type="checkbox"]', 'change', ev => {
-    sendMessage({
-        what: 'setDeveloperMode',
-        state: ev.target.checked,
-    });
+    const state = ev.target.checked;
+    sendMessage({ what: 'setDeveloperMode', state });
+    dom.body.dataset.develop = `${state}`;
 });
 
 /******************************************************************************/
 
 function renderTrustedSites() {
-    const textarea = qs$('#trustedSites');
     const hostnames = cachedRulesetData.trustedSites || [];
-    textarea.value = hostnames.map(hn => punycode.toUnicode(hn)).join('\n');
-    if ( textarea.value !== '' ) {
-        textarea.value += '\n';
-    }
+    let text = hostnames.map(hn => punycode.toUnicode(hn)).join('\n');
+    if ( text !== '' ) { text += '\n'; }
+    cmTrustedSites.dispatch({
+        changes: {
+            from: 0, to: cmTrustedSites.state.doc.length,
+            insert: text
+        },
+    });
 }
 
 function changeTrustedSites() {
@@ -186,8 +198,9 @@ function changeTrustedSites() {
 }
 
 function getStagedTrustedSites() {
-    const textarea = qs$('#trustedSites');
-    return textarea.value.split(/\s/).map(hn => {
+    const text = cmTrustedSites.state.doc.toString();
+    return text.split(/\s/).map(hn => {
+        if ( hn === '' ) { return ''; }
         try {
             return punycode.toASCII(
                 (new URL(`https://${hn}/`)).hostname
@@ -198,7 +211,7 @@ function getStagedTrustedSites() {
     }).filter(hn => hn !== '');
 }
 
-dom.on('#trustedSites', 'blur', changeTrustedSites);
+dom.on(cmTrustedSites.contentDOM, 'blur', changeTrustedSites);
 
 self.addEventListener('beforeunload', changeTrustedSites);
 
@@ -296,7 +309,8 @@ sendMessage({
         renderFilterLists(cachedRulesetData);
         renderWidgets();
         dom.cl.remove(dom.body, 'loading');
-    } catch {
+    } catch(reason) {
+        console.error(reason);
     }
     listen();
 }).catch(reason => {

package/extensions/ublocklite/js/theme.js

@@ -34,5 +34,7 @@ import { dom } from './dom.js';
 
 {
     const mql = self.matchMedia('(hover: hover)');
-    dom.cl.toggle(dom.html, 'mobile', mql.matches !== true);
+    const isTouchScreen = mql.matches !== true;
+    dom.cl.toggle(dom.html, 'mobile', isTouchScreen);
+    dom.cl.toggle(dom.html, 'desktop', isTouchScreen === false);
 }

package/extensions/ublocklite/js/zapper-ui.js

@@ -126,6 +126,7 @@ const svgListening = (( ) => {
     };
 
     return state => {
+        if ( $stor(':root.mobile') !== null ) { return; }
         if ( state === on ) { return; }
         on = state;
         if ( on ) {
@@ -190,6 +191,7 @@ const startZapper = function(port) {
     $stor('svg#sea').addEventListener('touchstart', onSvgTouch, { passive: true });
     $stor('svg#sea').addEventListener('touchend', onSvgTouch);
     $id('quit').addEventListener('click', quitZapper );
+    $id('pick').addEventListener('click', resetZapper );
     svgListening(true);
 };
 
@@ -201,6 +203,7 @@ const quitZapper = function() {
     $stor('svg#sea').removeEventListener('touchstart', onSvgTouch, { passive: true });
     $stor('svg#sea').removeEventListener('touchend', onSvgTouch);
     $id('quit').removeEventListener('click', quitZapper );
+    $id('pick').removeEventListener('click', resetZapper );
     svgListening(false);
     if ( zapperScriptPort ) {
         zapperScriptPort.postMessage({ what: 'quitZapper' });
@@ -213,6 +216,14 @@ const quitZapper = function() {
 
 /******************************************************************************/
 
+const resetZapper = function() {
+    zapperScriptPort.postMessage({
+        what: 'unhighlight'
+    });
+};
+
+/******************************************************************************/
+
 // Wait for the content script to establish communication
 
 globalThis.addEventListener('message', ev => {

package/extensions/ublocklite/lib/codemirror/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Bryan Gillespie
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/extensions/ublocklite/lib/codemirror/README.md

@@ -0,0 +1,9 @@
+Steps to build `cm6.bundle.ubol.min.js` -- command line from repo root:
+
+- `git submodule init platform/mv3/extension/lib/codemirror/codemirror-ubol`
+- `cd platform/mv3/extension/lib/codemirror/codemirror-ubol/`
+    - We are now in a customized repo forked from <https://github.com/RPGillespie6/codemirror-quickstart>
+- `npm install`
+- `npm run build`
+- `cm6.bundle.ubol.min.js` should be in `dist` directory
+- This is the origin of the `cm6.bundle.ubol.min.js` in the current directory