@browserless.io/browserless 2.5.0-beta-4 → 2.6.0

package/extensions/ublock/js/storage.js

@@ -19,44 +19,39 @@
     Home: https://github.com/gorhill/uBlock
 */
 
-'use strict';
-
 /******************************************************************************/
 
-import publicSuffixList from '../lib/publicsuffixlist/publicsuffixlist.js';
-import punycode from '../lib/punycode.js';
+import * as sfp from './static-filtering-parser.js';
 
-import io from './assets.js';
+import { CompiledListReader, CompiledListWriter } from './static-filtering-io.js';
+import { LineIterator, orphanizeString } from './text-utils.js';
 import { broadcast, filteringBehaviorChanged, onBroadcast } from './broadcast.js';
+import { i18n, i18n$ } from './i18n.js';
+import {
+    permanentFirewall,
+    permanentSwitches,
+    permanentURLFiltering,
+} from './filtering-engines.js';
+import { ubolog, ubologSet } from './console.js';
+
 import cosmeticFilteringEngine from './cosmetic-filtering.js';
+import { hostnameFromURI } from './uri-utils.js';
+import io from './assets.js';
 import logger from './logger.js';
 import lz4Codec from './lz4.js';
+import publicSuffixList from '../lib/publicsuffixlist/publicsuffixlist.js';
+import punycode from '../lib/punycode.js';
+import { redirectEngine } from './redirect-engine.js';
 import staticExtFilteringEngine from './static-ext-filtering.js';
 import staticFilteringReverseLookup from './reverselookup.js';
 import staticNetFilteringEngine from './static-net-filtering.js';
 import µb from './background.js';
-import { hostnameFromURI } from './uri-utils.js';
-import { i18n, i18n$ } from './i18n.js';
-import { redirectEngine } from './redirect-engine.js';
-import { sparseBase64 } from './base64-custom.js';
-import { ubolog, ubologSet } from './console.js';
-import * as sfp from './static-filtering-parser.js';
-
-import {
-    permanentFirewall,
-    permanentSwitches,
-    permanentURLFiltering,
-} from './filtering-engines.js';
 
-import {
-    CompiledListReader,
-    CompiledListWriter,
-} from './static-filtering-io.js';
+/******************************************************************************/
 
-import {
-    LineIterator,
-    orphanizeString,
-} from './text-utils.js';
+// https://eslint.org/docs/latest/rules/no-prototype-builtins
+const hasOwnProperty = (o, p) =>
+    Object.prototype.hasOwnProperty.call(o, p);
 
 /******************************************************************************/
 
@@ -98,24 +93,80 @@ import {
 /******************************************************************************/
 
 {
-    let localSettingsLastSaved = Date.now();
+    const requestStats = µb.requestStats;
+    let requestStatsDisabled = false;
+
+    µb.loadLocalSettings = async ( ) => {
+        requestStatsDisabled = µb.hiddenSettings.requestStatsDisabled;
+        if ( requestStatsDisabled ) { return; }
+        return Promise.all([
+            vAPI.sessionStorage.get('requestStats'),
+            vAPI.storage.get('requestStats'),
+            vAPI.storage.get([ 'blockedRequestCount', 'allowedRequestCount' ]),
+        ]).then(([ a, b, c ]) => {
+            if ( a instanceof Object && a.requestStats ) { return a.requestStats; }
+            if ( b instanceof Object && b.requestStats ) { return b.requestStats; }
+            if ( c instanceof Object && Object.keys(c).length === 2 ) {
+                return {
+                    blockedCount: c.blockedRequestCount,
+                    allowedCount: c.allowedRequestCount,
+                };
+            }
+            return { blockedCount: 0, allowedCount: 0 };
+        }).then(({ blockedCount, allowedCount }) => {
+            requestStats.blockedCount += blockedCount;
+            requestStats.allowedCount += allowedCount;
+        });
+    };
 
-    const shouldSave = ( ) => {
-        if ( µb.localSettingsLastModified > localSettingsLastSaved ) {
-            µb.saveLocalSettings();
-        }
-        saveTimer.on(saveDelay);
+    const SAVE_DELAY_IN_MINUTES = 3.6;
+    const QUICK_SAVE_DELAY_IN_SECONDS = 23;
+
+    const stopTimers = ( ) => {
+        vAPI.alarms.clear('saveLocalSettings');
+        quickSaveTimer.off();
+        saveTimer.off();
     };
 
-    const saveTimer = vAPI.defer.create(shouldSave);
-    const saveDelay = { sec: 23 };
+    const saveTimer = vAPI.defer.create(( ) => {
+        µb.saveLocalSettings();
+    });
+
+    const quickSaveTimer = vAPI.defer.create(( ) => {
+        if ( vAPI.sessionStorage.unavailable !== true ) {
+            vAPI.sessionStorage.set({ requestStats: requestStats });
+        }
+        if ( requestStatsDisabled ) { return; }
+        saveTimer.on({ min: SAVE_DELAY_IN_MINUTES });
+        vAPI.alarms.createIfNotPresent('saveLocalSettings', {
+            delayInMinutes: SAVE_DELAY_IN_MINUTES + 0.5
+        });
+    });
 
-    saveTimer.onidle(saveDelay);
+    µb.incrementRequestStats = (blocked, allowed) => {
+        requestStats.blockedCount += blocked;
+        requestStats.allowedCount += allowed;
+        quickSaveTimer.on({ sec: QUICK_SAVE_DELAY_IN_SECONDS });
+    };
 
-    µb.saveLocalSettings = function() {
-        localSettingsLastSaved = Date.now();
-        return vAPI.storage.set(this.localSettings);
+    µb.saveLocalSettings = ( ) => {
+        stopTimers();
+        if ( requestStatsDisabled ) { return; }
+        return vAPI.storage.set({ requestStats: µb.requestStats });
     };
+
+    onBroadcast(msg => {
+        if ( msg.what !== 'hiddenSettingsChanged' ) { return; }
+        const newState = µb.hiddenSettings.requestStatsDisabled;
+        if ( requestStatsDisabled === newState ) { return; }
+        requestStatsDisabled = newState;
+        if ( newState ) {
+            stopTimers();
+            µb.requestStats.blockedCount = µb.requestStats.allowedCount = 0;
+        } else {
+            µb.loadLocalSettings();
+        }
+    });
 }
 
 /******************************************************************************/
@@ -136,7 +187,7 @@ import {
         for ( const entry of adminSettings ) {
             if ( entry.length < 1 ) { continue; }
             const name = entry[0];
-            if ( usDefault.hasOwnProperty(name) === false ) { continue; }
+            if ( hasOwnProperty(usDefault, name) === false ) { continue; }
             const value = entry.length < 2
                 ? usDefault[name]
                 : this.settingValueFromString(usDefault, name, entry[1]);
@@ -165,8 +216,8 @@ import {
 
     const toRemove = [];
     for ( const key in this.userSettings ) {
-        if ( this.userSettings.hasOwnProperty(key) === false ) { continue; }
-        if ( toSave.hasOwnProperty(key) ) { continue; }
+        if ( hasOwnProperty(this.userSettings, key) === false ) { continue; }
+        if ( hasOwnProperty(toSave, key) ) { continue; }
         toRemove.push(key);
     }
     if ( toRemove.length !== 0 ) {
@@ -203,7 +254,7 @@ import {
             for ( const entry of advancedSettings ) {
                 if ( entry.length < 1 ) { continue; }
                 const name = entry[0];
-                if ( hsDefault.hasOwnProperty(name) === false ) { continue; }
+                if ( hasOwnProperty(hsDefault, name) === false ) { continue; }
                 const value = entry.length < 2
                     ? hsDefault[name]
                     : this.hiddenSettingValueFromString(name, entry[1]);
@@ -237,8 +288,8 @@ import {
     }
 
     for ( const key in hsDefault ) {
-        if ( hsDefault.hasOwnProperty(key) === false ) { continue; }
-        if ( hsAdmin.hasOwnProperty(name) ) { continue; }
+        if ( hasOwnProperty(hsDefault, key) === false ) { continue; }
+        if ( hasOwnProperty(hsAdmin, name) ) { continue; }
         if ( typeof hs[key] !== typeof hsDefault[key] ) { continue; }
         this.hiddenSettings[key] = hs[key];
     }
@@ -283,8 +334,8 @@ onBroadcast(msg => {
         const matches = /^\s*(\S+)\s+(.+)$/.exec(line);
         if ( matches === null || matches.length !== 3 ) { continue; }
         const name = matches[1];
-        if ( out.hasOwnProperty(name) === false ) { continue; }
-        if ( this.hiddenSettingsAdmin.hasOwnProperty(name) ) { continue; }
+        if ( hasOwnProperty(out, name) === false ) { continue; }
+        if ( hasOwnProperty(this.hiddenSettingsAdmin, name) ) { continue; }
         const value = this.hiddenSettingValueFromString(name, matches[2]);
         if ( value !== undefined ) {
             out[name] = value;
@@ -296,7 +347,7 @@ onBroadcast(msg => {
 µb.hiddenSettingValueFromString = function(name, value) {
     if ( typeof name !== 'string' || typeof value !== 'string' ) { return; }
     const hsDefault = this.hiddenSettingsDefault;
-    if ( hsDefault.hasOwnProperty(name) === false ) { return; }
+    if ( hasOwnProperty(hsDefault, name) === false ) { return; }
     let r;
     switch ( typeof hsDefault[name] ) {
     case 'boolean':
@@ -369,6 +420,9 @@ onBroadcast(msg => {
 /******************************************************************************/
 
 µb.isTrustedList = function(assetKey) {
+    if ( assetKey === this.userFiltersPath ) {
+        if ( this.userSettings.userFiltersTrusted ) { return true; }
+    }
     if ( this.parsedTrustedListPrefixes.length === 0 ) {
         this.parsedTrustedListPrefixes =
             µb.hiddenSettings.trustedListPrefixes.split(/ +/).map(prefix => {
@@ -530,7 +584,6 @@ onBroadcast(msg => {
     // https://github.com/gorhill/uBlock/issues/1022
     //   Be sure to end with an empty line.
     content = content.trim();
-    if ( content !== '' ) { content += '\n'; }
     this.removeCompiledFilterList(this.userFiltersPath);
     return io.put(this.userFiltersPath, content);
 };
@@ -626,6 +679,7 @@ onBroadcast(msg => {
     cosmeticFilteringEngine.removeFromSelectorCache(
         hostnameFromURI(details.docURL)
     );
+    staticFilteringReverseLookup.resetLists();
 };
 
 µb.userFiltersAreEnabled = function() {
@@ -637,7 +691,7 @@ onBroadcast(msg => {
 µb.autoSelectRegionalFilterLists = function(lists) {
     const selectedListKeys = [ this.userFiltersPath ];
     for ( const key in lists ) {
-        if ( lists.hasOwnProperty(key) === false ) { continue; }
+        if ( hasOwnProperty(lists, key) === false ) { continue; }
         const list = lists[key];
         if ( list.content !== 'filters' ) { continue; }
         if ( list.off !== true ) {
@@ -891,7 +945,7 @@ onBroadcast(msg => {
         let acceptedCount = snfe.acceptedCount + sxfe.acceptedCount;
         let discardedCount = snfe.discardedCount + sxfe.discardedCount;
         µb.applyCompiledFilters(compiled, assetKey === µb.userFiltersPath);
-        if ( µb.availableFilterLists.hasOwnProperty(assetKey) ) {
+        if ( hasOwnProperty(µb.availableFilterLists, assetKey) ) {
             const entry = µb.availableFilterLists[assetKey];
             entry.entryCount = snfe.acceptedCount + sxfe.acceptedCount -
                 acceptedCount;
@@ -927,7 +981,7 @@ onBroadcast(msg => {
         // content.
         const toLoad = [];
         for ( const assetKey in lists ) {
-            if ( lists.hasOwnProperty(assetKey) === false ) { continue; }
+            if ( hasOwnProperty(lists, assetKey) === false ) { continue; }
             if ( lists[assetKey].off ) { continue; }
             toLoad.push(
                 µb.getCompiledFilterList(assetKey).then(details => {
@@ -974,7 +1028,7 @@ onBroadcast(msg => {
 /******************************************************************************/
 
 µb.getCompiledFilterList = async function(assetKey) {
-    const compiledPath = 'compiled/' + assetKey;
+    const compiledPath = `compiled/${assetKey}`;
 
     // https://github.com/uBlockOrigin/uBlock-issues/issues/1365
     //   Verify that the list version matches that of the current compiled
@@ -983,11 +1037,10 @@ onBroadcast(msg => {
         this.compiledFormatChanged === false &&
         this.badLists.has(assetKey) === false
     ) {
-        const compiledDetails = await io.get(compiledPath);
+        const content = await io.fromCache(compiledPath);
         const compilerVersion = `${this.systemSettings.compiledMagic}\n`;
-        if ( compiledDetails.content.startsWith(compilerVersion) ) {
-            compiledDetails.assetKey = assetKey;
-            return compiledDetails;
+        if ( content.startsWith(compilerVersion) ) {
+            return { assetKey, content };
         }
     }
 
@@ -1017,7 +1070,7 @@ onBroadcast(msg => {
         assetKey,
         trustedSource: this.isTrustedList(assetKey),
     });
-    io.put(compiledPath, compiledContent);
+    io.toCache(compiledPath, compiledContent);
 
     return { assetKey, content: compiledContent };
 };
@@ -1046,7 +1099,7 @@ onBroadcast(msg => {
 /******************************************************************************/
 
 µb.removeCompiledFilterList = function(assetKey) {
-    io.remove('compiled/' + assetKey);
+    io.remove(`compiled/${assetKey}`);
 };
 
 µb.removeFilterList = function(assetKey) {
@@ -1149,7 +1202,10 @@ onBroadcast(msg => {
 µb.loadRedirectResources = async function() {
     try {
         const success = await redirectEngine.resourcesFromSelfie(io);
-        if ( success === true ) { return true; }
+        if ( success === true ) {
+            ubolog('Loaded redirect/scriptlets resources from selfie');
+            return true;
+        }
 
         const fetcher = (path, options = undefined) => {
             if ( path.startsWith('/web_accessible_resources/') ) {
@@ -1173,20 +1229,17 @@ onBroadcast(msg => {
         const results = await Promise.all(fetchPromises);
         if ( Array.isArray(results) === false ) { return results; }
 
-        let content = '';
+        const content = [];
         for ( let i = 1; i < results.length; i++ ) {
             const result = results[i];
-            if (
-                result instanceof Object === false ||
-                typeof result.content !== 'string' ||
-                result.content === ''
-            ) {
-                continue;
-            }
-            content += '\n\n' + result.content;
+            if ( result instanceof Object === false ) { continue; }
+            if ( typeof result.content !== 'string' ) { continue; }
+            if ( result.content === '' ) { continue; }
+            content.push(result.content);
+        }
+        if ( content.length !== 0 ) {
+            redirectEngine.resourcesFromString(content.join('\n\n'));
         }
-
-        redirectEngine.resourcesFromString(content);
         redirectEngine.selfieFromResources(io);
     } catch(ex) {
         ubolog(ex);
@@ -1225,8 +1278,11 @@ onBroadcast(msg => {
     }
 
     try {
-        const result = await io.get(`compiled/${this.pslAssetKey}`);
-        if ( psl.fromSelfie(result.content, sparseBase64) ) { return; }
+        const selfie = await io.fromCache(`selfie/${this.pslAssetKey}`);
+        if ( psl.fromSelfie(selfie) ) {
+            ubolog('Loaded PSL from selfie');
+            return;
+        }
     } catch (reason) {
         ubolog(reason);
     }
@@ -1240,7 +1296,8 @@ onBroadcast(msg => {
 µb.compilePublicSuffixList = function(content) {
     const psl = publicSuffixList;
     psl.parse(content, punycode.toASCII);
-    io.put(`compiled/${this.pslAssetKey}`, psl.toSelfie(sparseBase64));
+    ubolog(`Loaded PSL from ${this.pslAssetKey}`);
+    return io.toCache(`selfie/${this.pslAssetKey}`, psl.toSelfie());
 };
 
 /******************************************************************************/
@@ -1260,39 +1317,24 @@ onBroadcast(msg => {
         if ( µb.inMemoryFilters.length !== 0 ) { return; }
         if ( Object.keys(µb.availableFilterLists).length === 0 ) { return; }
         await Promise.all([
-            io.put(
-                'selfie/main',
-                JSON.stringify({
-                    magic: µb.systemSettings.selfieMagic,
-                    availableFilterLists: µb.availableFilterLists,
-                })
-            ),
-            redirectEngine.toSelfie('selfie/redirectEngine'),
-            staticExtFilteringEngine.toSelfie(
-                'selfie/staticExtFilteringEngine'
+            io.toCache('selfie/staticMain', {
+                magic: µb.systemSettings.selfieMagic,
+                availableFilterLists: µb.availableFilterLists,
+            }),
+            io.toCache('selfie/staticExtFilteringEngine',
+                staticExtFilteringEngine.toSelfie()
             ),
-            staticNetFilteringEngine.toSelfie(io,
-                'selfie/staticNetFilteringEngine'
+            io.toCache('selfie/staticNetFilteringEngine',
+                staticNetFilteringEngine.toSelfie()
             ),
         ]);
         lz4Codec.relinquish();
         µb.selfieIsInvalid = false;
+        ubolog('Filtering engine selfie created');
     };
 
     const loadMain = async function() {
-        const details = await io.get('selfie/main');
-        if (
-            details instanceof Object === false ||
-            typeof details.content !== 'string' ||
-            details.content === ''
-        ) {
-            return false;
-        }
-        let selfie;
-        try {
-            selfie = JSON.parse(details.content);
-        } catch(ex) {
-        }
+        const selfie = await io.fromCache('selfie/staticMain');
         if ( selfie instanceof Object === false ) { return false; }
         if ( selfie.magic !== µb.systemSettings.selfieMagic ) { return false; }
         if ( selfie.availableFilterLists instanceof Object === false ) { return false; }
@@ -1306,12 +1348,11 @@ onBroadcast(msg => {
         try {
             const results = await Promise.all([
                 loadMain(),
-                redirectEngine.fromSelfie('selfie/redirectEngine'),
-                staticExtFilteringEngine.fromSelfie(
-                    'selfie/staticExtFilteringEngine'
+                io.fromCache('selfie/staticExtFilteringEngine').then(selfie =>
+                    staticExtFilteringEngine.fromSelfie(selfie)
                 ),
-                staticNetFilteringEngine.fromSelfie(io,
-                    'selfie/staticNetFilteringEngine'
+                io.fromCache('selfie/staticNetFilteringEngine').then(selfie =>
+                    staticNetFilteringEngine.fromSelfie(selfie)
                 ),
             ]);
             if ( results.every(v => v) ) {
@@ -1321,33 +1362,26 @@ onBroadcast(msg => {
         catch (reason) {
             ubolog(reason);
         }
+        ubolog('Filtering engine selfie not available');
         destroy();
         return false;
     };
 
-    const destroy = function() {
+    const destroy = function(options = {}) {
         if ( µb.selfieIsInvalid === false ) {
-            io.remove(/^selfie\//);
+            io.remove(/^selfie\/static/, options);
             µb.selfieIsInvalid = true;
-        }
-        if ( µb.wakeupReason === 'createSelfie' ) {
-            µb.wakeupReason = '';
-            return createTimer.offon({ sec: 27 });
+            ubolog('Filtering engine selfie marked for invalidation');
         }
         vAPI.alarms.create('createSelfie', {
-            delayInMinutes: µb.hiddenSettings.selfieAfter
+            delayInMinutes: (µb.hiddenSettings.selfieDelayInSeconds + 17) / 60,
         });
-        createTimer.offon({ min: µb.hiddenSettings.selfieAfter });
+        createTimer.offon({ sec: µb.hiddenSettings.selfieDelayInSeconds });
     };
 
     const createTimer = vAPI.defer.create(create);
 
-    vAPI.alarms.onAlarm.addListener(alarm => {
-        if ( alarm.name !== 'createSelfie') { return; }
-        µb.wakeupReason = 'createSelfie';
-    });
-
-    µb.selfieManager = { load, destroy };
+    µb.selfieManager = { load, create, destroy };
 }
 
 /******************************************************************************/
@@ -1399,8 +1433,8 @@ onBroadcast(msg => {
         const µbus = this.userSettings;
         const adminus = data.userSettings;
         for ( const name in µbus ) {
-            if ( µbus.hasOwnProperty(name) === false ) { continue; }
-            if ( adminus.hasOwnProperty(name) === false ) { continue; }
+            if ( hasOwnProperty(µbus, name) === false ) { continue; }
+            if ( hasOwnProperty(adminus, name) === false ) { continue; }
             bin[name] = adminus[name];
             binNotEmpty = true;
         }
@@ -1463,13 +1497,21 @@ onBroadcast(msg => {
         vAPI.storage.set(bin);
     }
 
-    if (
-        Array.isArray(toOverwrite.filters) &&
-        toOverwrite.filters.length !== 0
-    ) {
-        this.saveUserFilters(toOverwrite.filters.join('\n'));
+    let userFiltersAfter;
+    if ( Array.isArray(toOverwrite.filters) ) {
+        userFiltersAfter = toOverwrite.filters.join('\n').trim();
     } else if ( typeof data.userFilters === 'string' ) {
-        this.saveUserFilters(data.userFilters);
+        userFiltersAfter = data.userFilters.trim();
+    }
+    if ( typeof userFiltersAfter === 'string' ) {
+        const bin = await vAPI.storage.get(this.userFiltersPath);
+        const userFiltersBefore = bin && bin[this.userFiltersPath] || '';
+        if ( userFiltersAfter !== userFiltersBefore ) {
+            await Promise.all([
+                this.saveUserFilters(userFiltersAfter),
+                this.selfieManager.destroy(),
+            ]);
+        }
     }
 };
 
@@ -1507,7 +1549,6 @@ onBroadcast(msg => {
 
 {
     let next = 0;
-    let lastEmergencyUpdate = 0;
 
     const launchTimer = vAPI.defer.create(fetchDelay => {
         next = 0;
@@ -1516,6 +1557,7 @@ onBroadcast(msg => {
 
     µb.scheduleAssetUpdater = async function(details = {}) {
         launchTimer.off();
+        vAPI.alarms.clear('assetUpdater');
 
         if ( details.now ) {
             next = 0;
@@ -1534,40 +1576,23 @@ onBroadcast(msg => {
             this.hiddenSettings.autoUpdatePeriod * 3600000;
 
         const now = Date.now();
-        let needEmergencyUpdate = false;
-
-        // Respect cooldown period before launching an emergency update.
-        const timeSinceLastEmergencyUpdate = (now - lastEmergencyUpdate) / 3600000;
-        if ( timeSinceLastEmergencyUpdate > 1 ) {
-            const entries = await io.getUpdateAges({
-                filters: µb.selectedFilterLists,
-                internal: [ '*' ],
-            });
-            for ( const entry of entries ) {
-                if ( entry.ageNormalized < 2 ) { continue; }
-                needEmergencyUpdate = true;
-                lastEmergencyUpdate = now;
-                break;
-            }
-        }
 
         // Use the new schedule if and only if it is earlier than the previous
         // one.
         if ( next !== 0 ) {
-            updateDelay = Math.min(updateDelay, Math.max(next - now, 0));
-        }
-
-        if ( needEmergencyUpdate ) {
-            updateDelay = Math.min(updateDelay, 15000);
+            updateDelay = Math.min(updateDelay, Math.max(next - now, 1));
         }
 
         next = now + updateDelay;
 
-        const fetchDelay = needEmergencyUpdate
-            ? 2000
-            : this.hiddenSettings.autoUpdateAssetFetchPeriod * 1000 || 60000;
+        const fetchDelay = details.fetchDelay ||
+            this.hiddenSettings.autoUpdateAssetFetchPeriod * 1000 ||
+            60000;
 
         launchTimer.on(updateDelay, fetchDelay);
+        vAPI.alarms.create('assetUpdater', {
+            delayInMinutes: Math.ceil(updateDelay / 60000) + 0.25
+        });
     };
 }
 
@@ -1580,7 +1605,7 @@ onBroadcast(msg => {
     if ( topic === 'before-asset-updated' ) {
         if ( details.type === 'filters' ) {
             if (
-                this.availableFilterLists.hasOwnProperty(details.assetKey) === false ||
+                hasOwnProperty(this.availableFilterLists, details.assetKey) === false ||
                 this.selectedFilterLists.indexOf(details.assetKey) === -1 ||
                 this.badLists.get(details.assetKey)
             ) {
@@ -1594,9 +1619,8 @@ onBroadcast(msg => {
     if ( topic === 'after-asset-updated' ) {
         // Skip selfie-related content.
         if ( details.assetKey.startsWith('selfie/') ) { return; }
-        const cached = typeof details.content === 'string' &&
-                       details.content !== '';
-        if ( this.availableFilterLists.hasOwnProperty(details.assetKey) ) {
+        const cached = typeof details.content === 'string' && details.content !== '';
+        if ( hasOwnProperty(this.availableFilterLists, details.assetKey) ) {
             if ( cached ) {
                 if ( this.selectedFilterLists.indexOf(details.assetKey) !== -1 ) {
                     this.extractFilterListMetadata(
@@ -1604,8 +1628,7 @@ onBroadcast(msg => {
                         details.content
                     );
                     if ( this.badLists.has(details.assetKey) === false ) {
-                        io.put(
-                            'compiled/' + details.assetKey,
+                        io.toCache(`compiled/${details.assetKey}`,
                             this.compileFilters(details.content, {
                                 assetKey: details.assetKey,
                                 trustedSource: this.isTrustedList(details.assetKey),

package/extensions/ublock/js/traffic.js

@@ -590,7 +590,7 @@ const onHeadersReceived = function(details) {
     }
 };
 
-const reMediaContentTypes = /^(?:audio|image|video)\//;
+const reMediaContentTypes = /^(?:audio|image|video)\/|(?:\/ogg)$/;
 
 /******************************************************************************/
 
@@ -749,7 +749,7 @@ const bodyFilterer = (( ) => {
             /* t */ if ( bytes[i+6] !== 0x74 ) { continue; }
             break;
         }
-        if ( (i - 40) >= 65536 ) { return; }
+        if ( (i + 40) >= 65536 ) { return; }
         i += 8;
         // find first alpha character
         let j = -1;
@@ -827,13 +827,17 @@ const bodyFilterer = (( ) => {
         }
         if ( this.status !== 'finishedtransferringdata' ) { return; }
 
-        // If encoding is still unknown, try to extract from stream data
+        // If encoding is still unknown, try to extract from stream data.
+        // Just assume utf-8 if ultimately no encoding can be looked up.
         if ( session.charset === undefined ) {
             const charsetFound = charsetFromStream(session.buffer);
-            if ( charsetFound === undefined ) { return streamClose(session); }
-            const charsetUsed = textEncode.normalizeCharset(charsetFound);
-            if ( charsetUsed === undefined ) { return streamClose(session); }
-            session.charset = charsetUsed;
+            if ( charsetFound !== undefined ) {
+                const charsetUsed = textEncode.normalizeCharset(charsetFound);
+                if ( charsetUsed === undefined ) { return streamClose(session); }
+                session.charset = charsetUsed;
+            } else {
+                session.charset = 'utf-8';
+            }
         }
 
         while ( session.jobs.length !== 0 ) {