@adguard/filters-compiler 3.2.12 → 3.3.0

package/README.md

@@ -467,7 +467,8 @@ always fetched remotely, even when using the local cache.
 1. Call `download(basePath, includedFilterIds, excludedFilterIds)` to save
    `stats.json` for filters listed in the remote `percent.json`.
    `includedFilterIds` limits processing to those IDs (default `[]` = all).
-   `excludedFilterIds` skips those IDs (default `[]` = none). The two cannot use both.
+   `excludedFilterIds` skips those IDs (default `[]` = none). When both are
+   given, a filter is processed only if it is included and not excluded.
 2. Call `use(basePath)` before `compile()` to tell
    `getOptimizationStatistics` to read stats from local files under
    `basePath` instead of fetching from the remote server.

package/dist/index.cjs

@@ -17,7 +17,8 @@ var util = require('util');
 var module$1 = require('module');
 var jsdom = require('jsdom');
 var crypto = require('crypto');
-var moment = require('moment');
+var dateFns = require('date-fns');
+var utc = require('@date-fns/utc');
 var fs$1 = require('fs/promises');
 var tldts = require('tldts');
 var Ajv = require('ajv');
@@ -1187,43 +1188,50 @@ function assertValidStats(filterId, stats) {
 const localOptimizationStatistics = {
     /**
      * Downloads `stats.json` files for filters listed in the remote
-     * `percent.json` and saves them to disk.
-     * Existing `stats.json` files will be overwritten.
+     * `percent.json` and saves them to disk. Downloads into a staged
+     * directory first and swaps it in only after all downloads succeed,
+     * so a failed refresh leaves any previously cached stats intact.
      *
-     * `includedFilterIds` and `excludedFilterIds` cannot both be non-empty.
+     * When both `includedFilterIds` and `excludedFilterIds` are non-empty,
+     * a filter is processed only if it is in `includedFilterIds` and not in
+     * `excludedFilterIds`.
      *
      * @param basePath - Directory to save `filters/<filterId>/stats.json` into.
      * @param includedFilterIds - Filter IDs to process; empty (default) processes all.
      * @param excludedFilterIds - Filter IDs to exclude; empty (default) excludes none.
-     * @throws {Error} When both `includedFilterIds` and `excludedFilterIds` are non-empty.
      */
     download: async (basePath, includedFilterIds = [], excludedFilterIds = []) => {
-        if (includedFilterIds.length > 0 && excludedFilterIds.length > 0) {
-            throw new Error('includedFilterIds and excludedFilterIds cannot both be non-empty');
-        }
         const percent = JSON.parse(await downloadOptimizationPercent());
-        const configs = percent.config.filter(({ filterId }) => {
-            if (includedFilterIds.length > 0) {
-                return includedFilterIds.includes(filterId);
-            }
-            if (excludedFilterIds.length > 0) {
-                return !excludedFilterIds.includes(filterId);
-            }
-            return true;
-        });
+        const configs = percent.config.filter(({ filterId }) => (includedFilterIds.length === 0 || includedFilterIds.includes(filterId))
+            && (excludedFilterIds.length === 0 || !excludedFilterIds.includes(filterId)));
         const FILTERS_PATH = path$1.join(basePath, FILTERS_DIR_NAME);
+        const STAGED_PREFIX = `${process.pid}_${Date.now()}`;
+        const STAGED_FILTERS_PATH = path$1.join(basePath, `${STAGED_PREFIX}_${FILTERS_DIR_NAME}`);
         /**
          * Bounds concurrency so a large `percent.json` cannot fan out into unbounded
          * parallel requests if the underlying transport ever becomes truly async.
          */
         const DOWNLOAD_CONCURRENCY = 8;
-        await mapWithConcurrency(configs, DOWNLOAD_CONCURRENCY, async ({ filterId }) => {
-            const dir = path$1.join(FILTERS_PATH, String(filterId));
-            const statsPath = path$1.join(dir, STATS_JSON);
-            const content = await downloadOptimizationStats(filterId);
-            await fs$1.mkdir(dir, { recursive: true });
-            await fs$1.writeFile(statsPath, content, 'utf-8');
-        });
+        try {
+            await fs$1.mkdir(STAGED_FILTERS_PATH, { recursive: true });
+            await mapWithConcurrency(configs, DOWNLOAD_CONCURRENCY, async ({ filterId }) => {
+                const dir = path$1.join(STAGED_FILTERS_PATH, String(filterId));
+                const statsPath = path$1.join(dir, STATS_JSON);
+                const content = await downloadOptimizationStats(filterId);
+                await fs$1.mkdir(dir, { recursive: true });
+                await fs$1.writeFile(statsPath, content, 'utf-8');
+            });
+        }
+        catch (error) {
+            // Downloads incomplete: staged dir is garbage, safe to discard.
+            await fs$1.rm(STAGED_FILTERS_PATH, { recursive: true, force: true });
+            throw error;
+        }
+        // Swap only after every fetch succeeded. From here on, staged dir is
+        // a complete cache — never rm it on failure, that'd destroy the only
+        // remaining valid copy once the old FILTERS_PATH is gone.
+        await fs$1.rm(FILTERS_PATH, { recursive: true, force: true });
+        await fs$1.rename(STAGED_FILTERS_PATH, FILTERS_PATH);
     },
     /**
      * Configures `getOptimizationStatistics` to read stats from local files under
@@ -1697,7 +1705,7 @@ const makeHeader = function (metadataFile, revisionFile, platformsJsonExpires) {
         `! Title: ${metadata.name}`,
         `! Description: ${metadata.description}`,
         `! Version: ${revision.version}`,
-        `! TimeUpdated: ${moment(revision.timeUpdated).format()}`,
+        `! TimeUpdated: ${dateFns.format(revision.timeUpdated, 'yyyy-MM-dd\'T\'HH:mm:ssxxx', { in: utc.utc })}`,
         `! Expires: ${expires} (update frequency)`,
     ];
 };
@@ -2417,8 +2425,8 @@ const loadFilterMetadata = function (filterDir, includedFilterIds, excludedFilte
 
     const result = JSON.parse(metadataString);
     result.version = revision.version;
-    result.timeUpdated = moment(revision.timeUpdated).format('YYYY-MM-DDTHH:mm:ssZZ');
-    result.timeAdded = moment(result.timeAdded).format('YYYY-MM-DDTHH:mm:ssZZ');
+    result.timeUpdated = dateFns.format(revision.timeUpdated, 'yyyy-MM-dd\'T\'HH:mm:ssxx', { in: utc.utc });
+    result.timeAdded = dateFns.format(result.timeAdded, 'yyyy-MM-dd\'T\'HH:mm:ssxx', { in: utc.utc });
     delete result.disabled;
 
     const { filterId } = result;
@@ -4391,6 +4399,25 @@ const REPLACE_MODIFIER_PATTERNS = [
     '\\$(?!#|(path|domain)=.*]|.*removeparam=).*replace(,|=|$)',
 ];
 
+/**
+ * Pattern to check if rule contains `$urltransform` modifier
+ * and does not contain non-basic modifiers like `$domain` or `$path`.
+ *
+ * Pattern parts:
+ * - `\\$` — modifiers divider
+ * - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
+ *   `[$path=...]##.textad,[urltransform="ads"]` and rules with `$removeparam` modifier like `$removeparam=urltransform`
+ * - `.*urltransform` — urltransform modifier itself
+ * - `(,|=|$)` — end of line or modifiers divider, as `$urltransform` can be followed by other modifiers (`,`),
+ *   it may have a value (`=`), or it may be the last modifier in the rule (`$`).
+ *
+ * @example
+ * ```||example.com^$urltransform=/firstpath/secondpath/```
+ */
+const URLTRANSFORM_MODIFIER_PATTERNS = [
+    '\\$(?!#|(path|domain)=.*]|.*removeparam=).*urltransform(,|=|$)',
+];
+
 /* eslint-disable max-len */
 
 /**
@@ -4669,6 +4696,7 @@ const SAFARI_BASED_EXTENSION_PATTERNS = [
     ...REMOVEHEADER_MODIFIER_PATTERNS,
     ...MP4_MODIFIER_PATTERNS,
     ...REPLACE_MODIFIER_PATTERNS,
+    ...URLTRANSFORM_MODIFIER_PATTERNS,
     ...STEALTH_MODIFIER_PATTERNS,
     ...COOKIE_MODIFIER_PATTERNS,
     ...APP_MODIFIER_PATTERNS,
@@ -4932,6 +4960,7 @@ const platformsConfig = {
                 ...CSS_RULES_PATTERNS,
                 ...MP4_MODIFIER_PATTERNS,
                 ...REPLACE_MODIFIER_PATTERNS,
+                ...URLTRANSFORM_MODIFIER_PATTERNS,
                 ...STEALTH_MODIFIER_PATTERNS,
                 ...COOKIE_MODIFIER_PATTERNS,
                 ...EMPTY_MODIFIER_PATTERNS,
@@ -4965,6 +4994,7 @@ const platformsConfig = {
                 ...HTML_FILTERING_MODIFIER_PATTERNS,
                 ...MP4_MODIFIER_PATTERNS,
                 ...REPLACE_MODIFIER_PATTERNS,
+                ...URLTRANSFORM_MODIFIER_PATTERNS,
                 ...STEALTH_MODIFIER_PATTERNS,
                 ...COOKIE_MODIFIER_PATTERNS,
                 ...APP_MODIFIER_PATTERNS,

package/dist/index.js

@@ -15,7 +15,8 @@ import { TextEncoder, TextDecoder } from 'util';
 import { createRequire } from 'module';
 import { JSDOM } from 'jsdom';
 import crypto from 'crypto';
-import moment from 'moment';
+import { format } from 'date-fns';
+import { utc } from '@date-fns/utc';
 import fs$1 from 'fs/promises';
 import { parse as parse$1 } from 'tldts';
 import Ajv from 'ajv';
@@ -1184,43 +1185,50 @@ function assertValidStats(filterId, stats) {
 const localOptimizationStatistics = {
     /**
      * Downloads `stats.json` files for filters listed in the remote
-     * `percent.json` and saves them to disk.
-     * Existing `stats.json` files will be overwritten.
+     * `percent.json` and saves them to disk. Downloads into a staged
+     * directory first and swaps it in only after all downloads succeed,
+     * so a failed refresh leaves any previously cached stats intact.
      *
-     * `includedFilterIds` and `excludedFilterIds` cannot both be non-empty.
+     * When both `includedFilterIds` and `excludedFilterIds` are non-empty,
+     * a filter is processed only if it is in `includedFilterIds` and not in
+     * `excludedFilterIds`.
      *
      * @param basePath - Directory to save `filters/<filterId>/stats.json` into.
      * @param includedFilterIds - Filter IDs to process; empty (default) processes all.
      * @param excludedFilterIds - Filter IDs to exclude; empty (default) excludes none.
-     * @throws {Error} When both `includedFilterIds` and `excludedFilterIds` are non-empty.
      */
     download: async (basePath, includedFilterIds = [], excludedFilterIds = []) => {
-        if (includedFilterIds.length > 0 && excludedFilterIds.length > 0) {
-            throw new Error('includedFilterIds and excludedFilterIds cannot both be non-empty');
-        }
         const percent = JSON.parse(await downloadOptimizationPercent());
-        const configs = percent.config.filter(({ filterId }) => {
-            if (includedFilterIds.length > 0) {
-                return includedFilterIds.includes(filterId);
-            }
-            if (excludedFilterIds.length > 0) {
-                return !excludedFilterIds.includes(filterId);
-            }
-            return true;
-        });
+        const configs = percent.config.filter(({ filterId }) => (includedFilterIds.length === 0 || includedFilterIds.includes(filterId))
+            && (excludedFilterIds.length === 0 || !excludedFilterIds.includes(filterId)));
         const FILTERS_PATH = path$1.join(basePath, FILTERS_DIR_NAME);
+        const STAGED_PREFIX = `${process.pid}_${Date.now()}`;
+        const STAGED_FILTERS_PATH = path$1.join(basePath, `${STAGED_PREFIX}_${FILTERS_DIR_NAME}`);
         /**
          * Bounds concurrency so a large `percent.json` cannot fan out into unbounded
          * parallel requests if the underlying transport ever becomes truly async.
          */
         const DOWNLOAD_CONCURRENCY = 8;
-        await mapWithConcurrency(configs, DOWNLOAD_CONCURRENCY, async ({ filterId }) => {
-            const dir = path$1.join(FILTERS_PATH, String(filterId));
-            const statsPath = path$1.join(dir, STATS_JSON);
-            const content = await downloadOptimizationStats(filterId);
-            await fs$1.mkdir(dir, { recursive: true });
-            await fs$1.writeFile(statsPath, content, 'utf-8');
-        });
+        try {
+            await fs$1.mkdir(STAGED_FILTERS_PATH, { recursive: true });
+            await mapWithConcurrency(configs, DOWNLOAD_CONCURRENCY, async ({ filterId }) => {
+                const dir = path$1.join(STAGED_FILTERS_PATH, String(filterId));
+                const statsPath = path$1.join(dir, STATS_JSON);
+                const content = await downloadOptimizationStats(filterId);
+                await fs$1.mkdir(dir, { recursive: true });
+                await fs$1.writeFile(statsPath, content, 'utf-8');
+            });
+        }
+        catch (error) {
+            // Downloads incomplete: staged dir is garbage, safe to discard.
+            await fs$1.rm(STAGED_FILTERS_PATH, { recursive: true, force: true });
+            throw error;
+        }
+        // Swap only after every fetch succeeded. From here on, staged dir is
+        // a complete cache — never rm it on failure, that'd destroy the only
+        // remaining valid copy once the old FILTERS_PATH is gone.
+        await fs$1.rm(FILTERS_PATH, { recursive: true, force: true });
+        await fs$1.rename(STAGED_FILTERS_PATH, FILTERS_PATH);
     },
     /**
      * Configures `getOptimizationStatistics` to read stats from local files under
@@ -1694,7 +1702,7 @@ const makeHeader = function (metadataFile, revisionFile, platformsJsonExpires) {
         `! Title: ${metadata.name}`,
         `! Description: ${metadata.description}`,
         `! Version: ${revision.version}`,
-        `! TimeUpdated: ${moment(revision.timeUpdated).format()}`,
+        `! TimeUpdated: ${format(revision.timeUpdated, 'yyyy-MM-dd\'T\'HH:mm:ssxxx', { in: utc })}`,
         `! Expires: ${expires} (update frequency)`,
     ];
 };
@@ -2414,8 +2422,8 @@ const loadFilterMetadata = function (filterDir, includedFilterIds, excludedFilte
 
     const result = JSON.parse(metadataString);
     result.version = revision.version;
-    result.timeUpdated = moment(revision.timeUpdated).format('YYYY-MM-DDTHH:mm:ssZZ');
-    result.timeAdded = moment(result.timeAdded).format('YYYY-MM-DDTHH:mm:ssZZ');
+    result.timeUpdated = format(revision.timeUpdated, 'yyyy-MM-dd\'T\'HH:mm:ssxx', { in: utc });
+    result.timeAdded = format(result.timeAdded, 'yyyy-MM-dd\'T\'HH:mm:ssxx', { in: utc });
     delete result.disabled;
 
     const { filterId } = result;
@@ -4388,6 +4396,25 @@ const REPLACE_MODIFIER_PATTERNS = [
     '\\$(?!#|(path|domain)=.*]|.*removeparam=).*replace(,|=|$)',
 ];
 
+/**
+ * Pattern to check if rule contains `$urltransform` modifier
+ * and does not contain non-basic modifiers like `$domain` or `$path`.
+ *
+ * Pattern parts:
+ * - `\\$` — modifiers divider
+ * - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
+ *   `[$path=...]##.textad,[urltransform="ads"]` and rules with `$removeparam` modifier like `$removeparam=urltransform`
+ * - `.*urltransform` — urltransform modifier itself
+ * - `(,|=|$)` — end of line or modifiers divider, as `$urltransform` can be followed by other modifiers (`,`),
+ *   it may have a value (`=`), or it may be the last modifier in the rule (`$`).
+ *
+ * @example
+ * ```||example.com^$urltransform=/firstpath/secondpath/```
+ */
+const URLTRANSFORM_MODIFIER_PATTERNS = [
+    '\\$(?!#|(path|domain)=.*]|.*removeparam=).*urltransform(,|=|$)',
+];
+
 /* eslint-disable max-len */
 
 /**
@@ -4666,6 +4693,7 @@ const SAFARI_BASED_EXTENSION_PATTERNS = [
     ...REMOVEHEADER_MODIFIER_PATTERNS,
     ...MP4_MODIFIER_PATTERNS,
     ...REPLACE_MODIFIER_PATTERNS,
+    ...URLTRANSFORM_MODIFIER_PATTERNS,
     ...STEALTH_MODIFIER_PATTERNS,
     ...COOKIE_MODIFIER_PATTERNS,
     ...APP_MODIFIER_PATTERNS,
@@ -4929,6 +4957,7 @@ const platformsConfig = {
                 ...CSS_RULES_PATTERNS,
                 ...MP4_MODIFIER_PATTERNS,
                 ...REPLACE_MODIFIER_PATTERNS,
+                ...URLTRANSFORM_MODIFIER_PATTERNS,
                 ...STEALTH_MODIFIER_PATTERNS,
                 ...COOKIE_MODIFIER_PATTERNS,
                 ...EMPTY_MODIFIER_PATTERNS,
@@ -4962,6 +4991,7 @@ const platformsConfig = {
                 ...HTML_FILTERING_MODIFIER_PATTERNS,
                 ...MP4_MODIFIER_PATTERNS,
                 ...REPLACE_MODIFIER_PATTERNS,
+                ...URLTRANSFORM_MODIFIER_PATTERNS,
                 ...STEALTH_MODIFIER_PATTERNS,
                 ...COOKIE_MODIFIER_PATTERNS,
                 ...APP_MODIFIER_PATTERNS,

package/dist/main/optimization.d.ts

@@ -52,15 +52,17 @@ export declare function assertValidStats(filterId: number, stats: unknown): asse
 export declare const localOptimizationStatistics: {
     /**
      * Downloads `stats.json` files for filters listed in the remote
-     * `percent.json` and saves them to disk.
-     * Existing `stats.json` files will be overwritten.
+     * `percent.json` and saves them to disk. Downloads into a staged
+     * directory first and swaps it in only after all downloads succeed,
+     * so a failed refresh leaves any previously cached stats intact.
      *
-     * `includedFilterIds` and `excludedFilterIds` cannot both be non-empty.
+     * When both `includedFilterIds` and `excludedFilterIds` are non-empty,
+     * a filter is processed only if it is in `includedFilterIds` and not in
+     * `excludedFilterIds`.
      *
      * @param basePath - Directory to save `filters/<filterId>/stats.json` into.
      * @param includedFilterIds - Filter IDs to process; empty (default) processes all.
      * @param excludedFilterIds - Filter IDs to exclude; empty (default) excludes none.
-     * @throws {Error} When both `includedFilterIds` and `excludedFilterIds` are non-empty.
      */
     download: (basePath: string, includedFilterIds?: number[], excludedFilterIds?: number[]) => Promise<void>;
     /**

package/dist/types/src/main/optimization.d.ts

@@ -52,15 +52,17 @@ export declare function assertValidStats(filterId: number, stats: unknown): asse
 export declare const localOptimizationStatistics: {
     /**
      * Downloads `stats.json` files for filters listed in the remote
-     * `percent.json` and saves them to disk.
-     * Existing `stats.json` files will be overwritten.
+     * `percent.json` and saves them to disk. Downloads into a staged
+     * directory first and swaps it in only after all downloads succeed,
+     * so a failed refresh leaves any previously cached stats intact.
      *
-     * `includedFilterIds` and `excludedFilterIds` cannot both be non-empty.
+     * When both `includedFilterIds` and `excludedFilterIds` are non-empty,
+     * a filter is processed only if it is in `includedFilterIds` and not in
+     * `excludedFilterIds`.
      *
      * @param basePath - Directory to save `filters/<filterId>/stats.json` into.
      * @param includedFilterIds - Filter IDs to process; empty (default) processes all.
      * @param excludedFilterIds - Filter IDs to exclude; empty (default) excludes none.
-     * @throws {Error} When both `includedFilterIds` and `excludedFilterIds` are non-empty.
      */
     download: (basePath: string, includedFilterIds?: number[], excludedFilterIds?: number[]) => Promise<void>;
     /**

package/dist/utils/trust-levels/exclusions-full.txt

@@ -5,6 +5,7 @@
 ! ##+js
 ! #$#
 ! $replace
+! $urltransform
 ! #%#
 ! #@%#
 !

package/dist/utils/trust-levels/exclusions-high.txt

@@ -9,5 +9,6 @@
 #%#//scriptlet('trusted-
 #%#//scriptlet("trusted-
 $replace
+$urltransform
 #%#
-#@%#
+#@%#

package/dist/utils/trust-levels/exclusions-low.txt

@@ -7,5 +7,6 @@
 ##+js
 #$#
 $replace
+$urltransform
 #%#
-#@%#
+#@%#

package/package.json

@@ -21,20 +21,21 @@
     "pnpm": ">=10.33.4 <11"
   },
   "dependencies": {
-    "@adguard/agtree": "4.1.2",
+    "@adguard/agtree": "4.2.1",
     "@adguard/css-tokenizer": "1.2.0",
     "@adguard/ecss-tree": "2.0.1",
-    "@adguard/extended-css": "2.1.1",
+    "@adguard/extended-css": "2.2.0",
     "@adguard/filters-downloader": "2.4.4",
     "@adguard/logger": "2.0.0",
-    "@adguard/scriptlets": "2.3.1",
-    "@adguard/tsurlfilter": "4.0.5",
+    "@adguard/scriptlets": "2.5.1",
+    "@adguard/tsurlfilter": "6.0.3",
+    "@date-fns/utc": "2.1.1",
     "@eslint/css-tree": "3.6.6",
     "ajv": "8.17.1",
     "child_process": "1.0.2",
+    "date-fns": "^4.4.0",
     "jsdom": "21.1.2",
     "md5": "2.3.0",
-    "moment": "2.30.1",
     "tldts": "5.7.112",
     "utf8": "3.0.0"
   },
@@ -58,7 +59,7 @@
     "typescript": "6.0.3",
     "vitest": "3.0.5"
   },
-  "version": "3.2.12",
+  "version": "3.3.0",
   "scripts": {
     "prebuild": "rimraf dist",
     "build": "rollup --config rollup.config.js --silent",