@adguard/filters-compiler 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +166 -0
- package/README.md +242 -0
- package/dist/build.txt +1 -0
- package/dist/index.cjs +4663 -0
- package/dist/index.js +4658 -0
- package/dist/schemas/filters.schema.json +275 -0
- package/dist/schemas/filters_i18n.schema.json +116592 -0
- package/dist/schemas/mac/filters.schema.json +184 -0
- package/dist/schemas/mac/filters_i18n.schema.json +22667 -0
- package/dist/schemas/mac_v2/filters.schema.json +264 -0
- package/dist/schemas/mac_v2/filters_i18n.schema.json +36358 -0
- package/package.json +56 -0
package/dist/index.cjs
ADDED
|
@@ -0,0 +1,4663 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var path = require('path');
|
|
4
|
+
var tsurlfilter = require('@adguard/tsurlfilter');
|
|
5
|
+
var fs = require('fs');
|
|
6
|
+
var md5 = require('md5');
|
|
7
|
+
var filtersDownloader = require('@adguard/filters-downloader');
|
|
8
|
+
var agtree = require('@adguard/agtree');
|
|
9
|
+
var logger$1 = require('@adguard/logger');
|
|
10
|
+
var parser = require('@adguard/agtree/parser');
|
|
11
|
+
var ecssTree = require('@adguard/ecss-tree');
|
|
12
|
+
var util = require('util');
|
|
13
|
+
var module$1 = require('module');
|
|
14
|
+
var jsdom = require('jsdom');
|
|
15
|
+
var crypto = require('crypto');
|
|
16
|
+
var moment = require('moment');
|
|
17
|
+
var tldts = require('tldts');
|
|
18
|
+
var Ajv = require('ajv');
|
|
19
|
+
|
|
20
|
+
var _documentCurrentScript = typeof document !== 'undefined' ? document.currentScript : null;
|
|
21
|
+
/**
|
|
22
|
+
* Version utility functions
|
|
23
|
+
*/
|
|
24
|
+
const version = {
|
|
25
|
+
/**
|
|
26
|
+
* Parses version from string
|
|
27
|
+
*
|
|
28
|
+
* @param v version string
|
|
29
|
+
* @returns {Array}
|
|
30
|
+
*/
|
|
31
|
+
parse(v) {
|
|
32
|
+
const version = [];
|
|
33
|
+
const parts = String(v || '').split('.');
|
|
34
|
+
|
|
35
|
+
const parseVersionPart = (part) => {
|
|
36
|
+
if (Number.isNaN(part)) {
|
|
37
|
+
return 0;
|
|
38
|
+
}
|
|
39
|
+
return Math.max(part - 0, 0);
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
43
|
+
for (const part of parts) {
|
|
44
|
+
version.push(parseVersionPart(part));
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
return version;
|
|
48
|
+
},
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Increments build part of version '0.0.0.0'
|
|
52
|
+
*
|
|
53
|
+
* @param v version string
|
|
54
|
+
* @returns {string}
|
|
55
|
+
*/
|
|
56
|
+
increment(v) {
|
|
57
|
+
const version = this.parse(v);
|
|
58
|
+
|
|
59
|
+
if (version.length > 0) {
|
|
60
|
+
version[version.length - 1] = version[version.length - 1] + 1;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
for (let i = version.length; i > 0; i -= 1) {
|
|
64
|
+
if (version[i] === 100) {
|
|
65
|
+
version[i] = 0;
|
|
66
|
+
version[i - 1] += 1;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
return version.join('.');
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Export logger implementation.
|
|
76
|
+
*/
|
|
77
|
+
const logger = new logger$1.Logger(console);
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Initializes logger
|
|
81
|
+
*
|
|
82
|
+
* @param path log file
|
|
83
|
+
* @param level log lvl
|
|
84
|
+
*/
|
|
85
|
+
logger.initialize = (path) => {
|
|
86
|
+
if (!path) {
|
|
87
|
+
/* eslint-disable-next-line no-console */
|
|
88
|
+
console.warn('Log file is not specified');
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
fs.openSync(path, 'w');
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
// TODO: a lot of these masks can be imported from @adguard/agtree
|
|
95
|
+
/**
|
|
96
|
+
* Rule masks constants
|
|
97
|
+
*/
|
|
98
|
+
const RuleMasks = {
|
|
99
|
+
MASK_REGEX_RULE: '/',
|
|
100
|
+
MASK_RULE_SEPARATOR: '^',
|
|
101
|
+
MASK_WHITE_LIST: '@@',
|
|
102
|
+
MASK_BASE_RULE: '||',
|
|
103
|
+
MASK_ELEMENT_HIDING: '##',
|
|
104
|
+
MASK_ELEMENT_HIDING_EXCEPTION: '#@#',
|
|
105
|
+
MASK_CSS: '#$#',
|
|
106
|
+
MASK_CSS_EXCEPTION: '#@$#',
|
|
107
|
+
MASK_CSS_EXTENDED_CSS_RULE: '#?#',
|
|
108
|
+
MASK_CSS_EXCEPTION_EXTENDED_CSS_RULE: '#@?#',
|
|
109
|
+
MASK_CSS_INJECT_EXTENDED_CSS_RULE: '#$?#',
|
|
110
|
+
MASK_CSS_EXCEPTION_INJECT_EXTENDED_CSS_RULE: '#@$?#',
|
|
111
|
+
MASK_SCRIPT: '#%#',
|
|
112
|
+
MASK_SCRIPT_EXCEPTION: '#@%#',
|
|
113
|
+
MASK_CONTENT: '$$',
|
|
114
|
+
MASK_CONTENT_EXCEPTION: '$@$',
|
|
115
|
+
MASK_COMMENT: '!',
|
|
116
|
+
MASK_HOST_FILE_COMMENT: '#',
|
|
117
|
+
MASK_HINT: '!+',
|
|
118
|
+
MASK_DIRECTIVES: '!#',
|
|
119
|
+
MASK_SCRIPTLET: '#%#//scriptlet',
|
|
120
|
+
MASK_SCRIPTLET_EXCEPTION: '#@%#//scriptlet',
|
|
121
|
+
MASK_TRUSTED_SCRIPTLET: 'trusted-',
|
|
122
|
+
};
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Excludes rule
|
|
126
|
+
* @param {string} rule
|
|
127
|
+
* @param {array} excluded
|
|
128
|
+
* @param {string} message
|
|
129
|
+
*/
|
|
130
|
+
const excludeRule$1 = (rule, excluded, message) => {
|
|
131
|
+
if (excluded) {
|
|
132
|
+
excluded.push(`! ${message}`);
|
|
133
|
+
excluded.push(rule);
|
|
134
|
+
}
|
|
135
|
+
};
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Converts rules to AdGuard syntax
|
|
139
|
+
* @param {array} rulesList
|
|
140
|
+
* @param {array} [excluded]
|
|
141
|
+
* @return {array} result
|
|
142
|
+
*/
|
|
143
|
+
const convertRulesToAdgSyntax = (rulesList, excluded) => {
|
|
144
|
+
const result = [];
|
|
145
|
+
|
|
146
|
+
for (let i = 0; i < rulesList.length; i += 1) {
|
|
147
|
+
const rule = rulesList[i];
|
|
148
|
+
try {
|
|
149
|
+
const ruleNode = agtree.RuleParser.parse(rule);
|
|
150
|
+
const conversionResult = agtree.RuleConverter.convertToAdg(ruleNode);
|
|
151
|
+
const convertedRules = conversionResult.result.map((r) => agtree.RuleGenerator.generate(r));
|
|
152
|
+
result.push(...convertedRules);
|
|
153
|
+
|
|
154
|
+
if (conversionResult.isConverted) {
|
|
155
|
+
const message = `Rule "${rule}" converted to: "${[...convertedRules]}"`;
|
|
156
|
+
excludeRule$1(rule, excluded, message);
|
|
157
|
+
}
|
|
158
|
+
} catch (e) {
|
|
159
|
+
const message = `Unable to convert rule to AdGuard syntax: "${rule}" due to error: ${e.message}`;
|
|
160
|
+
logger.info(message);
|
|
161
|
+
excludeRule$1(rule, excluded, message);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
return result;
|
|
166
|
+
};
|
|
167
|
+
|
|
168
|
+
const SINGLE_QUOTE = "'";
|
|
169
|
+
const DOUBLE_QUOTE = '"';
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Trims quotes (single or double) from the start and end of a string if they exist.
|
|
173
|
+
*
|
|
174
|
+
* @param {string} str - The string to trim.
|
|
175
|
+
* @returns {string} The trimmed string.
|
|
176
|
+
*/
|
|
177
|
+
const trimQuotes = (str) => {
|
|
178
|
+
if (
|
|
179
|
+
(str.startsWith(SINGLE_QUOTE) && str.endsWith(SINGLE_QUOTE))
|
|
180
|
+
|| (str.startsWith(DOUBLE_QUOTE) && str.endsWith(DOUBLE_QUOTE))
|
|
181
|
+
) {
|
|
182
|
+
return str.slice(1, -1);
|
|
183
|
+
}
|
|
184
|
+
return str;
|
|
185
|
+
};
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* Converts a list of rules into uBO (uBlock Origin) syntax.
|
|
189
|
+
*
|
|
190
|
+
* @param {string[]} rules - An array of rules to be converted.
|
|
191
|
+
* @returns {string[]} An array of converted rules in uBO syntax. If no rules are provided, an empty array is returned.
|
|
192
|
+
*
|
|
193
|
+
* @throws {Error} Logs an error message if a rule cannot be converted due to a parsing or conversion issue.
|
|
194
|
+
*/
|
|
195
|
+
const convertToUbo = (rules) => {
|
|
196
|
+
const modified = [];
|
|
197
|
+
if (!rules) {
|
|
198
|
+
return modified;
|
|
199
|
+
}
|
|
200
|
+
rules.forEach((rule) => {
|
|
201
|
+
if (rule) {
|
|
202
|
+
try {
|
|
203
|
+
const ruleNode = agtree.RuleParser.parse(rule);
|
|
204
|
+
// js injection rules are not supported in uBO
|
|
205
|
+
if (ruleNode.type === agtree.CosmeticRuleType.JsInjectionRule) {
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
if (ruleNode.type === agtree.CosmeticRuleType.ScriptletInjectionRule) {
|
|
209
|
+
const scriptletNode = ruleNode.body.children[0].children[0];
|
|
210
|
+
const scriptletNameString = trimQuotes(scriptletNode.value);
|
|
211
|
+
|
|
212
|
+
if (!scriptletNameString) {
|
|
213
|
+
// If the scriptlet name is missing, skip processing this rule
|
|
214
|
+
return;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
const scriptletName = trimQuotes(scriptletNameString);
|
|
218
|
+
|
|
219
|
+
// TODO: move this check to AGTree AG-41266
|
|
220
|
+
if (scriptletName.startsWith(RuleMasks.MASK_TRUSTED_SCRIPTLET)) {
|
|
221
|
+
// https://github.com/AdguardTeam/Scriptlets#trusted-scriptlets-restriction
|
|
222
|
+
// does not work in other blockers
|
|
223
|
+
const message = `Trusted scriptlets should not be converted to uBO syntax. Rule: "${rule}"`;
|
|
224
|
+
logger.info(message);
|
|
225
|
+
modified.push('');
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
const conversionResult = agtree.RuleConverter.convertToUbo(ruleNode);
|
|
230
|
+
const convertedRules = conversionResult.result.map((r) => agtree.RuleGenerator.generate(r));
|
|
231
|
+
modified.push(...convertedRules);
|
|
232
|
+
} catch (e) {
|
|
233
|
+
const message = `Unable to convert rule to Ubo syntax: "${rule}" due to error: ${e.message}`;
|
|
234
|
+
logger.info(message);
|
|
235
|
+
}
|
|
236
|
+
} else {
|
|
237
|
+
modified.push('');
|
|
238
|
+
}
|
|
239
|
+
});
|
|
240
|
+
return modified;
|
|
241
|
+
};
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* ExtendedCss is not supposed to work without window environment,
|
|
245
|
+
* so we pass some wrapper dummy.
|
|
246
|
+
*/
|
|
247
|
+
// TODO: switch to aglint in compiler
|
|
248
|
+
|
|
249
|
+
global.TextEncoder = util.TextEncoder;
|
|
250
|
+
global.TextDecoder = util.TextDecoder;
|
|
251
|
+
|
|
252
|
+
const dom = new jsdom.JSDOM('<!DOCTYPE html><p>Empty</p>');
|
|
253
|
+
|
|
254
|
+
global.window = dom.window;
|
|
255
|
+
global.document = global.window.document;
|
|
256
|
+
if (!global.navigator) {
|
|
257
|
+
global.navigator = global.window.navigator;
|
|
258
|
+
}
|
|
259
|
+
global.Element = global.window.Element;
|
|
260
|
+
|
|
261
|
+
const require$2 = module$1.createRequire((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('index.cjs', document.baseURI).href)));
|
|
262
|
+
const { ExtendedCss } = require$2('@adguard/extended-css');
|
|
263
|
+
|
|
264
|
+
// TODO: in future SelectorValidationResult from ExtendedCss may be imported and used instead of it
|
|
265
|
+
/**
|
|
266
|
+
* @typedef {Object} SelectorValidationResult
|
|
267
|
+
* @property {boolean} ok selector validation status
|
|
268
|
+
* @property {string|null} error reason of invalid selector for invalid selector
|
|
269
|
+
* and `null` for valid one
|
|
270
|
+
*/
|
|
271
|
+
|
|
272
|
+
/**
|
|
273
|
+
* Related to the bug — pseudo-class arg with combinators.
|
|
274
|
+
*
|
|
275
|
+
* @see {@link https://github.com/dperini/nwsapi/issues/55}
|
|
276
|
+
*
|
|
277
|
+
* @example
|
|
278
|
+
* '*:not(div > span)'
|
|
279
|
+
*/
|
|
280
|
+
const VALID_PSEUDO_CLASS_COMBINATOR_ARG_REGEXP = /(.+)?:(not|is)\((.+)?(~|>|\+)(.+)?\)(.+)?/;
|
|
281
|
+
|
|
282
|
+
/**
|
|
283
|
+
* Related to the bug — pseudo-class arg with parenthesis in attribute value.
|
|
284
|
+
*
|
|
285
|
+
* @see {@link https://github.com/dperini/nwsapi/issues/71}
|
|
286
|
+
*
|
|
287
|
+
* @example
|
|
288
|
+
* 'div:not([right=")"])'
|
|
289
|
+
* 'body *:not([left="("])'
|
|
290
|
+
*/
|
|
291
|
+
const VALID_PSEUDO_CLASS_PARENTHESIS_ARG_REGEXP = /(.+)?:(not|is)\((.+)?\[.+=("|')(.+)?(\(|\))(.+)?("|')\]\)/;
|
|
292
|
+
|
|
293
|
+
// TODO: remove backupValidate() after the bugs are fixed
|
|
294
|
+
/**
|
|
295
|
+
* Validates `selector` by its matching with specific regular expressions due to nwsapi bugs:
|
|
296
|
+
* @see {@link https://github.com/dperini/nwsapi/issues/55},
|
|
297
|
+
* @see {@link https://github.com/dperini/nwsapi/issues/71}.
|
|
298
|
+
*
|
|
299
|
+
* @param selector Selector to validate.
|
|
300
|
+
* @param originalError Previous validation error for selectors which are non-related to the bugs.
|
|
301
|
+
*
|
|
302
|
+
* @returns {SelectorValidationResult}
|
|
303
|
+
*/
|
|
304
|
+
const backupValidate = (selector, originalError) => {
|
|
305
|
+
const isValidArgBugRelated = VALID_PSEUDO_CLASS_COMBINATOR_ARG_REGEXP.test(selector)
|
|
306
|
+
|| VALID_PSEUDO_CLASS_PARENTHESIS_ARG_REGEXP.test(selector);
|
|
307
|
+
// if selector is not matched by the regexp specific to the bug
|
|
308
|
+
// original validate error should be returned
|
|
309
|
+
if (!isValidArgBugRelated) {
|
|
310
|
+
return { ok: false, error: originalError };
|
|
311
|
+
}
|
|
312
|
+
return { ok: true, error: null };
|
|
313
|
+
};
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* Validates css selector, uses ExtendedCss.validate() for it.
|
|
317
|
+
*
|
|
318
|
+
* @param selectorText
|
|
319
|
+
*
|
|
320
|
+
* @returns {SelectorValidationResult}
|
|
321
|
+
*/
|
|
322
|
+
const validateCssSelector = (selectorText) => {
|
|
323
|
+
// jsdom is crashing when selector is a script
|
|
324
|
+
if (selectorText.indexOf('##script:contains') !== -1
|
|
325
|
+
|| selectorText.indexOf('##script:inject') !== -1) {
|
|
326
|
+
return {
|
|
327
|
+
ok: false,
|
|
328
|
+
error: 'Selector as a script is not supported.',
|
|
329
|
+
};
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
// skip :before and :after selectors
|
|
333
|
+
if (selectorText.match(/[^:\s]([:]{1,2}before(\s|,|$))|[^:\s]([:]{1,2}after(\s|,|$))/ig)) {
|
|
334
|
+
return {
|
|
335
|
+
ok: true,
|
|
336
|
+
error: null,
|
|
337
|
+
};
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
// skip selectors with case-insensitive attribute, for example: div[class^="Abc_123" i]
|
|
341
|
+
if (selectorText.match(/\[[a-z\d-_]+[\^$*]?=['"]?[^'"]+['"]?\si]/g)) {
|
|
342
|
+
return {
|
|
343
|
+
ok: true,
|
|
344
|
+
error: null,
|
|
345
|
+
};
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
let validation = ExtendedCss.validate(selectorText);
|
|
349
|
+
// TODO: remove later when the bug is fixed
|
|
350
|
+
// https://github.com/dperini/nwsapi/issues/55
|
|
351
|
+
// ExtendedCss.validate() should be enough for selector validation
|
|
352
|
+
if (!validation.ok) {
|
|
353
|
+
validation = backupValidate(selectorText, validation.error);
|
|
354
|
+
}
|
|
355
|
+
return validation;
|
|
356
|
+
};
|
|
357
|
+
|
|
358
|
+
/**
|
|
359
|
+
* @typedef {import('@adguard/agtree').AnyRule} AnyRule
|
|
360
|
+
*/
|
|
361
|
+
|
|
362
|
+
const AFFINITY_DIRECTIVE = '!#safari_cb_affinity'; // used as closing directive
|
|
363
|
+
const AFFINITY_DIRECTIVE_OPEN = `${AFFINITY_DIRECTIVE}(`;
|
|
364
|
+
|
|
365
|
+
const NOT_VALIDATE_HINT = 'NOT_VALIDATE';
|
|
366
|
+
const SPACE$1 = ' ';
|
|
367
|
+
|
|
368
|
+
/**
|
|
369
|
+
* Push rule with warning message to excluded
|
|
370
|
+
* @param {array} excluded
|
|
371
|
+
* @param {string} warning
|
|
372
|
+
* @param {string} rule
|
|
373
|
+
*/
|
|
374
|
+
const excludeRule = (excluded, warning, rule) => {
|
|
375
|
+
if (excluded) {
|
|
376
|
+
excluded.push(warning);
|
|
377
|
+
excluded.push(rule);
|
|
378
|
+
}
|
|
379
|
+
};
|
|
380
|
+
|
|
381
|
+
/**
|
|
382
|
+
* @typedef {object} ValidationResult
|
|
383
|
+
* @property {boolean} valid Whether the rule is valid.
|
|
384
|
+
* @property {string|null} error Error message if the rule is invalid.
|
|
385
|
+
*/
|
|
386
|
+
|
|
387
|
+
/**
|
|
388
|
+
* Class to validate filter rules.
|
|
389
|
+
*/
|
|
390
|
+
class RuleValidator {
|
|
391
|
+
/**
|
|
392
|
+
* Creates validation result for rule.
|
|
393
|
+
*
|
|
394
|
+
* @param {boolean} valid Whether the rule is valid.
|
|
395
|
+
* @param {string} [error] Error message if the rule is invalid.
|
|
396
|
+
*/
|
|
397
|
+
static createValidationResult(valid, error) {
|
|
398
|
+
if (error) {
|
|
399
|
+
return { valid, error };
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
return { valid, error: null };
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
/**
|
|
406
|
+
* Validates regexp pattern.
|
|
407
|
+
*
|
|
408
|
+
* @param {string} pattern Regexp pattern to validate.
|
|
409
|
+
* @param {string} ruleText Rule text.
|
|
410
|
+
*
|
|
411
|
+
* @throws {SyntaxError} If the pattern is invalid, otherwise nothing.
|
|
412
|
+
*/
|
|
413
|
+
static validateRegexp(pattern, ruleText) {
|
|
414
|
+
if (!agtree.RegExpUtils.isRegexPattern(pattern)) {
|
|
415
|
+
return;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
try {
|
|
419
|
+
// eslint-disable-next-line no-new
|
|
420
|
+
new RegExp(pattern.slice(1, -1));
|
|
421
|
+
} catch (e) {
|
|
422
|
+
throw new SyntaxError(`Rule has invalid regex pattern: "${ruleText}"`);
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
/**
|
|
427
|
+
* Validates rule node.
|
|
428
|
+
*
|
|
429
|
+
* @param {AnyRule} ruleNode Rule node to validate.
|
|
430
|
+
*
|
|
431
|
+
* @returns {ValidationResult} Validation result.
|
|
432
|
+
*/
|
|
433
|
+
static validate(ruleNode) {
|
|
434
|
+
if (ruleNode.category === agtree.RuleCategory.Invalid) {
|
|
435
|
+
return RuleValidator.createValidationResult(
|
|
436
|
+
false,
|
|
437
|
+
ruleNode.error.message,
|
|
438
|
+
);
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
if (ruleNode.category === agtree.RuleCategory.Empty || ruleNode.category === agtree.RuleCategory.Comment) {
|
|
442
|
+
return RuleValidator.createValidationResult(true);
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
const ruleText = agtree.RuleGenerator.generate(ruleNode);
|
|
446
|
+
|
|
447
|
+
try {
|
|
448
|
+
// Validate cosmetic rules
|
|
449
|
+
if (ruleNode.category === agtree.RuleCategory.Cosmetic) {
|
|
450
|
+
// eslint-disable-next-line no-new
|
|
451
|
+
new tsurlfilter.CosmeticRule(ruleNode, 0);
|
|
452
|
+
return RuleValidator.createValidationResult(true);
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
// Validate network rules
|
|
456
|
+
const rule = new tsurlfilter.NetworkRule(ruleNode, 0);
|
|
457
|
+
RuleValidator.validateRegexp(rule.getPattern(), ruleText);
|
|
458
|
+
} catch (error) {
|
|
459
|
+
// TODO: add getErrorMessage as a helper
|
|
460
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
461
|
+
const errorMessage = `Error: "${message}" in the rule: "${ruleText}"`;
|
|
462
|
+
return RuleValidator.createValidationResult(false, errorMessage);
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
return RuleValidator.createValidationResult(true);
|
|
466
|
+
|
|
467
|
+
// TODO: validate host rules
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
/**
|
|
472
|
+
* Universal function for validating CSS context.
|
|
473
|
+
*
|
|
474
|
+
* @param {string} input - string to be validated.
|
|
475
|
+
* @param {string} contextName - context (e.g., 'selectorList', 'declarationList', 'mediaQueryList').
|
|
476
|
+
* @throws {Error} Throws an error if parsing fails.
|
|
477
|
+
*/
|
|
478
|
+
const validateCssContext = (input, contextName) => {
|
|
479
|
+
ecssTree.parse(input, {
|
|
480
|
+
context: contextName,
|
|
481
|
+
onParseError(error) {
|
|
482
|
+
throw error;
|
|
483
|
+
},
|
|
484
|
+
});
|
|
485
|
+
};
|
|
486
|
+
|
|
487
|
+
/**
|
|
488
|
+
* Removes invalid rules from the list of rules
|
|
489
|
+
* and logs process in the excluded list.
|
|
490
|
+
*
|
|
491
|
+
* @param {string[]} list List of rule texts.
|
|
492
|
+
* @param {string[]} excluded List of messages with validation results.
|
|
493
|
+
* @param {string[]} invalid List of messages with validation errors.
|
|
494
|
+
* @param {string} filterName Name of the filter.
|
|
495
|
+
*
|
|
496
|
+
* @returns {string[]} List of valid rules.
|
|
497
|
+
*/
|
|
498
|
+
// eslint-disable-next-line default-param-last
|
|
499
|
+
const validateAndFilterRules = (list, excluded, invalid = [], filterName) => {
|
|
500
|
+
if (!list) {
|
|
501
|
+
return [];
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
return list.filter((ruleText, index, array) => {
|
|
505
|
+
if (agtree.CommentParser.isCommentRule(ruleText)) {
|
|
506
|
+
return true;
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
const previousRule = index > 0 ? array[index - 1] : null;
|
|
510
|
+
// Skip validation if "ruleText" is preceded by "NOT_VALIDATE" hint
|
|
511
|
+
// https://github.com/AdguardTeam/FiltersCompiler/issues/245
|
|
512
|
+
if (
|
|
513
|
+
previousRule
|
|
514
|
+
&& previousRule.startsWith(RuleMasks.MASK_HINT)
|
|
515
|
+
&& (
|
|
516
|
+
previousRule.includes(`${SPACE$1}${NOT_VALIDATE_HINT}${SPACE$1}`)
|
|
517
|
+
|| previousRule.endsWith(`${SPACE$1}${NOT_VALIDATE_HINT}`)
|
|
518
|
+
)
|
|
519
|
+
|
|
520
|
+
) {
|
|
521
|
+
return true;
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
let convertedRuleNodes;
|
|
525
|
+
try {
|
|
526
|
+
const ruleNode = agtree.RuleParser.parse(ruleText, {
|
|
527
|
+
...parser.defaultParserOptions,
|
|
528
|
+
// tolerant mode is used for rather quick syntax validation
|
|
529
|
+
tolerant: true,
|
|
530
|
+
isLocIncluded: false,
|
|
531
|
+
includeRaws: false,
|
|
532
|
+
parseAbpSpecificRules: false,
|
|
533
|
+
parseUboSpecificRules: false,
|
|
534
|
+
});
|
|
535
|
+
const conversionResult = agtree.RuleConverter.convertToAdg(ruleNode);
|
|
536
|
+
convertedRuleNodes = conversionResult.result;
|
|
537
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
538
|
+
for (const convertedRuleNode of convertedRuleNodes) {
|
|
539
|
+
if (convertedRuleNode.category !== agtree.RuleCategory.Cosmetic) {
|
|
540
|
+
continue;
|
|
541
|
+
}
|
|
542
|
+
switch (convertedRuleNode.type) {
|
|
543
|
+
case agtree.CosmeticRuleType.ElementHidingRule: {
|
|
544
|
+
validateCssContext(convertedRuleNode.body.selectorList.value, 'selectorList');
|
|
545
|
+
break;
|
|
546
|
+
}
|
|
547
|
+
case agtree.CosmeticRuleType.CssInjectionRule: {
|
|
548
|
+
validateCssContext(convertedRuleNode.body.selectorList.value, 'selectorList');
|
|
549
|
+
if (convertedRuleNode.body.remove === true) {
|
|
550
|
+
break;
|
|
551
|
+
}
|
|
552
|
+
validateCssContext(convertedRuleNode.body.declarationList.value, 'declarationList');
|
|
553
|
+
if (convertedRuleNode.body.mediaQueryList && convertedRuleNode.body.mediaQueryList.value) {
|
|
554
|
+
validateCssContext(convertedRuleNode.body.mediaQueryList.value, 'mediaQueryList');
|
|
555
|
+
}
|
|
556
|
+
break;
|
|
557
|
+
}
|
|
558
|
+
default:
|
|
559
|
+
break;
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
} catch (e) {
|
|
563
|
+
logger.error(`Invalid rule in ${filterName}: ${ruleText}`);
|
|
564
|
+
excludeRule(excluded, e.message, ruleText);
|
|
565
|
+
invalid.push(e.message);
|
|
566
|
+
return false;
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
// optional chaining is needed for the length property because convertedRules can be undefined
|
|
570
|
+
// if RuleParser.parse() or RuleConverter.convertToAdg() throws an error
|
|
571
|
+
if (!convertedRuleNodes || convertedRuleNodes.length === 0) {
|
|
572
|
+
return false;
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
for (let i = 0; i < convertedRuleNodes.length; i += 1) {
|
|
576
|
+
const convertedRuleNode = convertedRuleNodes[i];
|
|
577
|
+
|
|
578
|
+
let validationResult = RuleValidator.validate(convertedRuleNode);
|
|
579
|
+
|
|
580
|
+
// TODO: remove this checking when $header is fixed
|
|
581
|
+
// https://github.com/AdguardTeam/AdguardBrowserExtension/issues/2942
|
|
582
|
+
if (
|
|
583
|
+
!validationResult.valid
|
|
584
|
+
&& validationResult.error.includes('$header rules are not compatible with some other modifiers')
|
|
585
|
+
) {
|
|
586
|
+
// $header rules are not compatible with other modifiers ONLY in the tsurlfilter
|
|
587
|
+
// but it is fine for corelibs
|
|
588
|
+
validationResult = { valid: true };
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
if (!validationResult.valid) {
|
|
592
|
+
// log source rule text to the excluded log
|
|
593
|
+
logger.error(`Invalid rule in ${filterName}: ${ruleText}`);
|
|
594
|
+
// save warning as comment
|
|
595
|
+
excludeRule(excluded, `! ${validationResult.error}`, ruleText);
|
|
596
|
+
// ruleText should be already included into the error text
|
|
597
|
+
invalid.push(validationResult.error);
|
|
598
|
+
return false;
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
const rule = tsurlfilter.RuleFactory.createRule(convertedRuleNode);
|
|
602
|
+
|
|
603
|
+
// It is impossible to bundle jsdom into tsurlfilter, so we check if rules are valid in the compiler
|
|
604
|
+
if (rule instanceof tsurlfilter.CosmeticRule && rule.getType() === agtree.CosmeticRuleType.ElementHidingRule) {
|
|
605
|
+
const validationResult = validateCssSelector(rule.getContent());
|
|
606
|
+
if (!validationResult.ok) {
|
|
607
|
+
// TODO: rule selector can be validated by agtree
|
|
608
|
+
logger.error(`Invalid rule selector in ${filterName}: ${ruleText}`);
|
|
609
|
+
// log source rule text to the excluded log
|
|
610
|
+
excludeRule(excluded, `! ${validationResult.error} in rule:`, ruleText);
|
|
611
|
+
invalid.push(`${validationResult.error} in rule: ${ruleText}`);
|
|
612
|
+
return false;
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
return true;
|
|
618
|
+
});
|
|
619
|
+
};
|
|
620
|
+
|
|
621
|
+
/**
|
|
622
|
+
* Validates !#safari_cb_affinity directives
|
|
623
|
+
*
|
|
624
|
+
* @param lines
|
|
625
|
+
*/
|
|
626
|
+
const checkAffinityDirectives = (lines) => {
|
|
627
|
+
if (!(lines && lines.length)) {
|
|
628
|
+
// skip empty filter
|
|
629
|
+
return true;
|
|
630
|
+
}
|
|
631
|
+
const stack = [];
|
|
632
|
+
|
|
633
|
+
for (let i = 0; i < lines.length; i += 1) {
|
|
634
|
+
const line = lines[i];
|
|
635
|
+
if (line.startsWith(AFFINITY_DIRECTIVE_OPEN)) {
|
|
636
|
+
stack.push(line);
|
|
637
|
+
continue;
|
|
638
|
+
}
|
|
639
|
+
if (line === AFFINITY_DIRECTIVE) {
|
|
640
|
+
const pop = stack.pop();
|
|
641
|
+
if (!(pop && pop.startsWith(AFFINITY_DIRECTIVE_OPEN))) {
|
|
642
|
+
return false;
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
return !stack.length;
|
|
648
|
+
};
|
|
649
|
+
|
|
650
|
+
// TODO: consider refactoring this file
|
|
651
|
+
// because agtree provides modern approach to solve such problems
|
|
652
|
+
|
|
653
|
+
// Based on: https://github.com/github/linguist/pull/5968/commits/f7c5c39139945576a5f9ff0b41c990e6b6019232
|
|
654
|
+
// eslint-disable-next-line max-len
|
|
655
|
+
const ADBLOCK_AGENT_PATTERN = /^(?:!|#)?\s*\[(?<AdblockInfo>\s*(?:[Aa]d[Bb]lock(?:\s+[Pp]lus)?|u[Bb]lock(?:\s+[Oo]rigin)?|[Aa]d[Gg]uard)(?:\s+(?:\d\.?)+)?\s*)(?:;\g<AdblockInfo>)*\]\s*$/;
|
|
656
|
+
|
|
657
|
+
/**
|
|
658
|
+
* CSS rules with width and height attributes break SVG rendering
|
|
659
|
+
* https://github.com/AdguardTeam/AdguardBrowserExtension/issues/683
|
|
660
|
+
*
|
|
661
|
+
* @param ruleText Rule text
|
|
662
|
+
*/
|
|
663
|
+
const fixCssRuleAttributesForEdge = function (ruleText) {
|
|
664
|
+
if (ruleText.includes(RuleMasks.MASK_CSS)
|
|
665
|
+
|| ruleText.includes(RuleMasks.MASK_CSS_EXCEPTION)
|
|
666
|
+
|| ruleText.includes(RuleMasks.MASK_ELEMENT_HIDING)
|
|
667
|
+
|| ruleText.includes(RuleMasks.MASK_ELEMENT_HIDING_EXCEPTION)) {
|
|
668
|
+
ruleText = ruleText.replace(/\[width=/gi, '[Width=');
|
|
669
|
+
ruleText = ruleText.replace('/[height=/gi', '[Height=');
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
return ruleText;
|
|
673
|
+
};
|
|
674
|
+
|
|
675
|
+
/**
|
|
676
|
+
* Updates rule text
|
|
677
|
+
*/
|
|
678
|
+
const overrideRule = function (ruleText, platform) {
|
|
679
|
+
if (platform === 'ext_edge') {
|
|
680
|
+
ruleText = fixCssRuleAttributesForEdge(ruleText);
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
return ruleText;
|
|
684
|
+
};
|
|
685
|
+
|
|
686
|
+
/**
|
|
687
|
+
* Modifies header for AdGuard Base filter
|
|
688
|
+
* https://github.com/AdguardTeam/FiltersCompiler/issues/78
|
|
689
|
+
* @param {array} header
|
|
690
|
+
* @param {boolean} optimized
|
|
691
|
+
*/
|
|
692
|
+
const modifyBaseFilterHeader = (header, optimized) => {
|
|
693
|
+
header[0] = `! Title: AdGuard Base filter + EasyList${optimized ? ' (Optimized)' : ''}`;
|
|
694
|
+
};
|
|
695
|
+
|
|
696
|
+
/**
|
|
697
|
+
* Rewrites title and description
|
|
698
|
+
* https://github.com/AdguardTeam/AdguardFilters/issues/5138#issuecomment-328847738
|
|
699
|
+
*/
|
|
700
|
+
const rewriteHeader = function (header) {
|
|
701
|
+
const result = [];
|
|
702
|
+
header.forEach((line) => {
|
|
703
|
+
if (line.startsWith('! Title: ')) {
|
|
704
|
+
line = '! Title: AdGuard Base filter';
|
|
705
|
+
} else if (line.startsWith('! Description: ')) {
|
|
706
|
+
line = '! Description: This filter is necessary for quality ad blocking.';
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
result.push(line);
|
|
710
|
+
});
|
|
711
|
+
|
|
712
|
+
return result;
|
|
713
|
+
};
|
|
714
|
+
|
|
715
|
+
/**
|
|
716
|
+
* Filters easylist block from list of rules
|
|
717
|
+
* https://github.com/AdguardTeam/AdguardFilters/issues/5138#issuecomment-328847738
|
|
718
|
+
*/
|
|
719
|
+
const rewriteRules = function (rules) {
|
|
720
|
+
const filtered = [];
|
|
721
|
+
let flag = -1;
|
|
722
|
+
for (let i = 0; i < rules.length; i += 1) {
|
|
723
|
+
const rule = rules[i];
|
|
724
|
+
|
|
725
|
+
if (flag >= 0 && rule.startsWith('!------------------')) {
|
|
726
|
+
if (flag !== i - 1) {
|
|
727
|
+
// we skip next line after block header
|
|
728
|
+
// looking for the end of easylist block
|
|
729
|
+
flag = -1;
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
continue;
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
if (rule.startsWith('!------------------ EasyList rules')) {
|
|
736
|
+
flag = i;
|
|
737
|
+
continue;
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
if (flag < 0) {
|
|
741
|
+
filtered.push(rule);
|
|
742
|
+
}
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
return filtered;
|
|
746
|
+
};
|
|
747
|
+
|
|
748
|
+
/**
|
|
749
|
+
* Replaces Version: with OriginalVersion: comments in case of some client cannot afford it.
|
|
750
|
+
*
|
|
751
|
+
* @param rules
|
|
752
|
+
*/
|
|
753
|
+
const fixVersionComments = (rules) => {
|
|
754
|
+
return rules.map((x) => {
|
|
755
|
+
if (x.startsWith('! Version:')) {
|
|
756
|
+
return x.replace('! Version:', '! OriginalVersion:');
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
return x;
|
|
760
|
+
});
|
|
761
|
+
};
|
|
762
|
+
|
|
763
|
+
// TODO: use agtree to remove adblock agent strings
|
|
764
|
+
/**
|
|
765
|
+
* Removes adblock agent strings, for example:
|
|
766
|
+
* - [AdBlock],
|
|
767
|
+
* - [Adblock Plus],
|
|
768
|
+
* - [Adblock Plus 2.0],
|
|
769
|
+
* - [AdGuard],
|
|
770
|
+
* - [uBlock] / [uBlock Origin], etc.
|
|
771
|
+
*
|
|
772
|
+
* @param lines
|
|
773
|
+
*/
|
|
774
|
+
const removeAdblockVersion = (lines) => {
|
|
775
|
+
return lines.filter((line) => !line.trim().match(ADBLOCK_AGENT_PATTERN));
|
|
776
|
+
};
|
|
777
|
+
|
|
778
|
+
/**
|
|
779
|
+
* Removes scriptlet rules
|
|
780
|
+
* @param {array} rules
|
|
781
|
+
* @return {array} rules
|
|
782
|
+
*/
|
|
783
|
+
const removeScriptletRules = (rules) => rules.filter((rule) => !rule.script.startsWith(agtree.ADG_SCRIPTLET_MASK));
|
|
784
|
+
|
|
785
|
+
/**
|
|
786
|
+
* Removes `groupDescription` field from `groups`.
|
|
787
|
+
*
|
|
788
|
+
* @param rawGroups
|
|
789
|
+
* @returns Corrected groups
|
|
790
|
+
*/
|
|
791
|
+
const removeGroupDescriptions$1 = function (rawGroups) {
|
|
792
|
+
const groups = [];
|
|
793
|
+
|
|
794
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
795
|
+
for (const g of rawGroups) {
|
|
796
|
+
const copy = { ...g };
|
|
797
|
+
|
|
798
|
+
delete copy.groupDescription;
|
|
799
|
+
|
|
800
|
+
groups.push(copy);
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
return groups;
|
|
804
|
+
};
|
|
805
|
+
|
|
806
|
+
/**
|
|
807
|
+
* Corrects metadata for backward compatibility with old clients on MAC (v1) platform
|
|
808
|
+
* Hides tag fields
|
|
809
|
+
*
|
|
810
|
+
* @param metadata
|
|
811
|
+
* @returns Corrected metadata
|
|
812
|
+
*/
|
|
813
|
+
const rewriteMetadataForOldMacV1 = function (metadata) {
|
|
814
|
+
const result = {
|
|
815
|
+
groups: removeGroupDescriptions$1(metadata.groups.slice(0)),
|
|
816
|
+
filters: [],
|
|
817
|
+
};
|
|
818
|
+
|
|
819
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
820
|
+
for (const f of metadata.filters) {
|
|
821
|
+
const copy = { ...f };
|
|
822
|
+
delete copy.tags;
|
|
823
|
+
delete copy.timeAdded;
|
|
824
|
+
delete copy.trustLevel;
|
|
825
|
+
delete copy.downloadUrl;
|
|
826
|
+
delete copy.deprecated;
|
|
827
|
+
|
|
828
|
+
result.filters.push(copy);
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
return result;
|
|
832
|
+
};
|
|
833
|
+
|
|
834
|
+
/**
|
|
835
|
+
* Corrects metadata for backward compatibility with old clients on MAC_V2 platform —
|
|
836
|
+
* removed `groupDescription` field from `groups`.
|
|
837
|
+
*
|
|
838
|
+
* @param metadata
|
|
839
|
+
* @returns Corrected metadata
|
|
840
|
+
*/
|
|
841
|
+
const rewriteMetadataForOldMacV2 = function (metadata) {
|
|
842
|
+
const result = { ...metadata };
|
|
843
|
+
result.groups = removeGroupDescriptions$1(result.groups.slice(0));
|
|
844
|
+
return result;
|
|
845
|
+
};
|
|
846
|
+
|
|
847
|
+
/* eslint-disable global-require */
|
|
848
|
+
|
|
849
|
+
const require$1 = module$1.createRequire((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('index.cjs', document.baseURI).href)));
|
|
850
|
+
|
|
851
|
+
/**
|
|
852
|
+
* Some sources require proper user-agents and forbid downloading without.
|
|
853
|
+
*/
|
|
854
|
+
const USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko)'
|
|
855
|
+
+ 'Chrome/63.0.3239.132 Mobile Safari/537.36';
|
|
856
|
+
|
|
857
|
+
/**
|
|
858
|
+
* Sync downloads file from url
|
|
859
|
+
*
|
|
860
|
+
* @param url
|
|
861
|
+
* @param {number} [retryNum=0] number of times to retry downloading, defaults to 0
|
|
862
|
+
* @returns {*}
|
|
863
|
+
*/
|
|
864
|
+
const tryDownloadFile = function (url, retryNum = 0) {
|
|
865
|
+
let args = ['--fail', '--silent', '--user-agent', USER_AGENT, '-L', url];
|
|
866
|
+
if (retryNum) {
|
|
867
|
+
args.push('--retry');
|
|
868
|
+
args.push(retryNum);
|
|
869
|
+
}
|
|
870
|
+
const options = { encoding: 'utf8', maxBuffer: Infinity };
|
|
871
|
+
const tlsCheck = process.env.TLS;
|
|
872
|
+
if (tlsCheck === 'insecure') {
|
|
873
|
+
args = ['--insecure'].concat(args);
|
|
874
|
+
}
|
|
875
|
+
return require$1('child_process')
|
|
876
|
+
.execFileSync('curl', args, options);
|
|
877
|
+
};
|
|
878
|
+
|
|
879
|
+
/**
|
|
880
|
+
* Sync downloads file from url with two attempts
|
|
881
|
+
*
|
|
882
|
+
* @param url
|
|
883
|
+
* @returns {*}
|
|
884
|
+
*/
|
|
885
|
+
const downloadFile = (url) => {
|
|
886
|
+
logger.info(`Downloading: ${url}`);
|
|
887
|
+
|
|
888
|
+
// 5 times to retry after first fail attempt:
|
|
889
|
+
// 1 sec for first time, double for every forthcoming attempts
|
|
890
|
+
// so it will take: 1 + 2 + 4 + 8 + 16 = 31 seconds
|
|
891
|
+
// https://curl.se/docs/manpage.html#--retry
|
|
892
|
+
const RETRY_NUM = 5;
|
|
893
|
+
|
|
894
|
+
try {
|
|
895
|
+
return tryDownloadFile(url);
|
|
896
|
+
} catch (e) {
|
|
897
|
+
logger.warn(e);
|
|
898
|
+
logger.warn(`Retry downloading: ${url}`);
|
|
899
|
+
return tryDownloadFile(url, RETRY_NUM);
|
|
900
|
+
}
|
|
901
|
+
};
|
|
902
|
+
|
|
903
|
+
/* eslint-disable global-require */
|
|
904
|
+
|
|
905
|
+
// Here we can access optimizable filters and its optimization percentages
|
|
906
|
+
// eslint-disable-next-line max-len
|
|
907
|
+
const OPTIMIZATION_PERCENT_URL = 'https://chrome.adtidy.org/optimization_config/percent.json?key=4DDBE80A3DA94D819A00523252FB6380';
|
|
908
|
+
// eslint-disable-next-line max-len
|
|
909
|
+
const OPTIMIZATION_STATS_URL = 'https://chrome.adtidy.org/filters/{0}/stats.json?key=4DDBE80A3DA94D819A00523252FB6380';
|
|
910
|
+
|
|
911
|
+
let filtersOptimizationPercent = null;
|
|
912
|
+
|
|
913
|
+
/**
|
|
914
|
+
* Downloads and caches filters optimization percentages configuration
|
|
915
|
+
*/
|
|
916
|
+
const getFiltersOptimizationPercent = () => {
|
|
917
|
+
|
|
918
|
+
if (filtersOptimizationPercent === null) {
|
|
919
|
+
filtersOptimizationPercent = JSON.parse(downloadFile(OPTIMIZATION_PERCENT_URL));
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
if (filtersOptimizationPercent.config.length === 0) {
|
|
923
|
+
// eslint-disable-next-line no-throw-literal
|
|
924
|
+
throw 'Invalid configuration';
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
return filtersOptimizationPercent;
|
|
928
|
+
};
|
|
929
|
+
|
|
930
|
+
/**
|
|
931
|
+
* Downloads filter optimization config for the filter
|
|
932
|
+
*/
|
|
933
|
+
const getFilterOptimizationConfig = (filterId) => {
|
|
934
|
+
|
|
935
|
+
// config: [{filterId: 1, percent: 45}, ...]
|
|
936
|
+
const filterOptimizationPercent = getFiltersOptimizationPercent().config
|
|
937
|
+
.find((config) => config.filterId === filterId);
|
|
938
|
+
|
|
939
|
+
let optimizationConfig = null;
|
|
940
|
+
if (filterOptimizationPercent) {
|
|
941
|
+
optimizationConfig = JSON.parse(downloadFile(OPTIMIZATION_STATS_URL.replace('{0}', filterId)));
|
|
942
|
+
if (!optimizationConfig || !optimizationConfig.groups || optimizationConfig.groups.length === 0) {
|
|
943
|
+
throw new Error(`Unable to retrieve optimization stats for ${filterId}`);
|
|
944
|
+
}
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
return optimizationConfig;
|
|
948
|
+
};
|
|
949
|
+
|
|
950
|
+
/**
|
|
951
|
+
* Checks if rule should be skipped, because optimization is enabled for this filter
|
|
952
|
+
* and hits of this rule is lower than some value
|
|
953
|
+
* @param ruleText Rule text
|
|
954
|
+
* @param optimizationConfig Optimization config for this filter (retrieved with getFilterOptimizationConfig)
|
|
955
|
+
*/
|
|
956
|
+
const skipRuleWithOptimization = (ruleText, optimizationConfig) => {
|
|
957
|
+
if (!optimizationConfig) {
|
|
958
|
+
return false;
|
|
959
|
+
}
|
|
960
|
+
|
|
961
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
962
|
+
for (const group of optimizationConfig.groups) {
|
|
963
|
+
const hits = group.rules[ruleText];
|
|
964
|
+
if (hits !== undefined && hits < group.config.hits) {
|
|
965
|
+
return true;
|
|
966
|
+
}
|
|
967
|
+
}
|
|
968
|
+
|
|
969
|
+
return false;
|
|
970
|
+
};
|
|
971
|
+
|
|
972
|
+
/**
|
|
973
|
+
* @typedef {object} OptimizationConfig
|
|
974
|
+
* @property {number} filterId - Filter identifier
|
|
975
|
+
* @property {number} percent - Expected optimization percent
|
|
976
|
+
* `~= (rules count in optimized filter) / (rules count in original filter) * 100`
|
|
977
|
+
* @property {number} minPercent - Lower bound of `percent` value
|
|
978
|
+
* @property {number} maxPercent - Upper bound of `percent` value
|
|
979
|
+
* @property {boolean} strict - If `percent < minPercent || percent > maxPercent`
|
|
980
|
+
* and strict mode is on then filter compilation should fail, otherwise original rules must be used
|
|
981
|
+
*/
|
|
982
|
+
|
|
983
|
+
|
|
984
|
+
const HINT_MASK = `${RuleMasks.MASK_HINT} `;
|
|
985
|
+
const COMMENT_REGEXP = '^\\!($|[^#])';
|
|
986
|
+
|
|
987
|
+
const PLATFORM_HINT_REGEXP = /(^| )PLATFORM\(([^)]+)\)/g;
|
|
988
|
+
const NOT_PLATFORM_HINT_REGEXP = /(.*)NOT_PLATFORM\(([^)]+)\)/g;
|
|
989
|
+
|
|
990
|
+
const NOT_OPTIMIZED_HINT$1 = 'NOT_OPTIMIZED';
|
|
991
|
+
|
|
992
|
+
/**
|
|
993
|
+
* Parses rule hints
|
|
994
|
+
*
|
|
995
|
+
* @param rules rules
|
|
996
|
+
* @param platform Platform
|
|
997
|
+
*/
|
|
998
|
+
const splitRuleHintLines = function (rules, platform) {
|
|
999
|
+
const result = [];
|
|
1000
|
+
if (rules) {
|
|
1001
|
+
for (let i = 0; i < rules.length; i += 1) {
|
|
1002
|
+
let rule = rules[i].trim();
|
|
1003
|
+
if (rule.startsWith(HINT_MASK)) {
|
|
1004
|
+
continue;
|
|
1005
|
+
}
|
|
1006
|
+
|
|
1007
|
+
rule = overrideRule(rule, platform);
|
|
1008
|
+
|
|
1009
|
+
const hint = i > 0 ? rules[i - 1] : null;
|
|
1010
|
+
result.push({
|
|
1011
|
+
rule,
|
|
1012
|
+
hint: (hint && hint.startsWith(HINT_MASK)) ? hint : null,
|
|
1013
|
+
});
|
|
1014
|
+
}
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
return result;
|
|
1018
|
+
};
|
|
1019
|
+
|
|
1020
|
+
/**
|
|
1021
|
+
* Joins hint rules
|
|
1022
|
+
*
|
|
1023
|
+
* @param hintLines
|
|
1024
|
+
* @returns {Array}
|
|
1025
|
+
*/
|
|
1026
|
+
const joinRuleHintLines = function (hintLines) {
|
|
1027
|
+
const result = [];
|
|
1028
|
+
hintLines.forEach((f) => {
|
|
1029
|
+
if (f.hint) {
|
|
1030
|
+
result.push(f.hint);
|
|
1031
|
+
}
|
|
1032
|
+
result.push(f.rule);
|
|
1033
|
+
});
|
|
1034
|
+
|
|
1035
|
+
return result;
|
|
1036
|
+
};
|
|
1037
|
+
|
|
1038
|
+
/**
|
|
1039
|
+
* Parses platforms by pattern
|
|
1040
|
+
*
|
|
1041
|
+
*
|
|
1042
|
+
* @param hint stripped hint
|
|
1043
|
+
* @param pattern regexp
|
|
1044
|
+
*/
|
|
1045
|
+
const parsePlatforms = function (hint, pattern) {
|
|
1046
|
+
const result = [];
|
|
1047
|
+
|
|
1048
|
+
if (!hint) {
|
|
1049
|
+
return result;
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
let match = pattern.exec(hint);
|
|
1053
|
+
while (match !== null) {
|
|
1054
|
+
const group = match[2];
|
|
1055
|
+
const split = group.split(',');
|
|
1056
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
1057
|
+
for (let s of split) {
|
|
1058
|
+
s = s.trim();
|
|
1059
|
+
result.push(s);
|
|
1060
|
+
}
|
|
1061
|
+
|
|
1062
|
+
match = pattern.exec(hint);
|
|
1063
|
+
}
|
|
1064
|
+
|
|
1065
|
+
return result;
|
|
1066
|
+
};
|
|
1067
|
+
|
|
1068
|
+
/**
|
|
1069
|
+
* Is rule supported with platform hint
|
|
1070
|
+
*
|
|
1071
|
+
* @param rule
|
|
1072
|
+
* @param platform
|
|
1073
|
+
*/
|
|
1074
|
+
const isPlatformSupported = function (rule, platform) {
|
|
1075
|
+
const { hint } = rule;
|
|
1076
|
+
if (!hint || !hint.startsWith(HINT_MASK)) {
|
|
1077
|
+
return true;
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
if (!platform) {
|
|
1081
|
+
return true;
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
const stripped = hint.substring(HINT_MASK.length).trim();
|
|
1085
|
+
|
|
1086
|
+
const supportedPlatforms = parsePlatforms(stripped, PLATFORM_HINT_REGEXP);
|
|
1087
|
+
const unsupportedPlatforms = parsePlatforms(stripped, NOT_PLATFORM_HINT_REGEXP);
|
|
1088
|
+
|
|
1089
|
+
const supported = supportedPlatforms.length === 0 || supportedPlatforms.indexOf(platform) >= 0;
|
|
1090
|
+
const unsupported = unsupportedPlatforms.length > 0 && unsupportedPlatforms.indexOf(platform) >= 0;
|
|
1091
|
+
|
|
1092
|
+
return supported && !unsupported;
|
|
1093
|
+
};
|
|
1094
|
+
|
|
1095
|
+
/**
|
|
1096
|
+
* Checks if rule supports optimization
|
|
1097
|
+
*
|
|
1098
|
+
* @param rule
|
|
1099
|
+
* @returns {boolean}
|
|
1100
|
+
*/
|
|
1101
|
+
const isOptimizationSupported = function (rule) {
|
|
1102
|
+
const { hint } = rule;
|
|
1103
|
+
if (!hint || !hint.startsWith(HINT_MASK)) {
|
|
1104
|
+
return true;
|
|
1105
|
+
}
|
|
1106
|
+
|
|
1107
|
+
const stripped = hint.substring(HINT_MASK.length).trim();
|
|
1108
|
+
|
|
1109
|
+
return !stripped.includes(NOT_OPTIMIZED_HINT$1);
|
|
1110
|
+
};
|
|
1111
|
+
|
|
1112
|
+
/**
|
|
1113
|
+
* Checks if rule should be omitted with specified configuration
|
|
1114
|
+
*
|
|
1115
|
+
* @param rule
|
|
1116
|
+
* @param config
|
|
1117
|
+
* @param filterId
|
|
1118
|
+
* @returns {boolean}
|
|
1119
|
+
*/
|
|
1120
|
+
const shouldOmitRule = function (rule, config, filterId) {
|
|
1121
|
+
const ruleText = rule.rule;
|
|
1122
|
+
|
|
1123
|
+
if (!ruleText) {
|
|
1124
|
+
return true;
|
|
1125
|
+
}
|
|
1126
|
+
|
|
1127
|
+
// Omit rules by filtration settings
|
|
1128
|
+
if (!config.configuration.ignoreRuleHints && !isPlatformSupported(rule, config.platform)) {
|
|
1129
|
+
logger.info(`${ruleText} removed with platform hint ${rule.hint}`);
|
|
1130
|
+
return true;
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
if (config.configuration.removeRulePatterns) {
|
|
1134
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
1135
|
+
for (const pattern of config.configuration.removeRulePatterns) {
|
|
1136
|
+
if (ruleText.match(new RegExp(pattern))) {
|
|
1137
|
+
// eslint-disable-next-line max-len
|
|
1138
|
+
logger.info(`${ruleText} removed with removeRulePattern ${pattern} in filter ${filterId} for ${config.platform} platform`);
|
|
1139
|
+
return true;
|
|
1140
|
+
}
|
|
1141
|
+
}
|
|
1142
|
+
}
|
|
1143
|
+
|
|
1144
|
+
return false;
|
|
1145
|
+
};
|
|
1146
|
+
|
|
1147
|
+
/**
|
|
1148
|
+
* Checks if rule should be omitted with specified configuration
|
|
1149
|
+
*
|
|
1150
|
+
* @param ruleLine
|
|
1151
|
+
* @param {OptimizationConfig} optimizationConfig
|
|
1152
|
+
*/
|
|
1153
|
+
const shouldOmitRuleWithOptimization = function (ruleLine, optimizationConfig) {
|
|
1154
|
+
const ruleText = ruleLine.rule;
|
|
1155
|
+
|
|
1156
|
+
if (!ruleText) {
|
|
1157
|
+
return true;
|
|
1158
|
+
}
|
|
1159
|
+
|
|
1160
|
+
if (!isOptimizationSupported(ruleLine)) {
|
|
1161
|
+
return false;
|
|
1162
|
+
}
|
|
1163
|
+
|
|
1164
|
+
return skipRuleWithOptimization(ruleText, optimizationConfig);
|
|
1165
|
+
};
|
|
1166
|
+
|
|
1167
|
+
/**
|
|
1168
|
+
* We want to be sure that our mobile optimization is correct and didn't remove valuable rules
|
|
1169
|
+
*
|
|
1170
|
+
* @param filterId
|
|
1171
|
+
* @param rules
|
|
1172
|
+
* @param optimizedRules
|
|
1173
|
+
* @param {OptimizationConfig} optimizationConfig
|
|
1174
|
+
*/
|
|
1175
|
+
const isOptimizationCorrect = function (filterId, rules, optimizedRules, optimizationConfig) {
|
|
1176
|
+
const filterRulesCount = rules.length;
|
|
1177
|
+
const optimizedRulesCount = optimizedRules.length;
|
|
1178
|
+
|
|
1179
|
+
// do not count decimal part of number
|
|
1180
|
+
const resultOptimizationPercent = Math.floor((optimizedRulesCount / filterRulesCount) * 100);
|
|
1181
|
+
|
|
1182
|
+
const expectedOptimizationPercent = optimizationConfig.percent;
|
|
1183
|
+
const minOptimizationPercent = optimizationConfig.minPercent;
|
|
1184
|
+
const maxOptimizationPercent = optimizationConfig.maxPercent;
|
|
1185
|
+
const { strict } = optimizationConfig;
|
|
1186
|
+
|
|
1187
|
+
const tooLow = resultOptimizationPercent < minOptimizationPercent;
|
|
1188
|
+
const tooHigh = resultOptimizationPercent > maxOptimizationPercent;
|
|
1189
|
+
|
|
1190
|
+
const incorrect = tooLow || tooHigh;
|
|
1191
|
+
|
|
1192
|
+
if (incorrect) {
|
|
1193
|
+
const message = `Unable to optimize filter ${filterId} with configuration`
|
|
1194
|
+
+ `[~=${expectedOptimizationPercent}%, min=${minOptimizationPercent}%, max=${maxOptimizationPercent}%],`
|
|
1195
|
+
+ `calculated = ${resultOptimizationPercent.toFixed(2)}%! `
|
|
1196
|
+
+ `Filter rules count: ${filterRulesCount}. Optimized rules count: ${optimizedRulesCount}.`;
|
|
1197
|
+
if (strict) {
|
|
1198
|
+
throw new Error(message);
|
|
1199
|
+
} else {
|
|
1200
|
+
logger.error(message);
|
|
1201
|
+
}
|
|
1202
|
+
}
|
|
1203
|
+
|
|
1204
|
+
logger.info(`Filter ${filterId} optimization: ${filterRulesCount} => ${optimizedRulesCount},`
|
|
1205
|
+
+ `${expectedOptimizationPercent}% => ${resultOptimizationPercent}%.`);
|
|
1206
|
+
|
|
1207
|
+
return !incorrect;
|
|
1208
|
+
};
|
|
1209
|
+
|
|
1210
|
+
/**
|
|
1211
|
+
* Filters set of rules with configuration
|
|
1212
|
+
*
|
|
1213
|
+
* @param rules
|
|
1214
|
+
* @param filterId
|
|
1215
|
+
* @param config
|
|
1216
|
+
*/
|
|
1217
|
+
const cleanupRules = (rules, config, filterId) => {
|
|
1218
|
+
const ruleLines = splitRuleHintLines(rules, config.platform);
|
|
1219
|
+
|
|
1220
|
+
const filtered = ruleLines.filter((r) => !shouldOmitRule(r, config, filterId));
|
|
1221
|
+
|
|
1222
|
+
return joinRuleHintLines(filtered);
|
|
1223
|
+
};
|
|
1224
|
+
|
|
1225
|
+
/**
|
|
1226
|
+
* Filters set of rules with configuration and optimization
|
|
1227
|
+
*
|
|
1228
|
+
* @param rules
|
|
1229
|
+
* @param config
|
|
1230
|
+
* @param {OptimizationConfig} optimizationConfig
|
|
1231
|
+
* @param filterId
|
|
1232
|
+
*/
|
|
1233
|
+
const cleanupAndOptimizeRules = function (rules, config, optimizationConfig, filterId) {
|
|
1234
|
+
config.configuration.removeRulePatterns = config.configuration.removeRulePatterns || [];
|
|
1235
|
+
config.configuration.removeRulePatterns.push(COMMENT_REGEXP);
|
|
1236
|
+
|
|
1237
|
+
const ruleLines = splitRuleHintLines(rules, config.platform);
|
|
1238
|
+
|
|
1239
|
+
const filtered = ruleLines.filter((r) => !shouldOmitRule(r, config, filterId));
|
|
1240
|
+
|
|
1241
|
+
const optimized = filtered.filter((r) => !shouldOmitRuleWithOptimization(r, optimizationConfig));
|
|
1242
|
+
|
|
1243
|
+
let result;
|
|
1244
|
+
// We check that our optimization is correct and didn't remove valuable rules
|
|
1245
|
+
// We do it via comparing expected optimization percent
|
|
1246
|
+
// with real (ratio between optimized rules number and all rules number)
|
|
1247
|
+
if (optimizationConfig && !isOptimizationCorrect(filterId, filtered, optimized, optimizationConfig)) {
|
|
1248
|
+
// Back to default OPTIMIZATION
|
|
1249
|
+
result = joinRuleHintLines(filtered);
|
|
1250
|
+
} else {
|
|
1251
|
+
result = joinRuleHintLines(optimized);
|
|
1252
|
+
}
|
|
1253
|
+
|
|
1254
|
+
config.configuration.removeRulePatterns.pop();
|
|
1255
|
+
return result;
|
|
1256
|
+
};
|
|
1257
|
+
|
|
1258
|
+
/* eslint-disable global-require */
|
|
1259
|
+
|
|
1260
|
+
const __dirname$4 = path.dirname(new URL((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('index.cjs', document.baseURI).href))).pathname);
|
|
1261
|
+
|
|
1262
|
+
const RULES_SEPARATOR = '\r\n';
|
|
1263
|
+
let filterIdsPool = [];
|
|
1264
|
+
const metadataFilterIdsPool = [];
|
|
1265
|
+
|
|
1266
|
+
const OPTIMIZED_PLATFORMS_LIST = ['ext_safari', 'android', 'ios'];
|
|
1267
|
+
|
|
1268
|
+
const PLATFORM_FILTERS_DIR = 'filters';
|
|
1269
|
+
const FILTERS_METADATA_FILE_JSON = 'filters.json';
|
|
1270
|
+
const FILTERS_I18N_METADATA_FILE_JSON = 'filters_i18n.json';
|
|
1271
|
+
|
|
1272
|
+
// AG-20175
|
|
1273
|
+
const FILTERS_METADATA_FILE_JS = 'filters.js';
|
|
1274
|
+
const FILTERS_I18N_METADATA_FILE_JS = 'filters_i18n.js';
|
|
1275
|
+
|
|
1276
|
+
/**
|
|
1277
|
+
* Tag id for obsolete filters.
|
|
1278
|
+
*
|
|
1279
|
+
* @see {@link https://github.com/AdguardTeam/FiltersRegistry/blob/85f2e9f6f5f7c04797017e713dc5986b22a78840/tags/metadata.json#L184}
|
|
1280
|
+
*/
|
|
1281
|
+
const OBSOLETE_TAG_ID = 46;
|
|
1282
|
+
|
|
1283
|
+
/**
|
|
1284
|
+
* From 1 to 99 we have AdGuard filters
|
|
1285
|
+
*
|
|
1286
|
+
* @type {number}
|
|
1287
|
+
*/
|
|
1288
|
+
const LAST_ADGUARD_FILTER_ID = 99;
|
|
1289
|
+
const LOCAL_SCRIPT_RULES_FILE = 'local_script_rules.txt';
|
|
1290
|
+
const LOCAL_SCRIPT_RULES_FILE_JSON = 'local_script_rules.json';
|
|
1291
|
+
|
|
1292
|
+
const LOCAL_SCRIPT_RULES_COMMENT = 'By the rules of AMO and addons.opera.com we cannot use remote scripts'
|
|
1293
|
+
+ '(and our JS injection rules could be counted as remote scripts).\r\n'
|
|
1294
|
+
+ 'So what we do:\r\n'
|
|
1295
|
+
+ '1. We gather all current JS rules in the DEFAULT_SCRIPT_RULES object'
|
|
1296
|
+
+ '(see lib/utils/local-script-rules.js)\r\n'
|
|
1297
|
+
+ '2. We disable JS rules got from remote server\r\n'
|
|
1298
|
+
+ '3. We allow only custom rules got from the User filter (which user creates manually)'
|
|
1299
|
+
+ 'or from this DEFAULT_SCRIPT_RULES object';
|
|
1300
|
+
|
|
1301
|
+
const ONE_HOUR_SEC = 60 * 60;
|
|
1302
|
+
const ONE_DAY_SEC = 24 * ONE_HOUR_SEC;
|
|
1303
|
+
|
|
1304
|
+
/**
|
|
1305
|
+
* Default value of filter expiration time
|
|
1306
|
+
* if impossible to parse a value specified in platforms.json or in filter metadata.
|
|
1307
|
+
*
|
|
1308
|
+
* Defaults to 1 day (or 86400 in seconds).
|
|
1309
|
+
*/
|
|
1310
|
+
const DEFAULT_EXPIRES_SEC = 1 * ONE_DAY_SEC;
|
|
1311
|
+
|
|
1312
|
+
/**
|
|
1313
|
+
* Platforms configurations
|
|
1314
|
+
*/
|
|
1315
|
+
let platformPathsConfig = null;
|
|
1316
|
+
let filterFile = null;
|
|
1317
|
+
let metadataFile = null;
|
|
1318
|
+
let revisionFile = null;
|
|
1319
|
+
let adguardFiltersServerUrl = null;
|
|
1320
|
+
|
|
1321
|
+
/**
|
|
1322
|
+
* Sync reads file content
|
|
1323
|
+
*
|
|
1324
|
+
* @param path
|
|
1325
|
+
* @returns {*}
|
|
1326
|
+
*/
|
|
1327
|
+
const readFile$2 = function (path) {
|
|
1328
|
+
try {
|
|
1329
|
+
return fs.readFileSync(path, { encoding: 'utf-8' });
|
|
1330
|
+
} catch (e) {
|
|
1331
|
+
return null;
|
|
1332
|
+
}
|
|
1333
|
+
};
|
|
1334
|
+
|
|
1335
|
+
/**
|
|
1336
|
+
* Creates header contents
|
|
1337
|
+
*
|
|
1338
|
+
* @param metadataFile
|
|
1339
|
+
* @param revisionFile
|
|
1340
|
+
* @param platformsJsonExpires
|
|
1341
|
+
* @returns {[*,*,*,*,string]}
|
|
1342
|
+
*/
|
|
1343
|
+
const makeHeader = function (metadataFile, revisionFile, platformsJsonExpires) {
|
|
1344
|
+
const metadataString = readFile$2(metadataFile);
|
|
1345
|
+
if (!metadataString) {
|
|
1346
|
+
throw new Error('Error reading metadata');
|
|
1347
|
+
}
|
|
1348
|
+
|
|
1349
|
+
const metadata = JSON.parse(metadataString);
|
|
1350
|
+
|
|
1351
|
+
const revisionString = readFile$2(revisionFile);
|
|
1352
|
+
if (!revisionString) {
|
|
1353
|
+
throw new Error('Error reading revision');
|
|
1354
|
+
}
|
|
1355
|
+
|
|
1356
|
+
const revision = JSON.parse(revisionString);
|
|
1357
|
+
|
|
1358
|
+
const expires = typeof platformsJsonExpires !== 'undefined'
|
|
1359
|
+
? platformsJsonExpires
|
|
1360
|
+
: metadata.expires;
|
|
1361
|
+
|
|
1362
|
+
return [
|
|
1363
|
+
`! Title: ${metadata.name}`,
|
|
1364
|
+
`! Description: ${metadata.description}`,
|
|
1365
|
+
`! Version: ${revision.version}`,
|
|
1366
|
+
`! TimeUpdated: ${moment(revision.timeUpdated).format()}`,
|
|
1367
|
+
`! Expires: ${expires} (update frequency)`,
|
|
1368
|
+
];
|
|
1369
|
+
};
|
|
1370
|
+
|
|
1371
|
+
/**
|
|
1372
|
+
* Strips spec character from end of string
|
|
1373
|
+
*
|
|
1374
|
+
* @param string
|
|
1375
|
+
* @param char
|
|
1376
|
+
*/
|
|
1377
|
+
const stripEnd = function (string, char) {
|
|
1378
|
+
if (string.endsWith(char)) {
|
|
1379
|
+
return stripEnd(string.substring(0, string.length - 1), char);
|
|
1380
|
+
}
|
|
1381
|
+
return string;
|
|
1382
|
+
};
|
|
1383
|
+
|
|
1384
|
+
/**
|
|
1385
|
+
* Checks if filter id is unique
|
|
1386
|
+
*
|
|
1387
|
+
* @param pool
|
|
1388
|
+
* @param filterId
|
|
1389
|
+
*/
|
|
1390
|
+
const checkFilterId = function (pool, filterId) {
|
|
1391
|
+
if (pool.indexOf(filterId) >= 0) {
|
|
1392
|
+
throw new Error(`Invalid filters: Filter identifier is not unique: ${filterId}`);
|
|
1393
|
+
}
|
|
1394
|
+
|
|
1395
|
+
pool.push(filterId);
|
|
1396
|
+
};
|
|
1397
|
+
|
|
1398
|
+
/**
|
|
1399
|
+
* Replaces newlines
|
|
1400
|
+
*
|
|
1401
|
+
* @param message
|
|
1402
|
+
* @returns {XML|string|*}
|
|
1403
|
+
*/
|
|
1404
|
+
const normalizeData = function (message) {
|
|
1405
|
+
message = message.replace(/\r/g, '');
|
|
1406
|
+
message = message.replace(/\n+/g, '\n');
|
|
1407
|
+
return message;
|
|
1408
|
+
};
|
|
1409
|
+
|
|
1410
|
+
/**
|
|
1411
|
+
* Calculates checksum
|
|
1412
|
+
* See:
|
|
1413
|
+
* https://adblockplus.org/en/filters#special-comments
|
|
1414
|
+
* https://hg.adblockplus.org/adblockplus/file/tip/addChecksum.py
|
|
1415
|
+
*
|
|
1416
|
+
* @param header Header lines
|
|
1417
|
+
* @param rules Rules lines
|
|
1418
|
+
*/
|
|
1419
|
+
const calculateChecksum = function (header, rules) {
|
|
1420
|
+
let content = header.concat(rules).join('\n');
|
|
1421
|
+
content = normalizeData(content);
|
|
1422
|
+
const checksum = crypto.createHash('md5').update(content).digest('base64');
|
|
1423
|
+
|
|
1424
|
+
return `! Checksum: ${stripEnd(checksum.trim(), '=')}`;
|
|
1425
|
+
};
|
|
1426
|
+
|
|
1427
|
+
/**
|
|
1428
|
+
* This will create a dir given a path such as './folder/subfolder'
|
|
1429
|
+
* @param {string} dir - The path of the directory to create. Can be absolute or relative.
|
|
1430
|
+
* @returns {string} The path of the last directory created or the existing directory.
|
|
1431
|
+
* @throws {Error} Throws an error if there is a permission issue or if the directory cannot be created.
|
|
1432
|
+
*/
|
|
1433
|
+
const createDir = (dir) => {
|
|
1434
|
+
const { sep } = path;
|
|
1435
|
+
const initDir = path.isAbsolute(dir) ? sep : '';
|
|
1436
|
+
// eslint-disable-next-line no-undef
|
|
1437
|
+
const baseDir = __dirname$4;
|
|
1438
|
+
|
|
1439
|
+
return dir.split(sep).reduce((parentDir, childDir) => {
|
|
1440
|
+
const curDir = path.resolve(baseDir, parentDir, childDir);
|
|
1441
|
+
try {
|
|
1442
|
+
fs.mkdirSync(curDir);
|
|
1443
|
+
} catch (err) {
|
|
1444
|
+
if (err.code === 'EEXIST') { // curDir already exists!
|
|
1445
|
+
return curDir;
|
|
1446
|
+
}
|
|
1447
|
+
|
|
1448
|
+
// To avoid `EISDIR` error on Mac and `EACCES`-->`ENOENT` and `EPERM` on Windows.
|
|
1449
|
+
if (err.code === 'ENOENT') { // Throw the original parentDir error on curDir `ENOENT` failure.
|
|
1450
|
+
throw new Error(`EACCES: permission denied, mkdir '${parentDir}'`);
|
|
1451
|
+
}
|
|
1452
|
+
|
|
1453
|
+
const caughtErr = ['EACCES', 'EPERM', 'EISDIR'].indexOf(err.code) > -1;
|
|
1454
|
+
if ((!caughtErr || caughtErr) && (curDir === path.resolve(dir))) {
|
|
1455
|
+
throw err; // Throw if it's just the last created dir.
|
|
1456
|
+
}
|
|
1457
|
+
}
|
|
1458
|
+
|
|
1459
|
+
return curDir;
|
|
1460
|
+
}, initDir);
|
|
1461
|
+
};
|
|
1462
|
+
|
|
1463
|
+
/**
|
|
1464
|
+
* Replaces tag keywords in the provided filters with their corresponding tag IDs.
|
|
1465
|
+
*
|
|
1466
|
+
* @param {Array<Object>} rawFilters - The array of filter objects to process.
|
|
1467
|
+
* @param {Array<Object>} tags - The array of tag objects.
|
|
1468
|
+
* @returns {Array<Object>} A new array of filter objects with tag keywords replaced by their corresponding tag IDs.
|
|
1469
|
+
* @throws {Error} If any tag keyword in the filters does not have a corresponding tag in the `tags` array.
|
|
1470
|
+
*/
|
|
1471
|
+
const replaceTagKeywords = function (rawFilters, tags) {
|
|
1472
|
+
const tagsMap = new Map();
|
|
1473
|
+
|
|
1474
|
+
tags.forEach((tag) => {
|
|
1475
|
+
tagsMap.set(tag.keyword, tag.tagId);
|
|
1476
|
+
});
|
|
1477
|
+
|
|
1478
|
+
// create new variable to avoid mutation of input parameters
|
|
1479
|
+
const filters = [];
|
|
1480
|
+
const lostTags = [];
|
|
1481
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
1482
|
+
for (const filter of rawFilters) {
|
|
1483
|
+
const newFilter = { ...filter };
|
|
1484
|
+
if (newFilter.tags) {
|
|
1485
|
+
const ids = [];
|
|
1486
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
1487
|
+
for (const t of newFilter.tags) {
|
|
1488
|
+
const id = tagsMap.get(t);
|
|
1489
|
+
if (id) {
|
|
1490
|
+
ids.push(id);
|
|
1491
|
+
} else {
|
|
1492
|
+
logger.error(`Missing tag with keyword: ${t}`);
|
|
1493
|
+
lostTags.push(t);
|
|
1494
|
+
}
|
|
1495
|
+
}
|
|
1496
|
+
|
|
1497
|
+
delete newFilter.tags;
|
|
1498
|
+
newFilter.tags = ids;
|
|
1499
|
+
}
|
|
1500
|
+
filters.push(newFilter);
|
|
1501
|
+
}
|
|
1502
|
+
|
|
1503
|
+
if (lostTags.length > 0) {
|
|
1504
|
+
throw new Error(`Missing tag with keyword: ${lostTags.join(', ')}`);
|
|
1505
|
+
}
|
|
1506
|
+
|
|
1507
|
+
return filters;
|
|
1508
|
+
};
|
|
1509
|
+
|
|
1510
|
+
/**
|
|
1511
|
+
* Converts `rawExpires` with `day` marker into **seconds**.
|
|
1512
|
+
*
|
|
1513
|
+
* @param {any} rawExpires Raw `expires` value from filter metadata or platforms.json.
|
|
1514
|
+
*
|
|
1515
|
+
* @returns {number} Parsed expires value from days to seconds,
|
|
1516
|
+
* or {@link DEFAULT_EXPIRES_SEC} if it cannot be parsed.
|
|
1517
|
+
*/
|
|
1518
|
+
const convertExpiresDaysToSeconds = (rawExpires) => {
|
|
1519
|
+
const expiresDays = parseInt(rawExpires, 10);
|
|
1520
|
+
if (Number.isNaN(expiresDays)) {
|
|
1521
|
+
return DEFAULT_EXPIRES_SEC;
|
|
1522
|
+
}
|
|
1523
|
+
return expiresDays * ONE_DAY_SEC;
|
|
1524
|
+
};
|
|
1525
|
+
|
|
1526
|
+
/**
|
|
1527
|
+
* Converts `rawExpires` with `hour` marker into **seconds**.
|
|
1528
|
+
*
|
|
1529
|
+
* @param {any} rawExpires Raw `expires` value from filter metadata or platforms.json.
|
|
1530
|
+
*
|
|
1531
|
+
* @returns {number} Parsed expires value from hours to seconds,
|
|
1532
|
+
* or {@link DEFAULT_EXPIRES_SEC} if it cannot be parsed.
|
|
1533
|
+
*/
|
|
1534
|
+
const convertExpiresHoursToSeconds = (rawExpires) => {
|
|
1535
|
+
const expiresHours = parseInt(rawExpires, 10);
|
|
1536
|
+
if (Number.isNaN(expiresHours)) {
|
|
1537
|
+
return DEFAULT_EXPIRES_SEC;
|
|
1538
|
+
}
|
|
1539
|
+
return expiresHours * ONE_HOUR_SEC;
|
|
1540
|
+
};
|
|
1541
|
+
|
|
1542
|
+
/**
|
|
1543
|
+
* Overrides filter metadata `expires` property with platforms.json's `expires` property.
|
|
1544
|
+
*
|
|
1545
|
+
* Then parses the value and converts it to **seconds**.
|
|
1546
|
+
* If it cannot be parsed or not set at all, the default value {@link DEFAULT_EXPIRES_SEC} is used.
|
|
1547
|
+
*
|
|
1548
|
+
* @example
|
|
1549
|
+
* `12 hours` → 43200
|
|
1550
|
+
* `1 day` → 86400
|
|
1551
|
+
* `2 days` → 172800
|
|
1552
|
+
*
|
|
1553
|
+
* @param rawFilters Input filters' metadata.
|
|
1554
|
+
* @param platformsJsonExpires Platforms.json's `expires` property to override filters' `expires`.
|
|
1555
|
+
*
|
|
1556
|
+
* @returns {Array<object>} Updated filters' metadata with `expires` property in **seconds**.
|
|
1557
|
+
*/
|
|
1558
|
+
const replaceExpires = function (rawFilters, platformsJsonExpires) {
|
|
1559
|
+
const filters = [];
|
|
1560
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
1561
|
+
for (const filter of rawFilters) {
|
|
1562
|
+
// do not mutate input parameters
|
|
1563
|
+
const newFilter = { ...filter };
|
|
1564
|
+
// expires value which is set in platforms.json has higher priority
|
|
1565
|
+
// https://github.com/AdguardTeam/FiltersCompiler/issues/198
|
|
1566
|
+
if (typeof platformsJsonExpires !== 'undefined') {
|
|
1567
|
+
newFilter.expires = platformsJsonExpires;
|
|
1568
|
+
}
|
|
1569
|
+
|
|
1570
|
+
if (newFilter.expires) {
|
|
1571
|
+
if (newFilter.expires.indexOf('day') > 0) {
|
|
1572
|
+
newFilter.expires = convertExpiresDaysToSeconds(newFilter.expires);
|
|
1573
|
+
} else if (newFilter.expires.indexOf('hour') > 0) {
|
|
1574
|
+
newFilter.expires = convertExpiresHoursToSeconds(newFilter.expires);
|
|
1575
|
+
}
|
|
1576
|
+
} else {
|
|
1577
|
+
// use default value if 'expires' is not set either in platforms.json or in filter metadata
|
|
1578
|
+
newFilter.expires = DEFAULT_EXPIRES_SEC;
|
|
1579
|
+
}
|
|
1580
|
+
filters.push(newFilter);
|
|
1581
|
+
}
|
|
1582
|
+
|
|
1583
|
+
return filters;
|
|
1584
|
+
};
|
|
1585
|
+
|
|
1586
|
+
/**
|
|
1587
|
+
* First step of processing filters metadata
|
|
1588
|
+
*
|
|
1589
|
+
* @param filtersMetadata
|
|
1590
|
+
*/
|
|
1591
|
+
const processFiltersFromMetadata = function (filtersMetadata) {
|
|
1592
|
+
// do not mutate input parameters
|
|
1593
|
+
const filters = [];
|
|
1594
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
1595
|
+
for (const filter of filtersMetadata) {
|
|
1596
|
+
const newFilter = {
|
|
1597
|
+
...filter,
|
|
1598
|
+
deprecated: Boolean(filter.deprecated),
|
|
1599
|
+
};
|
|
1600
|
+
|
|
1601
|
+
/**
|
|
1602
|
+
* In case of backward compatibility
|
|
1603
|
+
* Adds 'languages' metadata field parsed from 'lang:' tags
|
|
1604
|
+
*/
|
|
1605
|
+
if (newFilter.tags) {
|
|
1606
|
+
const filterLanguages = [];
|
|
1607
|
+
let hasRecommended = false;
|
|
1608
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
1609
|
+
for (const t of newFilter.tags) {
|
|
1610
|
+
if (!hasRecommended && t === 'recommended') {
|
|
1611
|
+
hasRecommended = true;
|
|
1612
|
+
}
|
|
1613
|
+
|
|
1614
|
+
if (t.startsWith('lang:')) {
|
|
1615
|
+
filterLanguages.push(t.substring(5));
|
|
1616
|
+
}
|
|
1617
|
+
}
|
|
1618
|
+
|
|
1619
|
+
// Languages will be added for recommended filters only
|
|
1620
|
+
newFilter.languages = hasRecommended ? filterLanguages : [];
|
|
1621
|
+
}
|
|
1622
|
+
|
|
1623
|
+
filters.push(newFilter);
|
|
1624
|
+
}
|
|
1625
|
+
|
|
1626
|
+
return filters;
|
|
1627
|
+
};
|
|
1628
|
+
|
|
1629
|
+
/**
|
|
1630
|
+
* Does several things with filters URLs:
|
|
1631
|
+
* 1. Rewrites subscription urls for specified platform config
|
|
1632
|
+
* 2. Adds downloadUrl field to the filter
|
|
1633
|
+
*
|
|
1634
|
+
* @param metadata
|
|
1635
|
+
* @param config
|
|
1636
|
+
*/
|
|
1637
|
+
const postProcessUrls = (metadata, config) => {
|
|
1638
|
+
const useOptimized = OPTIMIZED_PLATFORMS_LIST.indexOf(config.platform) >= 0;
|
|
1639
|
+
|
|
1640
|
+
const result = {};
|
|
1641
|
+
|
|
1642
|
+
result.groups = metadata.groups.slice(0);
|
|
1643
|
+
result.tags = metadata.tags.slice(0);
|
|
1644
|
+
result.filters = [];
|
|
1645
|
+
|
|
1646
|
+
const platformPath = config.path;
|
|
1647
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
1648
|
+
for (const f of metadata.filters) {
|
|
1649
|
+
const fileName = `${f.filterId}${useOptimized ? '_optimized' : ''}.txt`;
|
|
1650
|
+
const downloadUrl = `${adguardFiltersServerUrl}${platformPath}/filters/${fileName}`;
|
|
1651
|
+
|
|
1652
|
+
const copy = { ...f, downloadUrl };
|
|
1653
|
+
|
|
1654
|
+
if (copy.subscriptionUrl && copy.subscriptionUrl.startsWith(adguardFiltersServerUrl)) {
|
|
1655
|
+
copy.subscriptionUrl = downloadUrl;
|
|
1656
|
+
}
|
|
1657
|
+
|
|
1658
|
+
result.filters.push(copy);
|
|
1659
|
+
}
|
|
1660
|
+
|
|
1661
|
+
return result;
|
|
1662
|
+
};
|
|
1663
|
+
|
|
1664
|
+
/**
|
|
1665
|
+
* Removes redundant metadata for included or excluded filters for the current platform
|
|
1666
|
+
*
|
|
1667
|
+
* @param metadata
|
|
1668
|
+
* @param platform
|
|
1669
|
+
*
|
|
1670
|
+
* @returns {object} Metadata with filtered `filters` array due to the `platform`.
|
|
1671
|
+
*/
|
|
1672
|
+
const removeRedundantFiltersMetadata = (metadata, platform) => {
|
|
1673
|
+
// leaves only included filters metadata
|
|
1674
|
+
metadata.filters = metadata.filters.filter((filter) => !filter.platformsIncluded
|
|
1675
|
+
|| (filter.platformsIncluded && filter.platformsIncluded.includes(platform)));
|
|
1676
|
+
// removes excluded filters metadata
|
|
1677
|
+
metadata.filters = metadata.filters.filter((filter) => !filter.platformsExcluded
|
|
1678
|
+
|| (filter.platformsExcluded && !filter.platformsExcluded.includes(platform)));
|
|
1679
|
+
|
|
1680
|
+
return metadata;
|
|
1681
|
+
};
|
|
1682
|
+
|
|
1683
|
+
/**
|
|
1684
|
+
* Parses object info
|
|
1685
|
+
* Splits string {mask}{id}.{message} like "group.1.name" etc.
|
|
1686
|
+
*
|
|
1687
|
+
* @param string
|
|
1688
|
+
* @param mask
|
|
1689
|
+
* @returns {{id: *, message: *}}
|
|
1690
|
+
*/
|
|
1691
|
+
const parseInfo = (string, mask) => {
|
|
1692
|
+
const searchIndex = string.indexOf(mask) + mask.length;
|
|
1693
|
+
|
|
1694
|
+
return {
|
|
1695
|
+
id: string.substring(searchIndex, string.indexOf('.', searchIndex)),
|
|
1696
|
+
message: string.substring(string.lastIndexOf('.') + 1),
|
|
1697
|
+
};
|
|
1698
|
+
};
|
|
1699
|
+
|
|
1700
|
+
/**
|
|
1701
|
+
* Loads locale data from a specified directory and organizes it into groups, tags, and filters.
|
|
1702
|
+
*
|
|
1703
|
+
* @param {string} dir - The directory containing locale subdirectories with JSON files.
|
|
1704
|
+
* @returns {Object} An object containing the loaded locale data.
|
|
1705
|
+
*/
|
|
1706
|
+
const loadLocales = function (dir) {
|
|
1707
|
+
const result = {
|
|
1708
|
+
groups: {},
|
|
1709
|
+
tags: {},
|
|
1710
|
+
filters: {},
|
|
1711
|
+
};
|
|
1712
|
+
|
|
1713
|
+
const locales = fs.readdirSync(dir);
|
|
1714
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
1715
|
+
for (const directory of locales) {
|
|
1716
|
+
const localeDir = path.join(dir, directory);
|
|
1717
|
+
if (fs.lstatSync(localeDir).isDirectory()) {
|
|
1718
|
+
const groups = JSON.parse(readFile$2(path.join(localeDir, 'groups.json')));
|
|
1719
|
+
if (groups) {
|
|
1720
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
1721
|
+
for (const group of groups) {
|
|
1722
|
+
// eslint-disable-next-line guard-for-in,no-restricted-syntax
|
|
1723
|
+
for (const p in group) {
|
|
1724
|
+
const info = parseInfo(p, 'group.');
|
|
1725
|
+
if (!info || !info.id) {
|
|
1726
|
+
continue;
|
|
1727
|
+
}
|
|
1728
|
+
|
|
1729
|
+
const { id } = info;
|
|
1730
|
+
result.groups[id] = result.groups[id] || {};
|
|
1731
|
+
result.groups[id][directory] = result.groups[id][directory] || {};
|
|
1732
|
+
result.groups[id][directory][info.message] = group[p];
|
|
1733
|
+
}
|
|
1734
|
+
}
|
|
1735
|
+
}
|
|
1736
|
+
|
|
1737
|
+
const tags = JSON.parse(readFile$2(path.join(localeDir, 'tags.json')));
|
|
1738
|
+
if (tags) {
|
|
1739
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
1740
|
+
for (const tag of tags) {
|
|
1741
|
+
// eslint-disable-next-line guard-for-in,no-restricted-syntax
|
|
1742
|
+
for (const p in tag) {
|
|
1743
|
+
const info = parseInfo(p, 'tag.');
|
|
1744
|
+
if (!info || !info.id) {
|
|
1745
|
+
continue;
|
|
1746
|
+
}
|
|
1747
|
+
|
|
1748
|
+
const { id } = info;
|
|
1749
|
+
result.tags[id] = result.tags[id] || {};
|
|
1750
|
+
result.tags[id][directory] = result.tags[id][directory] || {};
|
|
1751
|
+
result.tags[id][directory][info.message] = tag[p];
|
|
1752
|
+
}
|
|
1753
|
+
}
|
|
1754
|
+
}
|
|
1755
|
+
|
|
1756
|
+
const filters = JSON.parse(readFile$2(path.join(localeDir, 'filters.json')));
|
|
1757
|
+
if (filters) {
|
|
1758
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
1759
|
+
for (const filter of filters) {
|
|
1760
|
+
// eslint-disable-next-line guard-for-in,no-restricted-syntax
|
|
1761
|
+
for (const p in filter) {
|
|
1762
|
+
const info = parseInfo(p, 'filter.');
|
|
1763
|
+
if (!info || !info.id) {
|
|
1764
|
+
continue;
|
|
1765
|
+
}
|
|
1766
|
+
|
|
1767
|
+
const { id } = info;
|
|
1768
|
+
result.filters[id] = result.filters[id] || {};
|
|
1769
|
+
result.filters[id][directory] = result.filters[id][directory] || {};
|
|
1770
|
+
result.filters[id][directory][info.message] = filter[p];
|
|
1771
|
+
}
|
|
1772
|
+
}
|
|
1773
|
+
}
|
|
1774
|
+
}
|
|
1775
|
+
}
|
|
1776
|
+
|
|
1777
|
+
return result;
|
|
1778
|
+
};
|
|
1779
|
+
|
|
1780
|
+
/**
|
|
1781
|
+
* Excludes obsolete filters data from localizations
|
|
1782
|
+
* @param {object} localizationsFilters
|
|
1783
|
+
* @param {array} obsoleteFilters
|
|
1784
|
+
* @return {object} result
|
|
1785
|
+
*/
|
|
1786
|
+
const excludeObsoleteFilters = (localizationsFilters, obsoleteFilters) => {
|
|
1787
|
+
const result = { ...localizationsFilters };
|
|
1788
|
+
obsoleteFilters.forEach((filter) => {
|
|
1789
|
+
delete result[filter.filterId];
|
|
1790
|
+
});
|
|
1791
|
+
return result;
|
|
1792
|
+
};
|
|
1793
|
+
|
|
1794
|
+
/**
|
|
1795
|
+
* Removes obsolete filters metadata
|
|
1796
|
+
* @param {object} metadata
|
|
1797
|
+
* @return {object} result
|
|
1798
|
+
*/
|
|
1799
|
+
const removeObsoleteFilters = (metadata) => {
|
|
1800
|
+
const result = { ...metadata };
|
|
1801
|
+
result.filters = metadata.filters.filter((filter) => !filter.tags.includes(OBSOLETE_TAG_ID));
|
|
1802
|
+
return result;
|
|
1803
|
+
};
|
|
1804
|
+
|
|
1805
|
+
/**
|
|
1806
|
+
* Sorts metadata's filters by `filterId` property.
|
|
1807
|
+
*
|
|
1808
|
+
* @param {object} metadata Metadata to sort filters for.
|
|
1809
|
+
* @returns {object} Metadata with sorted filters.
|
|
1810
|
+
*/
|
|
1811
|
+
const sortMetadataFilters = (metadata) => {
|
|
1812
|
+
const result = { ...metadata };
|
|
1813
|
+
result.filters.sort((a, b) => a.filterId - b.filterId);
|
|
1814
|
+
return result;
|
|
1815
|
+
};
|
|
1816
|
+
|
|
1817
|
+
/**
|
|
1818
|
+
* Checks whether the filter should be built for the specified platform.
|
|
1819
|
+
*
|
|
1820
|
+
* @param {object} metadata Filter metadata.
|
|
1821
|
+
* @param {string} platform Platform.
|
|
1822
|
+
*
|
|
1823
|
+
* @returns True if
|
|
1824
|
+
* - both `platformsIncluded` and `platformsExcluded` properties are not defined in the `metadata`,
|
|
1825
|
+
* - `platformsExcluded` does not contain the specified `platform`,
|
|
1826
|
+
* - `platformsIncluded` contains the specified `platform`.
|
|
1827
|
+
*
|
|
1828
|
+
* @throws An error if both `platformsIncluded` and `platformsExcluded` are defined.
|
|
1829
|
+
*/
|
|
1830
|
+
const shouldBuildFilterForPlatform = (metadata, platform) => {
|
|
1831
|
+
const { filterId, platformsExcluded, platformsIncluded } = metadata;
|
|
1832
|
+
|
|
1833
|
+
if (platformsExcluded && platformsIncluded) {
|
|
1834
|
+
let errorMessage = 'Both platformsIncluded and platformsExcluded cannot be defined simultaneously';
|
|
1835
|
+
if (filterId) {
|
|
1836
|
+
errorMessage += ` for filter ${filterId}`;
|
|
1837
|
+
}
|
|
1838
|
+
throw new Error(errorMessage);
|
|
1839
|
+
}
|
|
1840
|
+
|
|
1841
|
+
if (platformsExcluded && platformsExcluded.includes(platform)) {
|
|
1842
|
+
return false;
|
|
1843
|
+
}
|
|
1844
|
+
|
|
1845
|
+
if (!platformsIncluded) {
|
|
1846
|
+
return true;
|
|
1847
|
+
}
|
|
1848
|
+
|
|
1849
|
+
return platformsIncluded.includes(platform);
|
|
1850
|
+
};
|
|
1851
|
+
|
|
1852
|
+
/**
|
|
1853
|
+
* Removes group descriptions from the input groups.
|
|
1854
|
+
*
|
|
1855
|
+
* @param {Object} inputGroups Input groups metadata.
|
|
1856
|
+
* @returns {Object} Output groups metadata.
|
|
1857
|
+
*/
|
|
1858
|
+
const removeGroupDescriptions = (inputGroups) => {
|
|
1859
|
+
const result = {};
|
|
1860
|
+
Object.keys(inputGroups).forEach((groupId) => {
|
|
1861
|
+
result[groupId] = {};
|
|
1862
|
+
|
|
1863
|
+
Object.keys(inputGroups[groupId]).forEach((locale) => {
|
|
1864
|
+
const localeData = inputGroups[groupId][locale];
|
|
1865
|
+
const cleanedLocaleData = { ...localeData };
|
|
1866
|
+
|
|
1867
|
+
if (cleanedLocaleData.description) {
|
|
1868
|
+
delete cleanedLocaleData.description;
|
|
1869
|
+
}
|
|
1870
|
+
|
|
1871
|
+
result[groupId][locale] = cleanedLocaleData;
|
|
1872
|
+
});
|
|
1873
|
+
});
|
|
1874
|
+
|
|
1875
|
+
return result;
|
|
1876
|
+
};
|
|
1877
|
+
|
|
1878
|
+
/**
|
|
1879
|
+
* Writes filters metadata and localizations for different platforms.
|
|
1880
|
+
*
|
|
1881
|
+
* @param {string} platformsPath - The base path where platform-specific directories are located.
|
|
1882
|
+
* @param {string} filtersDir - The directory containing filters metadata and related files.
|
|
1883
|
+
* @param {Array<Object>} filtersMetadata - The metadata for filters to be processed.
|
|
1884
|
+
* @param {Array<string>} obsoleteFilters - A list of obsolete filters to be excluded.
|
|
1885
|
+
*
|
|
1886
|
+
* @returns {void}
|
|
1887
|
+
* @throws {Error} If reading or parsing metadata files fails.
|
|
1888
|
+
*/
|
|
1889
|
+
const writeFiltersMetadata = function (platformsPath, filtersDir, filtersMetadata, obsoleteFilters) {
|
|
1890
|
+
logger.info('Writing filters metadata');
|
|
1891
|
+
|
|
1892
|
+
const groups = JSON.parse(readFile$2(path.join(filtersDir, '../groups', 'metadata.json')));
|
|
1893
|
+
if (!groups) {
|
|
1894
|
+
logger.error('Error reading groups metadata');
|
|
1895
|
+
return;
|
|
1896
|
+
}
|
|
1897
|
+
|
|
1898
|
+
const tags = JSON.parse(readFile$2(path.join(filtersDir, '../tags', 'metadata.json')));
|
|
1899
|
+
if (!tags) {
|
|
1900
|
+
logger.error('Error reading tags metadata');
|
|
1901
|
+
return;
|
|
1902
|
+
}
|
|
1903
|
+
|
|
1904
|
+
// do not mutate input parameters
|
|
1905
|
+
const parsedLangTagsFiltersMetadata = processFiltersFromMetadata(filtersMetadata);
|
|
1906
|
+
const replacedTagKeywordsFiltersMetadata = replaceTagKeywords(parsedLangTagsFiltersMetadata, tags);
|
|
1907
|
+
|
|
1908
|
+
const localizations = loadLocales(path.join(filtersDir, '../locales'));
|
|
1909
|
+
|
|
1910
|
+
// eslint-disable-next-line guard-for-in,no-restricted-syntax
|
|
1911
|
+
for (const platform in platformPathsConfig) {
|
|
1912
|
+
const config = platformPathsConfig[platform];
|
|
1913
|
+
const platformDir = path.join(platformsPath, config.path);
|
|
1914
|
+
createDir(platformDir);
|
|
1915
|
+
|
|
1916
|
+
logger.info(`Writing filters metadata: ${config.path}`);
|
|
1917
|
+
const filtersFileJson = path.join(platformDir, FILTERS_METADATA_FILE_JSON);
|
|
1918
|
+
const filtersFileJs = path.join(platformDir, FILTERS_METADATA_FILE_JS);
|
|
1919
|
+
|
|
1920
|
+
const replacedExpiresFiltersMetadata = replaceExpires(replacedTagKeywordsFiltersMetadata, config.expires);
|
|
1921
|
+
|
|
1922
|
+
let metadata = {
|
|
1923
|
+
groups,
|
|
1924
|
+
tags,
|
|
1925
|
+
filters: replacedExpiresFiltersMetadata,
|
|
1926
|
+
};
|
|
1927
|
+
|
|
1928
|
+
metadata = postProcessUrls(metadata, config);
|
|
1929
|
+
metadata = removeRedundantFiltersMetadata(metadata, config.platform);
|
|
1930
|
+
|
|
1931
|
+
if (platform === 'MAC') {
|
|
1932
|
+
metadata = rewriteMetadataForOldMacV1(metadata);
|
|
1933
|
+
} else if (platform === 'MAC_V2') {
|
|
1934
|
+
metadata = rewriteMetadataForOldMacV2(metadata);
|
|
1935
|
+
metadata = removeObsoleteFilters(metadata);
|
|
1936
|
+
} else {
|
|
1937
|
+
metadata = removeObsoleteFilters(metadata);
|
|
1938
|
+
}
|
|
1939
|
+
|
|
1940
|
+
const filtersContent = JSON.stringify(sortMetadataFilters(metadata), null, '\t');
|
|
1941
|
+
|
|
1942
|
+
fs.writeFileSync(filtersFileJson, filtersContent, 'utf8');
|
|
1943
|
+
fs.writeFileSync(filtersFileJs, filtersContent, 'utf8');
|
|
1944
|
+
|
|
1945
|
+
logger.info(`Writing filters localizations: ${config.path}`);
|
|
1946
|
+
const filtersI18nFileJson = path.join(platformDir, FILTERS_I18N_METADATA_FILE_JSON);
|
|
1947
|
+
const filtersI18nFileJs = path.join(platformDir, FILTERS_I18N_METADATA_FILE_JS);
|
|
1948
|
+
|
|
1949
|
+
let localizedFilters = { ...localizations.filters };
|
|
1950
|
+
|
|
1951
|
+
filtersMetadata.forEach((metadata) => {
|
|
1952
|
+
if (!shouldBuildFilterForPlatform(metadata, config.platform)) {
|
|
1953
|
+
const { filterId } = metadata;
|
|
1954
|
+
// eslint-disable-next-line max-len
|
|
1955
|
+
logger.info(`Adding localization for filter ${filterId} skipped for platform '${config.platform}' due to platformsExcluded or platformsIncluded`);
|
|
1956
|
+
delete localizedFilters[filterId];
|
|
1957
|
+
}
|
|
1958
|
+
});
|
|
1959
|
+
|
|
1960
|
+
// old MAC platform may not support absence some filters i18n metadata,
|
|
1961
|
+
// for all other platforms we can exclude obsolete filters
|
|
1962
|
+
if (platform !== 'MAC') {
|
|
1963
|
+
localizedFilters = excludeObsoleteFilters(localizedFilters, obsoleteFilters);
|
|
1964
|
+
}
|
|
1965
|
+
|
|
1966
|
+
const i18nMetadata = {
|
|
1967
|
+
groups: localizations.groups,
|
|
1968
|
+
tags: localizations.tags,
|
|
1969
|
+
filters: localizedFilters,
|
|
1970
|
+
};
|
|
1971
|
+
|
|
1972
|
+
let i18nGroups = localizations.groups;
|
|
1973
|
+
|
|
1974
|
+
// no new fields should be added for old 'MAC' platform
|
|
1975
|
+
if (platform === 'MAC') {
|
|
1976
|
+
delete i18nMetadata.tags;
|
|
1977
|
+
i18nGroups = removeGroupDescriptions(localizations.groups);
|
|
1978
|
+
}
|
|
1979
|
+
|
|
1980
|
+
i18nMetadata.groups = i18nGroups;
|
|
1981
|
+
|
|
1982
|
+
const i18nContent = JSON.stringify(i18nMetadata, null, '\t');
|
|
1983
|
+
|
|
1984
|
+
fs.writeFileSync(filtersI18nFileJson, i18nContent, 'utf8');
|
|
1985
|
+
fs.writeFileSync(filtersI18nFileJs, i18nContent, 'utf8');
|
|
1986
|
+
}
|
|
1987
|
+
|
|
1988
|
+
logger.info('Writing filters metadata done');
|
|
1989
|
+
};
|
|
1990
|
+
|
|
1991
|
+
/**
|
|
1992
|
+
* Separates script rules from AG filters into specified file.
|
|
1993
|
+
*
|
|
1994
|
+
* @param platformsPath - Path to platforms folder
|
|
1995
|
+
*/
|
|
1996
|
+
const writeLocalScriptRules = function (platformsPath) {
|
|
1997
|
+
logger.info('Writing local script rules');
|
|
1998
|
+
|
|
1999
|
+
// eslint-disable-next-line guard-for-in,no-restricted-syntax
|
|
2000
|
+
for (const platform in platformPathsConfig) {
|
|
2001
|
+
const config = platformPathsConfig[platform];
|
|
2002
|
+
const platformDir = path.join(platformsPath, config.path);
|
|
2003
|
+
|
|
2004
|
+
const rulesTxt = [];
|
|
2005
|
+
const rulesJson = {
|
|
2006
|
+
comment: LOCAL_SCRIPT_RULES_COMMENT,
|
|
2007
|
+
rules: [],
|
|
2008
|
+
};
|
|
2009
|
+
|
|
2010
|
+
// TODO: find a better way to iterate over ag filters
|
|
2011
|
+
// because AdGuard Chinese filter has id 224
|
|
2012
|
+
// https://github.com/AdguardTeam/FiltersRegistry/blob/master/filters/filter_224_Chinese/metadata.json
|
|
2013
|
+
for (let i = 1; i <= LAST_ADGUARD_FILTER_ID; i += 1) {
|
|
2014
|
+
const filterRules = readFile$2(path.join(platformDir, PLATFORM_FILTERS_DIR, `${i}.txt`));
|
|
2015
|
+
if (!filterRules) {
|
|
2016
|
+
continue;
|
|
2017
|
+
}
|
|
2018
|
+
|
|
2019
|
+
const lines = filterRules.split('\n');
|
|
2020
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
2021
|
+
for (let rule of lines) {
|
|
2022
|
+
rule = rule.trim();
|
|
2023
|
+
|
|
2024
|
+
if (!rule
|
|
2025
|
+
|| rule.startsWith(RuleMasks.MASK_COMMENT)) {
|
|
2026
|
+
continue;
|
|
2027
|
+
}
|
|
2028
|
+
|
|
2029
|
+
if (rule.includes(RuleMasks.MASK_SCRIPT)) {
|
|
2030
|
+
rulesTxt.push(rule);
|
|
2031
|
+
|
|
2032
|
+
const m = rule.split(RuleMasks.MASK_SCRIPT);
|
|
2033
|
+
rulesJson.rules.push({
|
|
2034
|
+
domains: m[0],
|
|
2035
|
+
script: m[1],
|
|
2036
|
+
});
|
|
2037
|
+
}
|
|
2038
|
+
}
|
|
2039
|
+
}
|
|
2040
|
+
|
|
2041
|
+
// https://github.com/AdguardTeam/AdguardBrowserExtension/issues/1847
|
|
2042
|
+
// remove scriptlet rules in local_script_rules.json
|
|
2043
|
+
rulesJson.rules = removeScriptletRules(rulesJson.rules);
|
|
2044
|
+
|
|
2045
|
+
fs.writeFileSync(
|
|
2046
|
+
path.join(platformDir, LOCAL_SCRIPT_RULES_FILE),
|
|
2047
|
+
rulesTxt.join(RULES_SEPARATOR),
|
|
2048
|
+
'utf8',
|
|
2049
|
+
);
|
|
2050
|
+
fs.writeFileSync(
|
|
2051
|
+
path.join(platformDir, LOCAL_SCRIPT_RULES_FILE_JSON),
|
|
2052
|
+
JSON.stringify(rulesJson, null, 4),
|
|
2053
|
+
'utf8',
|
|
2054
|
+
);
|
|
2055
|
+
}
|
|
2056
|
+
|
|
2057
|
+
logger.info('Writing local script rules done');
|
|
2058
|
+
};
|
|
2059
|
+
|
|
2060
|
+
/**
|
|
2061
|
+
* Loads and processes filter metadata from the specified directory.
|
|
2062
|
+
*
|
|
2063
|
+
* @param {string} filterDir - The directory containing the filter metadata and revision files.
|
|
2064
|
+
* @param {number[]} [whitelist] - An optional array of whitelist filter IDs.
|
|
2065
|
+
* @param {number[]} [blacklist] - An optional array of blacklist filter IDs.
|
|
2066
|
+
* @returns {Object} The processed filter metadata.
|
|
2067
|
+
* @throws {Error} If the metadata or revision file cannot be read.
|
|
2068
|
+
*/
|
|
2069
|
+
const loadFilterMetadata = function (filterDir, whitelist, blacklist) {
|
|
2070
|
+
const metadataFilePath = path.join(filterDir, metadataFile);
|
|
2071
|
+
const metadataString = readFile$2(metadataFilePath);
|
|
2072
|
+
if (!metadataString) {
|
|
2073
|
+
throw new Error(`Error reading filter metadata:${filterDir}`);
|
|
2074
|
+
}
|
|
2075
|
+
|
|
2076
|
+
const revisionFilePath = path.join(filterDir, revisionFile);
|
|
2077
|
+
const revisionString = readFile$2(revisionFilePath);
|
|
2078
|
+
if (!revisionString) {
|
|
2079
|
+
throw new Error(`Error reading filter revision:${filterDir}`);
|
|
2080
|
+
}
|
|
2081
|
+
|
|
2082
|
+
const revision = JSON.parse(revisionString);
|
|
2083
|
+
|
|
2084
|
+
const result = JSON.parse(metadataString);
|
|
2085
|
+
result.version = revision.version;
|
|
2086
|
+
result.timeUpdated = moment(revision.timeUpdated).format('YYYY-MM-DDTHH:mm:ssZZ');
|
|
2087
|
+
result.timeAdded = moment(result.timeAdded).format('YYYY-MM-DDTHH:mm:ssZZ');
|
|
2088
|
+
delete result.disabled;
|
|
2089
|
+
|
|
2090
|
+
const { filterId } = result;
|
|
2091
|
+
if (
|
|
2092
|
+
(whitelist && whitelist.includes(filterId))
|
|
2093
|
+
|| (blacklist && !blacklist.includes(filterId))
|
|
2094
|
+
) {
|
|
2095
|
+
checkFilterId(metadataFilterIdsPool, filterId);
|
|
2096
|
+
}
|
|
2097
|
+
|
|
2098
|
+
return result;
|
|
2099
|
+
};
|
|
2100
|
+
|
|
2101
|
+
/**
|
|
2102
|
+
* Exclude `#%#` and `#@%#` rules from rules list.
|
|
2103
|
+
*
|
|
2104
|
+
* @param {Array<any>} rules Input list of rules.
|
|
2105
|
+
* @return {Array<any>} Filtered list of rules.
|
|
2106
|
+
*/
|
|
2107
|
+
const excludeScriptRules = (rules) => {
|
|
2108
|
+
return rules.filter((rule) => {
|
|
2109
|
+
return rule
|
|
2110
|
+
&& !rule.includes(RuleMasks.MASK_SCRIPT)
|
|
2111
|
+
&& !rule.includes(RuleMasks.MASK_SCRIPT_EXCEPTION);
|
|
2112
|
+
});
|
|
2113
|
+
};
|
|
2114
|
+
|
|
2115
|
+
/**
|
|
2116
|
+
* Calculates checksum and writes filter file
|
|
2117
|
+
*
|
|
2118
|
+
* @param filterFile
|
|
2119
|
+
* @param adbHeader
|
|
2120
|
+
* @param rulesHeader
|
|
2121
|
+
* @param rules
|
|
2122
|
+
*/
|
|
2123
|
+
const writeFilterFile = function (filterFile, adbHeader, rulesHeader, rules) {
|
|
2124
|
+
let header = rulesHeader;
|
|
2125
|
+
if (adbHeader) {
|
|
2126
|
+
// Add Adb Plus compatibility header
|
|
2127
|
+
header = [adbHeader].concat(rulesHeader);
|
|
2128
|
+
}
|
|
2129
|
+
|
|
2130
|
+
const checksum = calculateChecksum(header, rules);
|
|
2131
|
+
|
|
2132
|
+
let data = [checksum].concat(rulesHeader).concat(rules);
|
|
2133
|
+
if (adbHeader) {
|
|
2134
|
+
data = [adbHeader].concat(data);
|
|
2135
|
+
}
|
|
2136
|
+
|
|
2137
|
+
fs.writeFileSync(filterFile, data.join(RULES_SEPARATOR), 'utf8');
|
|
2138
|
+
};
|
|
2139
|
+
|
|
2140
|
+
/**
|
|
2141
|
+
* Writes filter platform build
|
|
2142
|
+
*/
|
|
2143
|
+
const writeFilterRules = function (filterId, dir, config, rulesHeader, rules, optimized) {
|
|
2144
|
+
createDir(dir);
|
|
2145
|
+
|
|
2146
|
+
const filterFile = path.join(dir, `${filterId}${optimized ? '_optimized' : ''}.txt`);
|
|
2147
|
+
let rulesList = rules;
|
|
2148
|
+
|
|
2149
|
+
// Convert Adguard scriptlets and redirect rules to UBlock syntax.
|
|
2150
|
+
// Exclude script rules
|
|
2151
|
+
// and script rules exceptions https://github.com/AdguardTeam/FiltersCompiler/issues/199
|
|
2152
|
+
// Modify title for base filter
|
|
2153
|
+
if (config.platform === 'ext_ublock') {
|
|
2154
|
+
rulesList = convertToUbo(rulesList);
|
|
2155
|
+
rulesList = excludeScriptRules(rulesList);
|
|
2156
|
+
if (filterId === 2) {
|
|
2157
|
+
modifyBaseFilterHeader(rulesHeader, optimized);
|
|
2158
|
+
}
|
|
2159
|
+
}
|
|
2160
|
+
|
|
2161
|
+
writeFilterFile(filterFile, config.configuration.adbHeader, rulesHeader, rulesList);
|
|
2162
|
+
|
|
2163
|
+
// For English filter only we should provide additional filter version.
|
|
2164
|
+
if (filterId === 2 && config.platform === 'ext_ublock' && !optimized) {
|
|
2165
|
+
const correctedHeader = rewriteHeader(rulesHeader);
|
|
2166
|
+
const correctedRules = rewriteRules(rulesList);
|
|
2167
|
+
|
|
2168
|
+
const correctedFile = path.join(dir, `${filterId}_without_easylist.txt`);
|
|
2169
|
+
writeFilterFile(correctedFile, config.configuration.adbHeader, correctedHeader, correctedRules);
|
|
2170
|
+
}
|
|
2171
|
+
};
|
|
2172
|
+
|
|
2173
|
+
/**
|
|
2174
|
+
* Removes rules array duplicates,
|
|
2175
|
+
* ignores comments and hinted rules
|
|
2176
|
+
*
|
|
2177
|
+
* @param list a list of rules to filter
|
|
2178
|
+
* @returns {*} a list of rules without duplicate
|
|
2179
|
+
*/
|
|
2180
|
+
const removeRuleDuplicates = function (list) {
|
|
2181
|
+
logger.info('Removing duplicates..');
|
|
2182
|
+
|
|
2183
|
+
return list.filter((item, pos) => {
|
|
2184
|
+
// Do not remove hinted duplicates
|
|
2185
|
+
if (pos > 0) {
|
|
2186
|
+
const previous = list[pos - 1];
|
|
2187
|
+
if (previous && previous.startsWith(RuleMasks.MASK_HINT)) {
|
|
2188
|
+
return true;
|
|
2189
|
+
}
|
|
2190
|
+
}
|
|
2191
|
+
|
|
2192
|
+
// Do not remove hinted duplicates
|
|
2193
|
+
const duplicatePosition = list.indexOf(item, pos > 0 ? pos - 1 : pos);
|
|
2194
|
+
if (duplicatePosition !== pos && duplicatePosition > 0) {
|
|
2195
|
+
const duplicate = list[duplicatePosition - 1];
|
|
2196
|
+
if (duplicate && duplicate.startsWith(RuleMasks.MASK_HINT)) {
|
|
2197
|
+
return true;
|
|
2198
|
+
}
|
|
2199
|
+
}
|
|
2200
|
+
|
|
2201
|
+
// Do not remove commented duplicates
|
|
2202
|
+
const result = item.startsWith(RuleMasks.MASK_COMMENT) || duplicatePosition === pos;
|
|
2203
|
+
|
|
2204
|
+
if (!result) {
|
|
2205
|
+
logger.info(`${item} removed as duplicate`);
|
|
2206
|
+
}
|
|
2207
|
+
|
|
2208
|
+
return result;
|
|
2209
|
+
});
|
|
2210
|
+
};
|
|
2211
|
+
|
|
2212
|
+
/**
|
|
2213
|
+
* Builds platforms for filter
|
|
2214
|
+
*
|
|
2215
|
+
* @param filterDir - Path to filter directory
|
|
2216
|
+
* @param platformsPath - Path to platforms folder
|
|
2217
|
+
* @param whitelist - Array of filter ids to whitelist
|
|
2218
|
+
* @param blacklist - Array of filter ids to blacklist
|
|
2219
|
+
*/
|
|
2220
|
+
const buildFilter$1 = async (filterDir, platformsPath, whitelist, blacklist) => {
|
|
2221
|
+
const originalRules = readFile$2(path.join(filterDir, filterFile)).split('\r\n');
|
|
2222
|
+
|
|
2223
|
+
const metadataFilePath = path.join(filterDir, metadataFile);
|
|
2224
|
+
const revisionFilePath = path.join(filterDir, revisionFile);
|
|
2225
|
+
|
|
2226
|
+
const metadata = JSON.parse(readFile$2(metadataFilePath));
|
|
2227
|
+
const { filterId } = metadata;
|
|
2228
|
+
checkFilterId(filterIdsPool, filterId);
|
|
2229
|
+
|
|
2230
|
+
if (whitelist && whitelist.length > 0 && whitelist.indexOf(filterId) < 0) {
|
|
2231
|
+
logger.info(`Filter ${filterId} skipped with whitelist`);
|
|
2232
|
+
return;
|
|
2233
|
+
}
|
|
2234
|
+
|
|
2235
|
+
if (blacklist && blacklist.length > 0 && blacklist.indexOf(filterId) >= 0) {
|
|
2236
|
+
logger.info(`Filter ${filterId} skipped with blacklist`);
|
|
2237
|
+
return;
|
|
2238
|
+
}
|
|
2239
|
+
|
|
2240
|
+
const optimizationConfig = getFilterOptimizationConfig(filterId);
|
|
2241
|
+
|
|
2242
|
+
// eslint-disable-next-line guard-for-in,no-restricted-syntax
|
|
2243
|
+
for (const platform in platformPathsConfig) {
|
|
2244
|
+
const config = platformPathsConfig[platform];
|
|
2245
|
+
|
|
2246
|
+
if (!shouldBuildFilterForPlatform(metadata, config.platform)) {
|
|
2247
|
+
// eslint-disable-next-line max-len
|
|
2248
|
+
logger.info(`Build of filter ${filterId} skipped for platform '${config.platform}' due to platformsExcluded or platformsIncluded`);
|
|
2249
|
+
continue;
|
|
2250
|
+
}
|
|
2251
|
+
|
|
2252
|
+
let rules = filtersDownloader.FiltersDownloader.resolveConditions(originalRules, config.defines);
|
|
2253
|
+
|
|
2254
|
+
// handle includes after resolving conditions:
|
|
2255
|
+
// if there is a bad include after resolving conditions, the generator should be terminated
|
|
2256
|
+
// https://github.com/AdguardTeam/FiltersCompiler/issues/84
|
|
2257
|
+
// eslint-disable-next-line no-await-in-loop
|
|
2258
|
+
rules = await filtersDownloader.FiltersDownloader.resolveIncludes(rules, filterDir, config.defines);
|
|
2259
|
+
|
|
2260
|
+
rules = cleanupRules(rules, config, filterId);
|
|
2261
|
+
rules = removeRuleDuplicates(rules);
|
|
2262
|
+
|
|
2263
|
+
// Apply replacement rules
|
|
2264
|
+
if (config.configuration?.replacements) {
|
|
2265
|
+
rules = rules.map((rule) => {
|
|
2266
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
2267
|
+
for (const repl of config.configuration.replacements) {
|
|
2268
|
+
rule = rule.replace(new RegExp(repl.from, 'g'), repl.to);
|
|
2269
|
+
}
|
|
2270
|
+
return rule;
|
|
2271
|
+
});
|
|
2272
|
+
}
|
|
2273
|
+
|
|
2274
|
+
const optimizedRules = cleanupAndOptimizeRules(rules, config, optimizationConfig, filterId);
|
|
2275
|
+
// eslint-disable-next-line max-len
|
|
2276
|
+
logger.info(`Filter ${filterId}. Rules ${originalRules.length} => ${rules.length} => ${optimizedRules.length}. PlatformPath: '${config.path}'`);
|
|
2277
|
+
|
|
2278
|
+
const header = makeHeader(metadataFilePath, revisionFilePath, config.expires);
|
|
2279
|
+
|
|
2280
|
+
const platformDir = path.join(platformsPath, config.path, PLATFORM_FILTERS_DIR);
|
|
2281
|
+
writeFilterRules(filterId, platformDir, config, header, rules, false);
|
|
2282
|
+
|
|
2283
|
+
// add '(Optimized)' to the '! Title:' for optimized filters
|
|
2284
|
+
// https://github.com/AdguardTeam/FiltersCompiler/issues/78
|
|
2285
|
+
const optimizedHeader = [...header];
|
|
2286
|
+
optimizedHeader[0] += ' (Optimized)';
|
|
2287
|
+
|
|
2288
|
+
writeFilterRules(filterId, platformDir, config, optimizedHeader, optimizedRules, true);
|
|
2289
|
+
}
|
|
2290
|
+
};
|
|
2291
|
+
|
|
2292
|
+
/**
|
|
2293
|
+
* Initializes service
|
|
2294
|
+
*
|
|
2295
|
+
* @param filterFileName output filter file name.
|
|
2296
|
+
* @param metadataFileName output metadata file name.
|
|
2297
|
+
* @param revisionFileName output revision file name.
|
|
2298
|
+
* @param platformsConfig platforms configuration object.
|
|
2299
|
+
* @param adguardFiltersServer server that will serve the filters that're being built.
|
|
2300
|
+
*/
|
|
2301
|
+
const init = function (
|
|
2302
|
+
filterFileName,
|
|
2303
|
+
metadataFileName,
|
|
2304
|
+
revisionFileName,
|
|
2305
|
+
platformsConfig,
|
|
2306
|
+
adguardFiltersServer,
|
|
2307
|
+
) {
|
|
2308
|
+
filterFile = filterFileName;
|
|
2309
|
+
metadataFile = metadataFileName;
|
|
2310
|
+
revisionFile = revisionFileName;
|
|
2311
|
+
|
|
2312
|
+
adguardFiltersServerUrl = adguardFiltersServer;
|
|
2313
|
+
|
|
2314
|
+
if (!platformsConfig) {
|
|
2315
|
+
throw new Error('Platforms config is not defined');
|
|
2316
|
+
}
|
|
2317
|
+
|
|
2318
|
+
platformPathsConfig = platformsConfig;
|
|
2319
|
+
};
|
|
2320
|
+
|
|
2321
|
+
/**
|
|
2322
|
+
* Checks if filter has 'obsolete' tag
|
|
2323
|
+
* @param {object} metadata
|
|
2324
|
+
* @returns {boolean}
|
|
2325
|
+
*/
|
|
2326
|
+
const isObsoleteFilter = (metadata) => metadata.tags && metadata.tags.some((tag) => tag === 'obsolete');
|
|
2327
|
+
|
|
2328
|
+
/**
|
|
2329
|
+
* Parses directory recursive
|
|
2330
|
+
*
|
|
2331
|
+
* @param filtersDir
|
|
2332
|
+
* @param filtersMetadata
|
|
2333
|
+
* @param platformsPath
|
|
2334
|
+
* @param whitelist
|
|
2335
|
+
* @param blacklist
|
|
2336
|
+
* @param obsoleteFiltersMetadata
|
|
2337
|
+
*/
|
|
2338
|
+
const parseDirectory$1 = async (
|
|
2339
|
+
filtersDir,
|
|
2340
|
+
filtersMetadata,
|
|
2341
|
+
platformsPath,
|
|
2342
|
+
whitelist,
|
|
2343
|
+
blacklist,
|
|
2344
|
+
obsoleteFiltersMetadata,
|
|
2345
|
+
) => {
|
|
2346
|
+
const items = fs.readdirSync(filtersDir);
|
|
2347
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
2348
|
+
for (const directory of items) {
|
|
2349
|
+
const filterDir = path.join(filtersDir, directory);
|
|
2350
|
+
if (fs.lstatSync(filterDir).isDirectory()) {
|
|
2351
|
+
const metadataFilePath = path.join(filterDir, metadataFile);
|
|
2352
|
+
if (fs.existsSync(metadataFilePath)) {
|
|
2353
|
+
logger.info(`Building filter platforms: ${directory}`);
|
|
2354
|
+
// eslint-disable-next-line no-await-in-loop
|
|
2355
|
+
await buildFilter$1(filterDir, platformsPath, whitelist, blacklist);
|
|
2356
|
+
logger.info(`Building filter platforms: ${directory} done`);
|
|
2357
|
+
const filterMetadata = loadFilterMetadata(filterDir, whitelist, blacklist);
|
|
2358
|
+
filtersMetadata.push(filterMetadata);
|
|
2359
|
+
if (isObsoleteFilter(filterMetadata)) {
|
|
2360
|
+
obsoleteFiltersMetadata.push(filterMetadata);
|
|
2361
|
+
}
|
|
2362
|
+
} else {
|
|
2363
|
+
// eslint-disable-next-line no-await-in-loop
|
|
2364
|
+
await parseDirectory$1(
|
|
2365
|
+
filterDir,
|
|
2366
|
+
filtersMetadata,
|
|
2367
|
+
platformsPath,
|
|
2368
|
+
whitelist,
|
|
2369
|
+
blacklist,
|
|
2370
|
+
obsoleteFiltersMetadata,
|
|
2371
|
+
);
|
|
2372
|
+
}
|
|
2373
|
+
}
|
|
2374
|
+
}
|
|
2375
|
+
};
|
|
2376
|
+
|
|
2377
|
+
/**
|
|
2378
|
+
* Generates platform-specific files and metadata based on the provided filters and configurations.
|
|
2379
|
+
*
|
|
2380
|
+
* @async
|
|
2381
|
+
* @param {string} filtersDir - The directory containing filter files to be processed.
|
|
2382
|
+
* @param {string} platformsPath - The output path where the generated platform files will be stored.
|
|
2383
|
+
* @param {Array<number>} whitelist - A list of whitelist filter IDs.
|
|
2384
|
+
* @param {Array<number>} blacklist - A list of blacklist filter IDs.
|
|
2385
|
+
* @returns {Promise<void>} Resolves when the generation process is complete.
|
|
2386
|
+
*
|
|
2387
|
+
* @throws {Error} If `platformsPath` or `platformPathsConfig` is not specified.
|
|
2388
|
+
*/
|
|
2389
|
+
const generate = async (filtersDir, platformsPath, whitelist, blacklist) => {
|
|
2390
|
+
if (!platformsPath) {
|
|
2391
|
+
logger.warn('Platforms build output path is not specified');
|
|
2392
|
+
return;
|
|
2393
|
+
}
|
|
2394
|
+
|
|
2395
|
+
if (!platformPathsConfig) {
|
|
2396
|
+
logger.warn('Platforms configuration is not specified');
|
|
2397
|
+
return;
|
|
2398
|
+
}
|
|
2399
|
+
|
|
2400
|
+
createDir(platformsPath);
|
|
2401
|
+
|
|
2402
|
+
const filtersMetadata = [];
|
|
2403
|
+
const obsoleteFiltersMetadata = [];
|
|
2404
|
+
|
|
2405
|
+
await parseDirectory$1(filtersDir, filtersMetadata, platformsPath, whitelist, blacklist, obsoleteFiltersMetadata);
|
|
2406
|
+
|
|
2407
|
+
writeFiltersMetadata(platformsPath, filtersDir, filtersMetadata, obsoleteFiltersMetadata);
|
|
2408
|
+
writeLocalScriptRules(platformsPath);
|
|
2409
|
+
|
|
2410
|
+
// reset af the end
|
|
2411
|
+
// TODO: find out better way to reset
|
|
2412
|
+
filterIdsPool = [];
|
|
2413
|
+
};
|
|
2414
|
+
|
|
2415
|
+
const { log } = console;
|
|
2416
|
+
const reportDate = new Date();
|
|
2417
|
+
|
|
2418
|
+
let reportData = `\nFiltersCompiler report ${reportDate.toLocaleDateString()} ${reportDate.toLocaleTimeString()}:\n\n`;
|
|
2419
|
+
|
|
2420
|
+
/**
|
|
2421
|
+
* Adds filters data to report
|
|
2422
|
+
* @param {object} metadata
|
|
2423
|
+
* @param {object} filterRules
|
|
2424
|
+
* @param {string[]} invalidRules
|
|
2425
|
+
*/
|
|
2426
|
+
const addFilter = (metadata, filterRules, invalidRules) => {
|
|
2427
|
+
if (metadata && filterRules) {
|
|
2428
|
+
const { filterId, name, subscriptionUrl } = metadata;
|
|
2429
|
+
const filterLength = filterRules.lines.length;
|
|
2430
|
+
const excludedLength = filterRules.excluded.length;
|
|
2431
|
+
|
|
2432
|
+
reportData += `Filter ID: ${filterId}\n`
|
|
2433
|
+
+ `Filter name: ${name}\n`
|
|
2434
|
+
+ `Compiled rules: ${filterLength}\n`
|
|
2435
|
+
+ `Excluded rules: ${excludedLength}\n`
|
|
2436
|
+
+ `URL: ${subscriptionUrl}\n`;
|
|
2437
|
+
// log list of invalid rules only if they exist
|
|
2438
|
+
if (invalidRules.length > 0) {
|
|
2439
|
+
reportData += 'INVALID RULES:\n'
|
|
2440
|
+
+ `${invalidRules.join('\n')}\n`;
|
|
2441
|
+
} else {
|
|
2442
|
+
reportData += 'All rules are valid.\n';
|
|
2443
|
+
}
|
|
2444
|
+
reportData += '---------------------------\n';
|
|
2445
|
+
}
|
|
2446
|
+
};
|
|
2447
|
+
|
|
2448
|
+
/**
|
|
2449
|
+
* Adds disabled filters data to report
|
|
2450
|
+
* @param {object} metadata
|
|
2451
|
+
*/
|
|
2452
|
+
const skipFilter = (metadata) => {
|
|
2453
|
+
if (metadata) {
|
|
2454
|
+
const { filterId, name, subscriptionUrl } = metadata;
|
|
2455
|
+
|
|
2456
|
+
reportData += `Filter ID: ${filterId}\n`
|
|
2457
|
+
+ `Filter name: ${name}\n`
|
|
2458
|
+
+ 'Filter is DISABLED!\n'
|
|
2459
|
+
+ `URL: ${subscriptionUrl}\n`
|
|
2460
|
+
+ '---------------------------\n';
|
|
2461
|
+
}
|
|
2462
|
+
};
|
|
2463
|
+
|
|
2464
|
+
/**
|
|
2465
|
+
* Creates report file or outputs report
|
|
2466
|
+
* @param {string} reportPath
|
|
2467
|
+
*/
|
|
2468
|
+
const create = (reportPath) => {
|
|
2469
|
+
if (reportPath) {
|
|
2470
|
+
fs.writeFileSync(reportPath, reportData, 'utf8');
|
|
2471
|
+
return;
|
|
2472
|
+
}
|
|
2473
|
+
log(reportData);
|
|
2474
|
+
};
|
|
2475
|
+
|
|
2476
|
+
/**
|
|
2477
|
+
* Returns filter ID from directory name if it has a number id,
|
|
2478
|
+
* otherwise returns 0.
|
|
2479
|
+
*
|
|
2480
|
+
* @param {string} str Directory name.
|
|
2481
|
+
*
|
|
2482
|
+
* @returns {number} Filter ID for filter directory, otherwise 0.
|
|
2483
|
+
*/
|
|
2484
|
+
const getFilterIdFromDirName = (str) => {
|
|
2485
|
+
const chunks = str.split('_');
|
|
2486
|
+
const rawId = chunks.length > 1 ? Number(chunks[1]) : 0;
|
|
2487
|
+
return !Number.isNaN(rawId) ? rawId : 0;
|
|
2488
|
+
};
|
|
2489
|
+
|
|
2490
|
+
const DOT = '.';
|
|
2491
|
+
|
|
2492
|
+
/**
|
|
2493
|
+
* Modifies string to handle domains without rule markers
|
|
2494
|
+
*
|
|
2495
|
+
* @param {string} rule - Rule in base adblock syntax.
|
|
2496
|
+
* @returns {string} - Domain without base rule markers.
|
|
2497
|
+
*/
|
|
2498
|
+
const removeRuleMarkers = (rule) => rule
|
|
2499
|
+
.replace(RuleMasks.MASK_BASE_RULE, '')
|
|
2500
|
+
.replace(RuleMasks.MASK_RULE_SEPARATOR, '');
|
|
2501
|
+
|
|
2502
|
+
/**
|
|
2503
|
+
* Checks if the line is in base rule style syntax with no modifier, i.e.,
|
|
2504
|
+
* starts with `||` and ends with `^`.
|
|
2505
|
+
*
|
|
2506
|
+
* @param {string} rule - Rule to check.
|
|
2507
|
+
* @returns {boolean} - True if the rule is in base rule style syntax.
|
|
2508
|
+
*/
|
|
2509
|
+
const shouldOptimize = (rule) => {
|
|
2510
|
+
return rule.startsWith(RuleMasks.MASK_BASE_RULE)
|
|
2511
|
+
&& rule.endsWith(RuleMasks.MASK_RULE_SEPARATOR);
|
|
2512
|
+
};
|
|
2513
|
+
|
|
2514
|
+
/**
|
|
2515
|
+
* Returns the top level domain of the given domain.
|
|
2516
|
+
*
|
|
2517
|
+
* @param {string} domain Domain to get the top level domain from.
|
|
2518
|
+
*
|
|
2519
|
+
* @returns {string} Top level domain.
|
|
2520
|
+
*/
|
|
2521
|
+
const getTopLevelDomain = (domain) => {
|
|
2522
|
+
const parsedDomain = tldts.parse(domain).domain;
|
|
2523
|
+
return typeof parsedDomain === 'string' ? parsedDomain : domain;
|
|
2524
|
+
};
|
|
2525
|
+
|
|
2526
|
+
/**
|
|
2527
|
+
* Finds the widest domains in the given list.
|
|
2528
|
+
*
|
|
2529
|
+
* @param {Set<string>} domains Set of domains to find the widest domains from.
|
|
2530
|
+
*
|
|
2531
|
+
* @returns {Set<string>} Set of widest domains.
|
|
2532
|
+
*
|
|
2533
|
+
* @example
|
|
2534
|
+
* - example.com, sub1.example.com, abc.sub2.example.com -> example.com
|
|
2535
|
+
* - example.org, example.com -> example.org, example.com
|
|
2536
|
+
*/
|
|
2537
|
+
const findWidestDomains = (domains) => {
|
|
2538
|
+
const sortedDomains = [...domains].sort((a, b) => {
|
|
2539
|
+
return a.split(DOT).length - b.split(DOT).length;
|
|
2540
|
+
});
|
|
2541
|
+
|
|
2542
|
+
const result = new Set();
|
|
2543
|
+
|
|
2544
|
+
sortedDomains.forEach((domain) => {
|
|
2545
|
+
let isSubdomain = false;
|
|
2546
|
+
result.forEach((parent) => {
|
|
2547
|
+
if (domain.endsWith(`${DOT}${parent}`)) {
|
|
2548
|
+
isSubdomain = true;
|
|
2549
|
+
}
|
|
2550
|
+
});
|
|
2551
|
+
|
|
2552
|
+
if (!isSubdomain) {
|
|
2553
|
+
result.add(domain);
|
|
2554
|
+
}
|
|
2555
|
+
});
|
|
2556
|
+
|
|
2557
|
+
return result;
|
|
2558
|
+
};
|
|
2559
|
+
|
|
2560
|
+
/**
|
|
2561
|
+
* Removes redundant rules from lines
|
|
2562
|
+
* @param {string[]} lines - An array of text lines.
|
|
2563
|
+
* @returns {string[]} - An array of of text lines with redundant rules removed.
|
|
2564
|
+
*/
|
|
2565
|
+
const optimizeDomainBlockingRules = async (lines) => {
|
|
2566
|
+
const linesToSkipOptimization = new Set();
|
|
2567
|
+
const rawDomainsToOptimize = new Set();
|
|
2568
|
+
|
|
2569
|
+
lines.forEach((line) => {
|
|
2570
|
+
if (!shouldOptimize(line)) {
|
|
2571
|
+
linesToSkipOptimization.add(line);
|
|
2572
|
+
return;
|
|
2573
|
+
}
|
|
2574
|
+
|
|
2575
|
+
const rawDomain = removeRuleMarkers(line);
|
|
2576
|
+
rawDomainsToOptimize.add(rawDomain);
|
|
2577
|
+
});
|
|
2578
|
+
|
|
2579
|
+
/**
|
|
2580
|
+
* Map of tld for all raw domains
|
|
2581
|
+
* @type {Map<string, Set<string>>}
|
|
2582
|
+
*
|
|
2583
|
+
* It is needed to group rawDomains by top level domain
|
|
2584
|
+
* so groups of related rawDomains can be optimized in parallel.
|
|
2585
|
+
*/
|
|
2586
|
+
const topDomainsMap = new Map();
|
|
2587
|
+
|
|
2588
|
+
rawDomainsToOptimize.forEach((rawDomain) => {
|
|
2589
|
+
const topLevelDomain = getTopLevelDomain(rawDomain);
|
|
2590
|
+
|
|
2591
|
+
if (!topDomainsMap.has(topLevelDomain)) {
|
|
2592
|
+
topDomainsMap.set(topLevelDomain, new Set([rawDomain]));
|
|
2593
|
+
} else {
|
|
2594
|
+
topDomainsMap.get(topLevelDomain).add(rawDomain);
|
|
2595
|
+
}
|
|
2596
|
+
});
|
|
2597
|
+
|
|
2598
|
+
const widerDomains = new Set();
|
|
2599
|
+
|
|
2600
|
+
/**
|
|
2601
|
+
* Runs optimization for a group of related domains -
|
|
2602
|
+
* finds the widest domains in the group and adds them to the result list of wider domains.
|
|
2603
|
+
*
|
|
2604
|
+
* @param {Set<string>} rawDomains Set of related raw domains to optimize.
|
|
2605
|
+
*/
|
|
2606
|
+
const optimizeDomains = (rawDomains) => {
|
|
2607
|
+
const widestDomains = findWidestDomains(rawDomains);
|
|
2608
|
+
widestDomains.forEach((domain) => {
|
|
2609
|
+
widerDomains.add(domain);
|
|
2610
|
+
});
|
|
2611
|
+
};
|
|
2612
|
+
|
|
2613
|
+
topDomainsMap.forEach((rawDomains) => {
|
|
2614
|
+
optimizeDomains(rawDomains);
|
|
2615
|
+
});
|
|
2616
|
+
|
|
2617
|
+
// return lines in the order they were given
|
|
2618
|
+
return lines.filter((line) => {
|
|
2619
|
+
if (linesToSkipOptimization.has(line)) {
|
|
2620
|
+
return true;
|
|
2621
|
+
}
|
|
2622
|
+
|
|
2623
|
+
const domain = removeRuleMarkers(line);
|
|
2624
|
+
|
|
2625
|
+
return widerDomains.has(domain);
|
|
2626
|
+
});
|
|
2627
|
+
};
|
|
2628
|
+
|
|
2629
|
+
/* eslint-disable global-require */
|
|
2630
|
+
|
|
2631
|
+
|
|
2632
|
+
const __dirname$3 = path.dirname(new URL((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('index.cjs', document.baseURI).href))).pathname);
|
|
2633
|
+
|
|
2634
|
+
const TEMPLATE_FILE = 'template.txt';
|
|
2635
|
+
const FILTER_FILE = 'filter.txt';
|
|
2636
|
+
const REVISION_FILE = 'revision.json';
|
|
2637
|
+
const EXCLUDE_FILE = 'exclude.txt';
|
|
2638
|
+
const EXCLUDED_LINES_FILE = 'diff.txt';
|
|
2639
|
+
const METADATA_FILE = 'metadata.json';
|
|
2640
|
+
const ADGUARD_FILTERS_SERVER_URL = 'https://filters.adtidy.org/';
|
|
2641
|
+
const TRUST_LEVEL_DIR = './utils/trust-levels';
|
|
2642
|
+
const DEFAULT_TRUST_LEVEL = 'low';
|
|
2643
|
+
|
|
2644
|
+
const SPACE = ' ';
|
|
2645
|
+
const SLASH = '/';
|
|
2646
|
+
const COMMA = ',';
|
|
2647
|
+
const EQUAL_SIGN = '=';
|
|
2648
|
+
const MODIFIERS_SEPARATOR = '$';
|
|
2649
|
+
const INCLUDE_DIRECTIVE = '@include ';
|
|
2650
|
+
const STRIP_COMMENTS_OPTION = 'stripComments';
|
|
2651
|
+
const OPTIMIZE_DOMAIN_BLOCKING_RULES = 'optimizeDomainBlockingRules';
|
|
2652
|
+
const NOT_OPTIMIZED_OPTION = 'notOptimized';
|
|
2653
|
+
const EXCLUDE_OPTION = 'exclude';
|
|
2654
|
+
const ADD_MODIFIERS_OPTION = 'addModifiers';
|
|
2655
|
+
const IGNORE_TRUST_LEVEL_OPTION = 'ignoreTrustLevel';
|
|
2656
|
+
|
|
2657
|
+
const NOT_OPTIMIZED_HINT = '!+ NOT_OPTIMIZED';
|
|
2658
|
+
|
|
2659
|
+
/**
|
|
2660
|
+
* Filter header tag for diff path which is needed for patch updates.
|
|
2661
|
+
*/
|
|
2662
|
+
const DIFF_PATH_TAG = 'Diff-Path:';
|
|
2663
|
+
|
|
2664
|
+
// TODO: Move include directive option functions to utils/builder-utils.js
|
|
2665
|
+
|
|
2666
|
+
/**
|
|
2667
|
+
* Sync reads file content
|
|
2668
|
+
*
|
|
2669
|
+
* @param path
|
|
2670
|
+
* @returns {*}
|
|
2671
|
+
*/
|
|
2672
|
+
const readFile$1 = function (path) {
|
|
2673
|
+
if (!fs.existsSync(path)) {
|
|
2674
|
+
return null;
|
|
2675
|
+
}
|
|
2676
|
+
|
|
2677
|
+
return fs.readFileSync(path, { encoding: 'utf-8' });
|
|
2678
|
+
};
|
|
2679
|
+
|
|
2680
|
+
/**
|
|
2681
|
+
* Sync writes content to file
|
|
2682
|
+
*
|
|
2683
|
+
* @param path
|
|
2684
|
+
* @param data
|
|
2685
|
+
*/
|
|
2686
|
+
const writeFile = function (path, data) {
|
|
2687
|
+
fs.writeFileSync(path, data, 'utf8');
|
|
2688
|
+
};
|
|
2689
|
+
|
|
2690
|
+
/**
|
|
2691
|
+
* Splits lines
|
|
2692
|
+
*
|
|
2693
|
+
* @param string
|
|
2694
|
+
*/
|
|
2695
|
+
const splitLines = function (string) {
|
|
2696
|
+
return string.split(/\r?\n/);
|
|
2697
|
+
};
|
|
2698
|
+
|
|
2699
|
+
/**
|
|
2700
|
+
* Removes comments from lines
|
|
2701
|
+
*
|
|
2702
|
+
* @param lines
|
|
2703
|
+
*/
|
|
2704
|
+
const stripComments = function (lines) {
|
|
2705
|
+
logger.info('Stripping comments..');
|
|
2706
|
+
|
|
2707
|
+
return lines.filter((line, pos) => {
|
|
2708
|
+
if (pos > 0 && lines[pos - 1].startsWith(RuleMasks.MASK_HINT)) {
|
|
2709
|
+
return true;
|
|
2710
|
+
}
|
|
2711
|
+
|
|
2712
|
+
if (
|
|
2713
|
+
line.startsWith(RuleMasks.MASK_HINT)
|
|
2714
|
+
|| line.startsWith(RuleMasks.MASK_DIRECTIVES)
|
|
2715
|
+
) {
|
|
2716
|
+
return true;
|
|
2717
|
+
}
|
|
2718
|
+
|
|
2719
|
+
return !line.startsWith(RuleMasks.MASK_COMMENT);
|
|
2720
|
+
});
|
|
2721
|
+
};
|
|
2722
|
+
|
|
2723
|
+
/**
|
|
2724
|
+
* Adds not optimized hints
|
|
2725
|
+
*
|
|
2726
|
+
* @param lines
|
|
2727
|
+
*/
|
|
2728
|
+
const addNotOptimizedHints = function (lines) {
|
|
2729
|
+
logger.info('Adding hints..');
|
|
2730
|
+
|
|
2731
|
+
const result = [];
|
|
2732
|
+
|
|
2733
|
+
lines.forEach((v) => {
|
|
2734
|
+
if (!v) {
|
|
2735
|
+
return;
|
|
2736
|
+
}
|
|
2737
|
+
|
|
2738
|
+
if (!v.startsWith(RuleMasks.MASK_COMMENT) && !v.startsWith(`${RuleMasks.MASK_HINT}${SPACE}`)) {
|
|
2739
|
+
result.push(NOT_OPTIMIZED_HINT);
|
|
2740
|
+
}
|
|
2741
|
+
|
|
2742
|
+
result.push(v);
|
|
2743
|
+
});
|
|
2744
|
+
|
|
2745
|
+
return result;
|
|
2746
|
+
};
|
|
2747
|
+
/**
|
|
2748
|
+
* Adds or updates modifiers in each line of the given array of lines.
|
|
2749
|
+
*
|
|
2750
|
+
* @param {string[]} lines - An array of text lines.
|
|
2751
|
+
* @param {string} modifiersStr - Modifiers as a string to add.
|
|
2752
|
+
* @returns {string[]} - An array of modified lines.
|
|
2753
|
+
*/
|
|
2754
|
+
const addModifiers = (lines, modifiersStr) => {
|
|
2755
|
+
return lines.map((line) => {
|
|
2756
|
+
// If the line is empty or contains only whitespace, it returns a comment mask
|
|
2757
|
+
if (!line || line.trim() === '') {
|
|
2758
|
+
return RuleMasks.MASK_COMMENT;
|
|
2759
|
+
}
|
|
2760
|
+
// If the line starts with a host file comment mask, it replaces it with a comment mask
|
|
2761
|
+
if (line.startsWith(RuleMasks.MASK_HOST_FILE_COMMENT)) {
|
|
2762
|
+
return line.replace(RuleMasks.MASK_HOST_FILE_COMMENT, RuleMasks.MASK_COMMENT);
|
|
2763
|
+
}
|
|
2764
|
+
// If the line starts with a comment mask, it returns the line as is.
|
|
2765
|
+
if (line.startsWith(RuleMasks.MASK_COMMENT)) {
|
|
2766
|
+
return line;
|
|
2767
|
+
}
|
|
2768
|
+
// If the line does not contain a modifiers separator, it appends the given modifiers string
|
|
2769
|
+
if (!line.includes(MODIFIERS_SEPARATOR)) {
|
|
2770
|
+
return `${line}${MODIFIERS_SEPARATOR}${modifiersStr}`;
|
|
2771
|
+
}
|
|
2772
|
+
// If the line already contains modifiers, combine the existing modifiers with the new ones,
|
|
2773
|
+
// ensuring no duplicates, and return the line with the updated modifiers.
|
|
2774
|
+
const [rule, existingModifiersStr] = line.split(MODIFIERS_SEPARATOR);
|
|
2775
|
+
const existingModifiers = existingModifiersStr.split(COMMA);
|
|
2776
|
+
const newModifiers = modifiersStr.split(COMMA);
|
|
2777
|
+
const combinedModifiers = [...new Set([...existingModifiers, ...newModifiers])].join(COMMA);
|
|
2778
|
+
|
|
2779
|
+
return `${rule}${MODIFIERS_SEPARATOR}${combinedModifiers}`;
|
|
2780
|
+
});
|
|
2781
|
+
};
|
|
2782
|
+
|
|
2783
|
+
/**
|
|
2784
|
+
* Checks case when '#%#' rules are excluded to DON'T exclude '#%#//scriptlet' rules
|
|
2785
|
+
*
|
|
2786
|
+
* @param {string} line
|
|
2787
|
+
* @param {string} exclusion
|
|
2788
|
+
* @return {boolean}
|
|
2789
|
+
*/
|
|
2790
|
+
const scriptletException = (line, exclusion) => (exclusion === '#%#' && line.includes('#%#//scriptlet'))
|
|
2791
|
+
|| (exclusion === '#@%#' && line.includes('#@%#//scriptlet'));
|
|
2792
|
+
|
|
2793
|
+
/**
|
|
2794
|
+
* Checks if line is excluded with specified set of exclusions
|
|
2795
|
+
*
|
|
2796
|
+
* @param line
|
|
2797
|
+
* @param exclusions
|
|
2798
|
+
* @param excluded
|
|
2799
|
+
* @param reason
|
|
2800
|
+
*/
|
|
2801
|
+
const isExcluded = function (line, exclusions, excluded, reason) {
|
|
2802
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
2803
|
+
for (let exclusion of exclusions) {
|
|
2804
|
+
exclusion = exclusion.trim();
|
|
2805
|
+
|
|
2806
|
+
if (exclusion && !exclusion.startsWith(RuleMasks.MASK_COMMENT)) {
|
|
2807
|
+
const message = `${line} is excluded by "${exclusion}" in ${reason}`;
|
|
2808
|
+
|
|
2809
|
+
const isExcludedByRegexp = exclusion.endsWith(SLASH) && exclusion.startsWith(SLASH)
|
|
2810
|
+
&& line.match(new RegExp(exclusion.substring(1, exclusion.length - 1)));
|
|
2811
|
+
|
|
2812
|
+
if ((isExcludedByRegexp || line.includes(exclusion)) && !scriptletException(line, exclusion)) {
|
|
2813
|
+
logger.info(message);
|
|
2814
|
+
excluded.push(`${RuleMasks.MASK_COMMENT}${SPACE}${message}`);
|
|
2815
|
+
excluded.push(line);
|
|
2816
|
+
return exclusion;
|
|
2817
|
+
}
|
|
2818
|
+
}
|
|
2819
|
+
}
|
|
2820
|
+
|
|
2821
|
+
return null;
|
|
2822
|
+
};
|
|
2823
|
+
|
|
2824
|
+
/**
|
|
2825
|
+
* Applies exclusion from exclusions file
|
|
2826
|
+
*
|
|
2827
|
+
* @param lines
|
|
2828
|
+
* @param exclusionsFile
|
|
2829
|
+
* @param excluded
|
|
2830
|
+
* @returns {*}
|
|
2831
|
+
*/
|
|
2832
|
+
const exclude = function (lines, exclusionsFile, excluded) {
|
|
2833
|
+
logger.info('Applying exclusions..');
|
|
2834
|
+
|
|
2835
|
+
let exclusions = readFile$1(exclusionsFile);
|
|
2836
|
+
if (!exclusions) {
|
|
2837
|
+
return lines;
|
|
2838
|
+
}
|
|
2839
|
+
|
|
2840
|
+
exclusions = splitLines(exclusions);
|
|
2841
|
+
|
|
2842
|
+
const exclusionsFileName = path.parse(exclusionsFile).base;
|
|
2843
|
+
const result = [];
|
|
2844
|
+
|
|
2845
|
+
lines.forEach((line, pos) => {
|
|
2846
|
+
const exclusion = isExcluded(line, exclusions, excluded, exclusionsFileName);
|
|
2847
|
+
if (exclusion) {
|
|
2848
|
+
if (pos > 0 && lines[pos - 1].startsWith(RuleMasks.MASK_HINT)) {
|
|
2849
|
+
result.push(`${RuleMasks.MASK_COMMENT} [excluded by ${exclusion}] ${line}`);
|
|
2850
|
+
}
|
|
2851
|
+
} else {
|
|
2852
|
+
result.push(line);
|
|
2853
|
+
}
|
|
2854
|
+
});
|
|
2855
|
+
|
|
2856
|
+
return result;
|
|
2857
|
+
};
|
|
2858
|
+
|
|
2859
|
+
/**
|
|
2860
|
+
* Strips leading and trailing quotes from string
|
|
2861
|
+
*
|
|
2862
|
+
* @param s
|
|
2863
|
+
* @returns {*}
|
|
2864
|
+
*/
|
|
2865
|
+
const stripEndQuotes = function (s) {
|
|
2866
|
+
let t = s.length;
|
|
2867
|
+
if (s.charAt(0) === '"') {
|
|
2868
|
+
s = s.substring(1, t -= 1);
|
|
2869
|
+
}
|
|
2870
|
+
// eslint-disable-next-line no-plusplus
|
|
2871
|
+
if (s.charAt(--t) === '"') {
|
|
2872
|
+
s = s.substring(0, t);
|
|
2873
|
+
}
|
|
2874
|
+
return s;
|
|
2875
|
+
};
|
|
2876
|
+
|
|
2877
|
+
/**
|
|
2878
|
+
* Extracts the value of an attribute.
|
|
2879
|
+
*
|
|
2880
|
+
* @param {string} attribute - The input attribute string.
|
|
2881
|
+
* @returns {string} The extracted attribute value without quotes if it is not empty.
|
|
2882
|
+
* @throws {Error} If the attribute value is empty.
|
|
2883
|
+
*/
|
|
2884
|
+
const getOptionValue = (attribute) => {
|
|
2885
|
+
const quotedValue = attribute.substring(attribute.indexOf(EQUAL_SIGN) + 1);
|
|
2886
|
+
const value = stripEndQuotes(quotedValue).trim();
|
|
2887
|
+
if (value.length === 0) {
|
|
2888
|
+
throw new Error(`Include directive value cannot be empty: '${attribute}'`);
|
|
2889
|
+
}
|
|
2890
|
+
return value;
|
|
2891
|
+
};
|
|
2892
|
+
|
|
2893
|
+
/**
|
|
2894
|
+
* @typedef {object} IncludeOption
|
|
2895
|
+
* @property {string} name Option name
|
|
2896
|
+
* @property {boolean|string} value Option value
|
|
2897
|
+
*/
|
|
2898
|
+
|
|
2899
|
+
/**
|
|
2900
|
+
* @typedef {object} ParsedIncludeData
|
|
2901
|
+
* @property {string} url Parsed url of file to include
|
|
2902
|
+
* @property {IncludeOption[]} options Array of parsed include directive options
|
|
2903
|
+
*/
|
|
2904
|
+
|
|
2905
|
+
/**
|
|
2906
|
+
* Parses an include directive line to extract the URL and options.
|
|
2907
|
+
*
|
|
2908
|
+
* @param {string} lineWithDirective - The input line with directive to parse.
|
|
2909
|
+
* @returns {ParsedIncludeData} Parsed result containing the URL and options.
|
|
2910
|
+
*/
|
|
2911
|
+
const parseIncludeDirective = function (lineWithDirective) {
|
|
2912
|
+
const parts = lineWithDirective.trim().split(SPACE);
|
|
2913
|
+
let url = parts[1].trim();
|
|
2914
|
+
url = stripEndQuotes(url);
|
|
2915
|
+
// Initialize an options array to store the parsed options.
|
|
2916
|
+
const options = [];
|
|
2917
|
+
// Stack options into the array in the sequence in which they found in the string
|
|
2918
|
+
for (let i = 1; i < parts.length; i += 1) {
|
|
2919
|
+
const attribute = parts[i].trim();
|
|
2920
|
+
if (attribute.startsWith(`${SLASH}${STRIP_COMMENTS_OPTION}`)) {
|
|
2921
|
+
options.push({ name: STRIP_COMMENTS_OPTION, value: true });
|
|
2922
|
+
} else if (attribute.startsWith(`${SLASH}${OPTIMIZE_DOMAIN_BLOCKING_RULES}`)) {
|
|
2923
|
+
options.push({ name: OPTIMIZE_DOMAIN_BLOCKING_RULES, value: true });
|
|
2924
|
+
} else if (attribute.startsWith(`${SLASH}${NOT_OPTIMIZED_OPTION}`)) {
|
|
2925
|
+
options.push({ name: NOT_OPTIMIZED_OPTION, value: true });
|
|
2926
|
+
} else if (attribute.startsWith(`${SLASH}${EXCLUDE_OPTION}${EQUAL_SIGN}`)) {
|
|
2927
|
+
options.push({ name: EXCLUDE_OPTION, value: getOptionValue(attribute) });
|
|
2928
|
+
} else if (attribute.startsWith(`${SLASH}${ADD_MODIFIERS_OPTION}${EQUAL_SIGN}`)) {
|
|
2929
|
+
options.push({ name: ADD_MODIFIERS_OPTION, value: getOptionValue(attribute) });
|
|
2930
|
+
} else if (attribute.startsWith(`${SLASH}${IGNORE_TRUST_LEVEL_OPTION}`)) {
|
|
2931
|
+
options.push({ name: IGNORE_TRUST_LEVEL_OPTION, value: true });
|
|
2932
|
+
}
|
|
2933
|
+
}
|
|
2934
|
+
return { url, options };
|
|
2935
|
+
};
|
|
2936
|
+
|
|
2937
|
+
/**
|
|
2938
|
+
* Checks if lines contains invalid redirect directives
|
|
2939
|
+
*
|
|
2940
|
+
* @param lines
|
|
2941
|
+
* @param url
|
|
2942
|
+
*/
|
|
2943
|
+
const checkRedirects = function (lines, url) {
|
|
2944
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
2945
|
+
for (const line of lines) {
|
|
2946
|
+
if (/^!\s?[Rr]edirect:/.test(line)) {
|
|
2947
|
+
throw new Error(`Error: include ${url} contains redirect directive: ${line}`);
|
|
2948
|
+
}
|
|
2949
|
+
}
|
|
2950
|
+
};
|
|
2951
|
+
|
|
2952
|
+
/**
|
|
2953
|
+
* @typedef {object} ParsedIncludeResult
|
|
2954
|
+
* @property {string[]} includedLines Included rules.
|
|
2955
|
+
* @property {boolean} shouldIgnoreTrustLevel Indicates whether the metadata's trust level should be ignored.
|
|
2956
|
+
*/
|
|
2957
|
+
|
|
2958
|
+
/**
|
|
2959
|
+
* Creates content from compiler's `@include` directive, not `!#include` preprocessor directive.
|
|
2960
|
+
*
|
|
2961
|
+
* @param {string} filterDir Filter directory.
|
|
2962
|
+
* @param {string} directiveLine The include line containing the URL or file path and optional options.
|
|
2963
|
+
* @param {Array<string>} excluded An array of strings representing excluded content.
|
|
2964
|
+
*
|
|
2965
|
+
* @returns {Promise<ParsedIncludeResult>} Parsed include data.
|
|
2966
|
+
* @throws {Error} Throws an error if there is an issue handling the include operation.
|
|
2967
|
+
*/
|
|
2968
|
+
const include = async (filterDir, directiveLine, excluded) => {
|
|
2969
|
+
let includedLines = [];
|
|
2970
|
+
let shouldIgnoreTrustLevel = false;
|
|
2971
|
+
|
|
2972
|
+
const { url, options } = parseIncludeDirective(directiveLine);
|
|
2973
|
+
|
|
2974
|
+
if (!url) {
|
|
2975
|
+
logger.warn('Invalid include url');
|
|
2976
|
+
return includedLines;
|
|
2977
|
+
}
|
|
2978
|
+
|
|
2979
|
+
logger.info(`Applying inclusion from: ${url}`);
|
|
2980
|
+
|
|
2981
|
+
const externalInclude = url.includes(':');
|
|
2982
|
+
|
|
2983
|
+
const included = externalInclude
|
|
2984
|
+
? downloadFile(url)
|
|
2985
|
+
: readFile$1(path.join(filterDir, url));
|
|
2986
|
+
|
|
2987
|
+
if (included) {
|
|
2988
|
+
includedLines = splitLines(included);
|
|
2989
|
+
|
|
2990
|
+
checkRedirects(includedLines, url);
|
|
2991
|
+
|
|
2992
|
+
// resolved `@include` directive url
|
|
2993
|
+
const originUrl = externalInclude
|
|
2994
|
+
? filtersDownloader.FiltersDownloader.getFilterUrlOrigin(url)
|
|
2995
|
+
: filterDir;
|
|
2996
|
+
|
|
2997
|
+
includedLines = await filtersDownloader.FiltersDownloader.resolveIncludes(includedLines, originUrl);
|
|
2998
|
+
|
|
2999
|
+
includedLines = removeAdblockVersion(includedLines);
|
|
3000
|
+
|
|
3001
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
3002
|
+
for (const { name, value } of options) {
|
|
3003
|
+
let optionsExcludePath;
|
|
3004
|
+
switch (name) {
|
|
3005
|
+
case EXCLUDE_OPTION:
|
|
3006
|
+
optionsExcludePath = path.join(filterDir, value);
|
|
3007
|
+
includedLines = exclude(includedLines, optionsExcludePath, excluded);
|
|
3008
|
+
break;
|
|
3009
|
+
case STRIP_COMMENTS_OPTION:
|
|
3010
|
+
includedLines = stripComments(includedLines);
|
|
3011
|
+
break;
|
|
3012
|
+
case OPTIMIZE_DOMAIN_BLOCKING_RULES:
|
|
3013
|
+
// eslint-disable-next-line no-await-in-loop
|
|
3014
|
+
includedLines = await optimizeDomainBlockingRules(includedLines);
|
|
3015
|
+
break;
|
|
3016
|
+
case NOT_OPTIMIZED_OPTION:
|
|
3017
|
+
includedLines = addNotOptimizedHints(includedLines);
|
|
3018
|
+
break;
|
|
3019
|
+
case ADD_MODIFIERS_OPTION:
|
|
3020
|
+
includedLines = addModifiers(includedLines, value);
|
|
3021
|
+
break;
|
|
3022
|
+
case IGNORE_TRUST_LEVEL_OPTION:
|
|
3023
|
+
if (externalInclude) {
|
|
3024
|
+
// eslint-disable-next-line max-len
|
|
3025
|
+
throw new Error(`Trust level ignoring option is not supported for external includes: ${directiveLine}`);
|
|
3026
|
+
}
|
|
3027
|
+
shouldIgnoreTrustLevel = true;
|
|
3028
|
+
break;
|
|
3029
|
+
}
|
|
3030
|
+
}
|
|
3031
|
+
|
|
3032
|
+
includedLines = fixVersionComments(includedLines);
|
|
3033
|
+
} else {
|
|
3034
|
+
throw new Error(`Error handling include from: ${options.url}`);
|
|
3035
|
+
}
|
|
3036
|
+
|
|
3037
|
+
logger.info(`Inclusion lines: ${includedLines.length}`);
|
|
3038
|
+
logger.info(`Should be filtered due to trust level: ${shouldIgnoreTrustLevel}`);
|
|
3039
|
+
|
|
3040
|
+
return {
|
|
3041
|
+
includedLines,
|
|
3042
|
+
shouldIgnoreTrustLevel,
|
|
3043
|
+
};
|
|
3044
|
+
};
|
|
3045
|
+
|
|
3046
|
+
/**
|
|
3047
|
+
* Resolves preprocessor `!#include` directives by FiltersDownloader.
|
|
3048
|
+
*
|
|
3049
|
+
* @param {string} filterDir Filter directory.
|
|
3050
|
+
* @param {string} filterName Filter name.
|
|
3051
|
+
* @param {string[]} lines Array of lines.
|
|
3052
|
+
*
|
|
3053
|
+
* @returns {Promise<string[]>} Promise which resolves
|
|
3054
|
+
* to array of rules as a result of resolving preprocessor `!#include` directives.
|
|
3055
|
+
*
|
|
3056
|
+
* @throws {Error} Throws an error if unable to resolve `!#include` directives.
|
|
3057
|
+
*/
|
|
3058
|
+
const getResolvedPreprocessorIncludes = async function (filterDir, filterName, lines) {
|
|
3059
|
+
let rules = lines;
|
|
3060
|
+
// bad includes are ignored here because they'll be handled in generator after resolving conditions
|
|
3061
|
+
// https://github.com/AdguardTeam/FiltersCompiler/issues/84
|
|
3062
|
+
try {
|
|
3063
|
+
rules = await filtersDownloader.FiltersDownloader.resolveIncludes(lines, filterDir);
|
|
3064
|
+
} catch (e) {
|
|
3065
|
+
logger.warn(`Error resolving includes in ${filterName}: ${e.message}`);
|
|
3066
|
+
}
|
|
3067
|
+
return rules;
|
|
3068
|
+
};
|
|
3069
|
+
|
|
3070
|
+
/**
|
|
3071
|
+
* Resolved preprocessor `!#include` directives and converts rules to Adguard syntax.
|
|
3072
|
+
*
|
|
3073
|
+
* @param {string} filterDir Filter directory.
|
|
3074
|
+
* @param {string} filterName Filter name.
|
|
3075
|
+
* @param {string[]} lines Array of raw rules and possibly preprocessor `!#include` directives.
|
|
3076
|
+
* @param {string} excluded Array of rules to exclude.
|
|
3077
|
+
*
|
|
3078
|
+
* @returns {Promise<string[]>} Promise which resolves to array of AdGuard syntax rules.
|
|
3079
|
+
*/
|
|
3080
|
+
const prepareAdgRules = async function (filterDir, filterName, lines, excluded) {
|
|
3081
|
+
const resolvedIncludes = await getResolvedPreprocessorIncludes(filterDir, filterName, lines);
|
|
3082
|
+
return convertRulesToAdgSyntax(resolvedIncludes, excluded);
|
|
3083
|
+
};
|
|
3084
|
+
|
|
3085
|
+
/**
|
|
3086
|
+
* Checks whether the line contain `Diff-Path` header tag.
|
|
3087
|
+
*
|
|
3088
|
+
* `Diff-Path` header tag should be removed from the rules
|
|
3089
|
+
* because some third-party filters may contain it
|
|
3090
|
+
* but @adguard/diff-builder does not support third-party patches.
|
|
3091
|
+
* Anyway the `Diff-Path` header tag should be added by the diff-builder during patch building
|
|
3092
|
+
* and should not be present in the filter file.
|
|
3093
|
+
*
|
|
3094
|
+
* @param {string} line Line to check.
|
|
3095
|
+
*
|
|
3096
|
+
* @returns {boolean} `true` if the line contains `Diff-Path` header tag, `false` otherwise.
|
|
3097
|
+
*/
|
|
3098
|
+
const isDiffPathHeaderTag = (line) => {
|
|
3099
|
+
// non-comment lines should be kept
|
|
3100
|
+
if (
|
|
3101
|
+
!line.startsWith(RuleMasks.MASK_COMMENT)
|
|
3102
|
+
// ubo supports `#` for comments
|
|
3103
|
+
&& !line.startsWith(RuleMasks.MASK_HOST_FILE_COMMENT)
|
|
3104
|
+
) {
|
|
3105
|
+
return false;
|
|
3106
|
+
}
|
|
3107
|
+
|
|
3108
|
+
return line.includes(DIFF_PATH_TAG);
|
|
3109
|
+
};
|
|
3110
|
+
|
|
3111
|
+
/**
|
|
3112
|
+
* @typedef {object} CompileResult
|
|
3113
|
+
* @property {string[]} lines Compiled rules.
|
|
3114
|
+
* @property {string[]} excluded Excluded rules.
|
|
3115
|
+
* @property {string[]} invalid Invalid rules.
|
|
3116
|
+
*/
|
|
3117
|
+
|
|
3118
|
+
/**
|
|
3119
|
+
* Compiles filter lines.
|
|
3120
|
+
*
|
|
3121
|
+
* @param filterDir Filter directory.
|
|
3122
|
+
* @param filterName Filter name.
|
|
3123
|
+
* @param templateContent Content of template.txt file.
|
|
3124
|
+
* @param trustLevelSettings Trust level settings for the filter.
|
|
3125
|
+
*
|
|
3126
|
+
* @returns {Promise<CompileResult>} Promise which resolves to compiled data for the filter.
|
|
3127
|
+
*/
|
|
3128
|
+
const compile$1 = async function (filterDir, filterName, templateContent, trustLevelSettings) {
|
|
3129
|
+
let result = [];
|
|
3130
|
+
const excluded = [];
|
|
3131
|
+
|
|
3132
|
+
// collect invalid rules for report
|
|
3133
|
+
// https://github.com/AdguardTeam/FiltersCompiler/issues/87
|
|
3134
|
+
const invalid = [];
|
|
3135
|
+
|
|
3136
|
+
const lines = splitLines(templateContent);
|
|
3137
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
3138
|
+
for (const line of lines) {
|
|
3139
|
+
if (line.startsWith(INCLUDE_DIRECTIVE)) {
|
|
3140
|
+
// eslint-disable-next-line no-await-in-loop
|
|
3141
|
+
const { includedLines, shouldIgnoreTrustLevel } = await include(filterDir, line, excluded);
|
|
3142
|
+
|
|
3143
|
+
let includedRules = [];
|
|
3144
|
+
|
|
3145
|
+
includedLines.forEach((line) => includedRules.push(line.trim()));
|
|
3146
|
+
|
|
3147
|
+
// eslint-disable-next-line no-await-in-loop
|
|
3148
|
+
includedRules = await prepareAdgRules(filterDir, filterName, includedRules, excluded);
|
|
3149
|
+
|
|
3150
|
+
if (shouldIgnoreTrustLevel) {
|
|
3151
|
+
logger.info(`Ignoring trust level for ${filterName} due to @include directive: ${line}`);
|
|
3152
|
+
} else {
|
|
3153
|
+
logger.info(`Applying trust-level exclusions to ${filterName} @include directive: ${line}`);
|
|
3154
|
+
includedRules = exclude(includedRules, trustLevelSettings, excluded);
|
|
3155
|
+
}
|
|
3156
|
+
|
|
3157
|
+
// 'for' loop is used in purpose instead of spread operator
|
|
3158
|
+
// to avoid 'Maximum call stack size exceeded' error on large number of rules
|
|
3159
|
+
for (let i = 0; i < includedRules.length; i += 1) {
|
|
3160
|
+
result.push(includedRules[i]);
|
|
3161
|
+
}
|
|
3162
|
+
} else {
|
|
3163
|
+
let inlineRules = [line.trim()];
|
|
3164
|
+
|
|
3165
|
+
// eslint-disable-next-line no-await-in-loop
|
|
3166
|
+
inlineRules = await prepareAdgRules(filterDir, filterName, inlineRules, excluded);
|
|
3167
|
+
|
|
3168
|
+
logger.info('Applying trust-level exclusions to inline template.txt rules...');
|
|
3169
|
+
inlineRules = exclude(inlineRules, trustLevelSettings, excluded);
|
|
3170
|
+
|
|
3171
|
+
// 'for' loop is used in purpose instead of spread operator
|
|
3172
|
+
// to avoid 'Maximum call stack size exceeded' error on large number of rules
|
|
3173
|
+
for (let i = 0; i < inlineRules.length; i += 1) {
|
|
3174
|
+
result.push(inlineRules[i]);
|
|
3175
|
+
}
|
|
3176
|
+
}
|
|
3177
|
+
}
|
|
3178
|
+
|
|
3179
|
+
result = result.filter((line) => !isDiffPathHeaderTag(line));
|
|
3180
|
+
|
|
3181
|
+
const excludeFilePath = path.join(filterDir, EXCLUDE_FILE);
|
|
3182
|
+
result = exclude(result, excludeFilePath, excluded);
|
|
3183
|
+
|
|
3184
|
+
result = validateAndFilterRules(result, excluded, invalid, filterName);
|
|
3185
|
+
|
|
3186
|
+
return {
|
|
3187
|
+
lines: result,
|
|
3188
|
+
excluded,
|
|
3189
|
+
invalid,
|
|
3190
|
+
};
|
|
3191
|
+
};
|
|
3192
|
+
|
|
3193
|
+
/**
|
|
3194
|
+
* Creates revision object,
|
|
3195
|
+
* doesn't increment version if hash is not changed
|
|
3196
|
+
*
|
|
3197
|
+
* @param {string} path - The path to the revision file.
|
|
3198
|
+
* @param {string} hash - The hash of the filter file.
|
|
3199
|
+
* @returns {{version: string, timeUpdated: number}}
|
|
3200
|
+
*/
|
|
3201
|
+
const makeRevision = function (path, hash) {
|
|
3202
|
+
const result = {
|
|
3203
|
+
version: '1.0.0.0',
|
|
3204
|
+
timeUpdated: new Date().getTime(),
|
|
3205
|
+
hash,
|
|
3206
|
+
};
|
|
3207
|
+
|
|
3208
|
+
const current = readFile$1(path);
|
|
3209
|
+
if (current) {
|
|
3210
|
+
const currentRevision = JSON.parse(current);
|
|
3211
|
+
if (currentRevision.version) {
|
|
3212
|
+
result.version = currentRevision.version;
|
|
3213
|
+
|
|
3214
|
+
if (currentRevision.timeUpdated) {
|
|
3215
|
+
result.timeUpdated = currentRevision.timeUpdated;
|
|
3216
|
+
}
|
|
3217
|
+
|
|
3218
|
+
if (!currentRevision.hash || currentRevision.hash !== result.hash) {
|
|
3219
|
+
result.version = version.increment(currentRevision.version);
|
|
3220
|
+
result.timeUpdated = new Date().getTime();
|
|
3221
|
+
}
|
|
3222
|
+
}
|
|
3223
|
+
}
|
|
3224
|
+
|
|
3225
|
+
return result;
|
|
3226
|
+
};
|
|
3227
|
+
|
|
3228
|
+
/**
|
|
3229
|
+
* Builds filter txt file from directory contents
|
|
3230
|
+
*
|
|
3231
|
+
* @param {string} filterDir - The path to the directory containing filters.
|
|
3232
|
+
* @param {Array<number>} whitelist - An array whitelist filters IDs.
|
|
3233
|
+
* @param {Array<number>} blacklist - An array blacklist filters IDs.
|
|
3234
|
+
* @returns {Promise<void>} A promise that resolves when all filters and its subdirectories have been processed.
|
|
3235
|
+
*/
|
|
3236
|
+
const buildFilter = async function (filterDir, whitelist, blacklist) {
|
|
3237
|
+
const templateContent = readFile$1(path.join(filterDir, TEMPLATE_FILE));
|
|
3238
|
+
if (!templateContent) {
|
|
3239
|
+
throw new Error('Invalid template');
|
|
3240
|
+
}
|
|
3241
|
+
|
|
3242
|
+
const metadata = JSON.parse(readFile$1(path.join(filterDir, METADATA_FILE)));
|
|
3243
|
+
|
|
3244
|
+
const { filterId } = metadata;
|
|
3245
|
+
|
|
3246
|
+
if (whitelist && whitelist.length > 0 && whitelist.indexOf(filterId) < 0) {
|
|
3247
|
+
logger.info(`Filter ${filterId} skipped due to '--include' option`);
|
|
3248
|
+
return;
|
|
3249
|
+
}
|
|
3250
|
+
|
|
3251
|
+
if (blacklist && blacklist.length > 0 && blacklist.indexOf(filterId) >= 0) {
|
|
3252
|
+
logger.info(`Filter ${filterId} skipped due to '--skip' option`);
|
|
3253
|
+
return;
|
|
3254
|
+
}
|
|
3255
|
+
|
|
3256
|
+
// check whether the filter is disabled after other checks to avoid unnecessary logging
|
|
3257
|
+
if (metadata.disabled) {
|
|
3258
|
+
logger.warn(`Filter ${filterId} skipped as disabled`);
|
|
3259
|
+
skipFilter(metadata);
|
|
3260
|
+
return;
|
|
3261
|
+
}
|
|
3262
|
+
|
|
3263
|
+
const trustLevel = metadata.trustLevel ? metadata.trustLevel : DEFAULT_TRUST_LEVEL;
|
|
3264
|
+
// eslint-disable-next-line no-undef
|
|
3265
|
+
const trustLevelSettings = path.resolve(__dirname$3, TRUST_LEVEL_DIR, `exclusions-${trustLevel}.txt`);
|
|
3266
|
+
|
|
3267
|
+
const { name: filterName } = metadata;
|
|
3268
|
+
logger.info(`Compiling ${filterName}`);
|
|
3269
|
+
const result = await compile$1(filterDir, filterName, templateContent, trustLevelSettings);
|
|
3270
|
+
|
|
3271
|
+
if (!checkAffinityDirectives(result.lines)) {
|
|
3272
|
+
throw new Error(`Error validating !#safari_cb_affinity directive in filter ${filterId}`);
|
|
3273
|
+
}
|
|
3274
|
+
|
|
3275
|
+
const compiled = result.lines;
|
|
3276
|
+
const { excluded, invalid } = result;
|
|
3277
|
+
|
|
3278
|
+
addFilter(metadata, result, invalid);
|
|
3279
|
+
logger.info(`Compiled length:${compiled.length}`);
|
|
3280
|
+
logger.info(`Excluded length:${excluded.length}`);
|
|
3281
|
+
|
|
3282
|
+
const compiledData = compiled.join('\r\n');
|
|
3283
|
+
|
|
3284
|
+
logger.info(`Writing filter file, lines:${compiled.length}`);
|
|
3285
|
+
writeFile(path.join(filterDir, FILTER_FILE), compiledData);
|
|
3286
|
+
logger.info(`Writing excluded file, lines:${excluded.length}`);
|
|
3287
|
+
writeFile(path.join(filterDir, EXCLUDED_LINES_FILE), excluded.join('\r\n'));
|
|
3288
|
+
logger.info('Writing revision file..');
|
|
3289
|
+
|
|
3290
|
+
// eslint-disable-next-line no-buffer-constructor
|
|
3291
|
+
const hash = Buffer.from(md5(compiledData, { asString: true })).toString('base64').trim();
|
|
3292
|
+
const revisionFile = path.join(filterDir, REVISION_FILE);
|
|
3293
|
+
const revision = makeRevision(revisionFile, hash);
|
|
3294
|
+
writeFile(revisionFile, JSON.stringify(revision, null, '\t'));
|
|
3295
|
+
};
|
|
3296
|
+
|
|
3297
|
+
/**
|
|
3298
|
+
* Asynchronously parses a directory and processes filters based on the provided whitelist and blacklist.
|
|
3299
|
+
*
|
|
3300
|
+
* @param {string} filtersDir - The path to the directory containing filters.
|
|
3301
|
+
* @param {Array<number>} whitelist - An array whitelist filters IDs.
|
|
3302
|
+
* @param {Array<number>} blacklist - An array blacklist filters IDs.
|
|
3303
|
+
* @returns {Promise<void>} A promise that resolves when all filters and its subdirectories have been processed.
|
|
3304
|
+
*/
|
|
3305
|
+
const parseDirectory = async function (filtersDir, whitelist, blacklist) {
|
|
3306
|
+
const items = fs.readdirSync(filtersDir)
|
|
3307
|
+
.sort((a, b) => getFilterIdFromDirName(a) - getFilterIdFromDirName(b));
|
|
3308
|
+
|
|
3309
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
3310
|
+
for (const directory of items) {
|
|
3311
|
+
const filterDir = path.join(filtersDir, directory);
|
|
3312
|
+
if (fs.lstatSync(filterDir).isDirectory()) {
|
|
3313
|
+
const template = path.join(filterDir, TEMPLATE_FILE);
|
|
3314
|
+
if (fs.existsSync(template)) {
|
|
3315
|
+
logger.info(`Building filter ${directory}...`);
|
|
3316
|
+
// eslint-disable-next-line no-await-in-loop
|
|
3317
|
+
await buildFilter(filterDir, whitelist, blacklist);
|
|
3318
|
+
logger.info(`Filter ${directory} ok`);
|
|
3319
|
+
} else {
|
|
3320
|
+
// eslint-disable-next-line no-await-in-loop
|
|
3321
|
+
await parseDirectory(filterDir, whitelist, blacklist);
|
|
3322
|
+
}
|
|
3323
|
+
}
|
|
3324
|
+
}
|
|
3325
|
+
};
|
|
3326
|
+
|
|
3327
|
+
/**
|
|
3328
|
+
* Asynchronously builds and processes filter files, generates platform data, and creates a report.
|
|
3329
|
+
*
|
|
3330
|
+
* @async
|
|
3331
|
+
* @function build
|
|
3332
|
+
* @param {string} filtersDir - The directory containing filter files to be processed.
|
|
3333
|
+
* @param {string} logFile - The path to the log file where logs will be written.
|
|
3334
|
+
* @param {string} reportFile - The path to the report file to be created.
|
|
3335
|
+
* @param {string} platformsPath - The path where platform data will be generated.
|
|
3336
|
+
* @param {Object} platformsConfig - The configuration object for platforms.
|
|
3337
|
+
* @param {Array<number>} whitelist - A list of filter file names to include in processing.
|
|
3338
|
+
* @param {Array<number>} blacklist - A list of filter file names to exclude from processing.
|
|
3339
|
+
* @returns {Promise<void>} A promise that resolves when the build process is complete.
|
|
3340
|
+
*/
|
|
3341
|
+
const build = async (
|
|
3342
|
+
filtersDir,
|
|
3343
|
+
logFile,
|
|
3344
|
+
reportFile,
|
|
3345
|
+
platformsPath,
|
|
3346
|
+
platformsConfig,
|
|
3347
|
+
whitelist,
|
|
3348
|
+
blacklist,
|
|
3349
|
+
) => {
|
|
3350
|
+
logger.initialize(logFile);
|
|
3351
|
+
init(FILTER_FILE, METADATA_FILE, REVISION_FILE, platformsConfig, ADGUARD_FILTERS_SERVER_URL);
|
|
3352
|
+
|
|
3353
|
+
await parseDirectory(filtersDir, whitelist, blacklist);
|
|
3354
|
+
|
|
3355
|
+
logger.info('Generating platforms');
|
|
3356
|
+
await generate(filtersDir, platformsPath, whitelist, blacklist);
|
|
3357
|
+
logger.info('Generating platforms done');
|
|
3358
|
+
create(reportFile);
|
|
3359
|
+
};
|
|
3360
|
+
|
|
3361
|
+
/* eslint-disable global-require */
|
|
3362
|
+
|
|
3363
|
+
|
|
3364
|
+
const SCHEMA_EXTENSION = '.schema.json';
|
|
3365
|
+
const OLD_MAC_V1_PLATFORM = 'mac';
|
|
3366
|
+
const OLD_MAC_V2_PLATFORM = 'mac_v2';
|
|
3367
|
+
|
|
3368
|
+
/**
|
|
3369
|
+
* Loads all available schemas from dir
|
|
3370
|
+
*
|
|
3371
|
+
* @param {string} dir - The directory path containing the schema files.
|
|
3372
|
+
* @returns {Object} An object with keys - schema file names and values - parsed JSON schema objects.
|
|
3373
|
+
*/
|
|
3374
|
+
const loadSchemas = (dir) => {
|
|
3375
|
+
const schemas = {};
|
|
3376
|
+
|
|
3377
|
+
const items = fs.readdirSync(dir);
|
|
3378
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
3379
|
+
for (const f of items) {
|
|
3380
|
+
if (f.endsWith(SCHEMA_EXTENSION)) {
|
|
3381
|
+
const validationFileName = f.substr(0, f.indexOf(SCHEMA_EXTENSION));
|
|
3382
|
+
|
|
3383
|
+
logger.info(`Loading schema for ${validationFileName}`);
|
|
3384
|
+
schemas[validationFileName] = JSON.parse(fs.readFileSync(path.join(dir, f)));
|
|
3385
|
+
}
|
|
3386
|
+
}
|
|
3387
|
+
|
|
3388
|
+
return schemas;
|
|
3389
|
+
};
|
|
3390
|
+
|
|
3391
|
+
/**
|
|
3392
|
+
* Recursively validates dir content with provided schemas
|
|
3393
|
+
*
|
|
3394
|
+
* @param dir
|
|
3395
|
+
* @param validator
|
|
3396
|
+
* @param schemas
|
|
3397
|
+
* @param oldSchemas
|
|
3398
|
+
* @param filtersRequiredAmount
|
|
3399
|
+
* @returns {boolean}
|
|
3400
|
+
*/
|
|
3401
|
+
const validateDir = (dir, validator, schemas, oldSchemas, filtersRequiredAmount) => {
|
|
3402
|
+
let items;
|
|
3403
|
+
try {
|
|
3404
|
+
items = fs.readdirSync(dir);
|
|
3405
|
+
} catch (e) {
|
|
3406
|
+
logger.info(e.message);
|
|
3407
|
+
return false;
|
|
3408
|
+
}
|
|
3409
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
3410
|
+
for (const f of items) {
|
|
3411
|
+
const item = path.join(dir, f);
|
|
3412
|
+
if (fs.lstatSync(item).isDirectory()) {
|
|
3413
|
+
if (!validateDir(item, validator, schemas, oldSchemas)) {
|
|
3414
|
+
return false;
|
|
3415
|
+
}
|
|
3416
|
+
} else {
|
|
3417
|
+
const fileName = path.basename(item, '.json');
|
|
3418
|
+
let schema = schemas[fileName];
|
|
3419
|
+
|
|
3420
|
+
// Validate `mac` (mac v1) dir with old schemas
|
|
3421
|
+
if (path.basename(path.dirname(item)) === OLD_MAC_V1_PLATFORM) {
|
|
3422
|
+
logger.info('Look up old schemas for mac directory');
|
|
3423
|
+
schema = oldSchemas[OLD_MAC_V1_PLATFORM][fileName];
|
|
3424
|
+
}
|
|
3425
|
+
|
|
3426
|
+
// Validate `mac_v2` dir with old schemas
|
|
3427
|
+
if (path.basename(path.dirname(item)) === OLD_MAC_V2_PLATFORM) {
|
|
3428
|
+
logger.info('Look up old schemas for mac_v2 directory');
|
|
3429
|
+
schema = oldSchemas[OLD_MAC_V2_PLATFORM][fileName];
|
|
3430
|
+
}
|
|
3431
|
+
|
|
3432
|
+
if (schema) {
|
|
3433
|
+
logger.info(`Validating ${item}`);
|
|
3434
|
+
|
|
3435
|
+
const json = JSON.parse(fs.readFileSync(item));
|
|
3436
|
+
|
|
3437
|
+
// Validate filters amount
|
|
3438
|
+
if (fileName === 'filters') {
|
|
3439
|
+
if (json.filters.length < filtersRequiredAmount) {
|
|
3440
|
+
logger.error(`Invalid filters amount in ${item}`);
|
|
3441
|
+
return false;
|
|
3442
|
+
}
|
|
3443
|
+
}
|
|
3444
|
+
|
|
3445
|
+
const validate = validator.compile(schema);
|
|
3446
|
+
const valid = validate(json);
|
|
3447
|
+
|
|
3448
|
+
// json can be updated with default values
|
|
3449
|
+
fs.writeFileSync(item, JSON.stringify(json, null, '\t'));
|
|
3450
|
+
|
|
3451
|
+
// duplicate to .js file as well
|
|
3452
|
+
const jsFileName = `${fileName}.js`;
|
|
3453
|
+
fs.writeFileSync(
|
|
3454
|
+
path.join(path.dirname(item), jsFileName),
|
|
3455
|
+
JSON.stringify(json, null, '\t'),
|
|
3456
|
+
);
|
|
3457
|
+
|
|
3458
|
+
if (!valid) {
|
|
3459
|
+
logger.error(`Invalid json in ${item}, errors:`);
|
|
3460
|
+
logger.error(validate.errors);
|
|
3461
|
+
return false;
|
|
3462
|
+
}
|
|
3463
|
+
}
|
|
3464
|
+
}
|
|
3465
|
+
}
|
|
3466
|
+
|
|
3467
|
+
return true;
|
|
3468
|
+
};
|
|
3469
|
+
|
|
3470
|
+
/**
|
|
3471
|
+
* Validates json schemas for all the filters.json and filters_i18n.json found in platforms path
|
|
3472
|
+
*
|
|
3473
|
+
* @param platformsPath - Path to platforms folder
|
|
3474
|
+
* @param jsonSchemasConfigDir - Path to json schemas config folder
|
|
3475
|
+
* @param filtersRequiredAmount - Minimum required amount of filters
|
|
3476
|
+
*/
|
|
3477
|
+
const validate$1 = (platformsPath, jsonSchemasConfigDir, filtersRequiredAmount) => {
|
|
3478
|
+
logger.info('Validating json schemas for platforms');
|
|
3479
|
+
|
|
3480
|
+
const schemas = loadSchemas(jsonSchemasConfigDir);
|
|
3481
|
+
|
|
3482
|
+
const oldSchemasMacV1 = loadSchemas(path.join(jsonSchemasConfigDir, OLD_MAC_V1_PLATFORM));
|
|
3483
|
+
const oldSchemasMacV2 = loadSchemas(path.join(jsonSchemasConfigDir, OLD_MAC_V2_PLATFORM));
|
|
3484
|
+
const oldSchemas = {
|
|
3485
|
+
[OLD_MAC_V1_PLATFORM]: oldSchemasMacV1,
|
|
3486
|
+
[OLD_MAC_V2_PLATFORM]: oldSchemasMacV2,
|
|
3487
|
+
};
|
|
3488
|
+
|
|
3489
|
+
const ajv = new Ajv({
|
|
3490
|
+
allErrors: true,
|
|
3491
|
+
useDefaults: true,
|
|
3492
|
+
});
|
|
3493
|
+
|
|
3494
|
+
const result = validateDir(platformsPath, ajv, schemas, oldSchemas, filtersRequiredAmount);
|
|
3495
|
+
|
|
3496
|
+
logger.info('Validating json schemas for platforms - done');
|
|
3497
|
+
logger.info(`Validation result: ${result}`);
|
|
3498
|
+
|
|
3499
|
+
return result;
|
|
3500
|
+
};
|
|
3501
|
+
|
|
3502
|
+
const schemaValidator = { validate: validate$1 };
|
|
3503
|
+
|
|
3504
|
+
/* eslint-disable global-require */
|
|
3505
|
+
|
|
3506
|
+
const __dirname$2 = path.dirname(new URL((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('index.cjs', document.baseURI).href))).pathname);
|
|
3507
|
+
|
|
3508
|
+
/**
|
|
3509
|
+
* Each filter, group, tag should have two keys.
|
|
3510
|
+
*/
|
|
3511
|
+
const REQUIRED_ENDINGS = [
|
|
3512
|
+
'name',
|
|
3513
|
+
'description',
|
|
3514
|
+
];
|
|
3515
|
+
|
|
3516
|
+
const LOCALES_FILE_EXTENSION = '.json';
|
|
3517
|
+
const BASE_LOCALE = 'en';
|
|
3518
|
+
|
|
3519
|
+
const REQUIRED_FILES = [
|
|
3520
|
+
'filters',
|
|
3521
|
+
'groups',
|
|
3522
|
+
'tags',
|
|
3523
|
+
].map((el) => `${el}${LOCALES_FILE_EXTENSION}`);
|
|
3524
|
+
|
|
3525
|
+
// each message key should consist of three parts
|
|
3526
|
+
// e.g. 'filter.3.name' or 'tag.29.description'
|
|
3527
|
+
const MESSAGE_KEY_NAME_PARTS_COUNT = 3;
|
|
3528
|
+
|
|
3529
|
+
const WARNING_REASONS = {
|
|
3530
|
+
MISSED_FILES: 'missed files',
|
|
3531
|
+
NO_MESSAGES: 'empty file or no messages in file',
|
|
3532
|
+
INVALID_DATA_OBJ: 'invalid or absent message key/value',
|
|
3533
|
+
};
|
|
3534
|
+
|
|
3535
|
+
const WARNING_TYPES = {
|
|
3536
|
+
CRITICAL: 'critical',
|
|
3537
|
+
LOW: 'low',
|
|
3538
|
+
};
|
|
3539
|
+
|
|
3540
|
+
/**
|
|
3541
|
+
* Sync reads file content
|
|
3542
|
+
* @param filePath - path to locales file
|
|
3543
|
+
*/
|
|
3544
|
+
const readFile = (filePath) => fs.readFileSync(path.resolve(__dirname$2, filePath), 'utf8');
|
|
3545
|
+
|
|
3546
|
+
/**
|
|
3547
|
+
* Sync reads directory content
|
|
3548
|
+
* @param dirPath - path to directory
|
|
3549
|
+
*/
|
|
3550
|
+
const readDir = (dirPath) => fs.readdirSync(path.resolve(__dirname$2, dirPath), 'utf8');
|
|
3551
|
+
|
|
3552
|
+
/**
|
|
3553
|
+
* Validates messages keys
|
|
3554
|
+
* @param {Array} keys locale messages keys
|
|
3555
|
+
* @param {string} id filters / groups / tags
|
|
3556
|
+
*/
|
|
3557
|
+
const areValidMessagesKeys = (keys, id) => {
|
|
3558
|
+
if (keys.length !== REQUIRED_ENDINGS.length) {
|
|
3559
|
+
return false;
|
|
3560
|
+
}
|
|
3561
|
+
const areValidKeys = !keys
|
|
3562
|
+
.find((key) => {
|
|
3563
|
+
const keyNameParts = key.split('.');
|
|
3564
|
+
const propPrefix = id.slice(0, -1);
|
|
3565
|
+
const filterId = Number(keyNameParts[1]);
|
|
3566
|
+
return keyNameParts.length !== MESSAGE_KEY_NAME_PARTS_COUNT
|
|
3567
|
+
|| keyNameParts[0] !== propPrefix
|
|
3568
|
+
|| !(Number.isInteger(filterId))
|
|
3569
|
+
|| !(filterId > 0)
|
|
3570
|
+
|| !(REQUIRED_ENDINGS.includes(keyNameParts[2]));
|
|
3571
|
+
});
|
|
3572
|
+
return areValidKeys;
|
|
3573
|
+
};
|
|
3574
|
+
|
|
3575
|
+
/**
|
|
3576
|
+
* Validates locale messages values
|
|
3577
|
+
* @param {string[]} values
|
|
3578
|
+
*/
|
|
3579
|
+
const areValidMessagesValues = (values) => values.every((v) => v !== '');
|
|
3580
|
+
|
|
3581
|
+
/**
|
|
3582
|
+
* Prepares invalid locales data object for results
|
|
3583
|
+
* @param {Object} obj iterable locales messages object
|
|
3584
|
+
* @returns {Array}
|
|
3585
|
+
*/
|
|
3586
|
+
const prepareWarningDetails = (obj) => Object.entries(obj).map(([key, value]) => `"${key}": "${value}"`);
|
|
3587
|
+
|
|
3588
|
+
/**
|
|
3589
|
+
* Returns map of base locale keys
|
|
3590
|
+
* @param dirPath
|
|
3591
|
+
*/
|
|
3592
|
+
const getBaseLocaleKeys = (dirPath) => {
|
|
3593
|
+
const baseLocaleKeys = {};
|
|
3594
|
+
|
|
3595
|
+
const baseLocalePath = path.join(dirPath, BASE_LOCALE);
|
|
3596
|
+
const baseLocaleFiles = readDir(baseLocalePath);
|
|
3597
|
+
|
|
3598
|
+
baseLocaleFiles.forEach((fileName) => {
|
|
3599
|
+
const baseLocaleData = JSON.parse(readFile(path.join(baseLocalePath, fileName)));
|
|
3600
|
+
baseLocaleKeys[fileName] = baseLocaleData.flatMap((entry) => Object.keys(entry));
|
|
3601
|
+
});
|
|
3602
|
+
return baseLocaleKeys;
|
|
3603
|
+
};
|
|
3604
|
+
|
|
3605
|
+
/**
|
|
3606
|
+
* Compares messagesData keys to base locale keys
|
|
3607
|
+
* @param baseLocaleKeys
|
|
3608
|
+
* @param messagesData
|
|
3609
|
+
* @param localeWarnings
|
|
3610
|
+
*/
|
|
3611
|
+
const compareKeys = (baseLocaleKeys, messagesData, localeWarnings) => {
|
|
3612
|
+
const messagesDataKeys = messagesData.flatMap((entry) => Object.keys(entry));
|
|
3613
|
+
|
|
3614
|
+
baseLocaleKeys.forEach((entry) => {
|
|
3615
|
+
if (!messagesDataKeys.includes(entry)) {
|
|
3616
|
+
localeWarnings.push([
|
|
3617
|
+
WARNING_TYPES.CRITICAL,
|
|
3618
|
+
WARNING_REASONS.INVALID_DATA_OBJ,
|
|
3619
|
+
[entry],
|
|
3620
|
+
]);
|
|
3621
|
+
}
|
|
3622
|
+
});
|
|
3623
|
+
};
|
|
3624
|
+
|
|
3625
|
+
/**
|
|
3626
|
+
* Prepares raw warnings for results
|
|
3627
|
+
* @param {Array[]} warnings collected raw warnings
|
|
3628
|
+
* @returns {Warning[]}
|
|
3629
|
+
*/
|
|
3630
|
+
const prepareWarnings = (warnings) => warnings.map(([type, reason, details]) => ({ type, reason, details }));
|
|
3631
|
+
|
|
3632
|
+
/**
|
|
3633
|
+
* @typedef {Object} Warning
|
|
3634
|
+
* @property {string} type
|
|
3635
|
+
* @property {string} reason
|
|
3636
|
+
* @property {string[]} details
|
|
3637
|
+
*/
|
|
3638
|
+
|
|
3639
|
+
/**
|
|
3640
|
+
* @typedef {Object} Result
|
|
3641
|
+
* @property {string} locale
|
|
3642
|
+
* @property {Warning[]} warnings
|
|
3643
|
+
*/
|
|
3644
|
+
|
|
3645
|
+
/**
|
|
3646
|
+
* Logs collected results of locales validation
|
|
3647
|
+
* @param {Result[]} results
|
|
3648
|
+
* @returns {string}
|
|
3649
|
+
*/
|
|
3650
|
+
const createLog = (results) => {
|
|
3651
|
+
const log = [];
|
|
3652
|
+
log.push('There are issues with:');
|
|
3653
|
+
results.forEach((res) => {
|
|
3654
|
+
log.push(`- ${res.locale}:`);
|
|
3655
|
+
res.warnings.forEach((warning) => {
|
|
3656
|
+
log.push(` - ${warning.type} priority - ${warning.reason}:`);
|
|
3657
|
+
warning.details.forEach((detail) => {
|
|
3658
|
+
log.push(` ${detail}`);
|
|
3659
|
+
});
|
|
3660
|
+
});
|
|
3661
|
+
});
|
|
3662
|
+
return log.join('\n');
|
|
3663
|
+
};
|
|
3664
|
+
|
|
3665
|
+
/**
|
|
3666
|
+
* @typedef {Object} ValidationResult
|
|
3667
|
+
* @property {boolean} ok
|
|
3668
|
+
* @property {Result[]} data
|
|
3669
|
+
* @property {string} log
|
|
3670
|
+
*/
|
|
3671
|
+
|
|
3672
|
+
/**
|
|
3673
|
+
* Validates locales messages
|
|
3674
|
+
* @param {string} dirPath relative path to locales directory
|
|
3675
|
+
* @returns {ValidationResult}
|
|
3676
|
+
*/
|
|
3677
|
+
const validate = (dirPath, requiredLocales) => {
|
|
3678
|
+
logger.info('Validating locales...');
|
|
3679
|
+
const results = [];
|
|
3680
|
+
let locales;
|
|
3681
|
+
try {
|
|
3682
|
+
locales = readDir(dirPath);
|
|
3683
|
+
} catch (e) {
|
|
3684
|
+
throw new Error(`There is no locales dir '${dirPath}'`);
|
|
3685
|
+
}
|
|
3686
|
+
|
|
3687
|
+
if (locales.length === 0) {
|
|
3688
|
+
throw new Error(`Locales dir '${dirPath}' is empty`);
|
|
3689
|
+
}
|
|
3690
|
+
|
|
3691
|
+
const baseLocaleKeysMap = getBaseLocaleKeys(dirPath);
|
|
3692
|
+
|
|
3693
|
+
locales.forEach((locale) => {
|
|
3694
|
+
const localeWarnings = [];
|
|
3695
|
+
const filesList = readDir(path.join(dirPath, locale));
|
|
3696
|
+
// checks all needed files presence
|
|
3697
|
+
const missedFiles = REQUIRED_FILES
|
|
3698
|
+
.filter((el) => !filesList.includes(el));
|
|
3699
|
+
if (missedFiles.length !== 0) {
|
|
3700
|
+
localeWarnings.push([
|
|
3701
|
+
// if there are missedFiles, we consider it's critical
|
|
3702
|
+
WARNING_TYPES.CRITICAL,
|
|
3703
|
+
WARNING_REASONS.MISSED_FILES,
|
|
3704
|
+
missedFiles,
|
|
3705
|
+
]);
|
|
3706
|
+
}
|
|
3707
|
+
|
|
3708
|
+
const presentFiles = REQUIRED_FILES
|
|
3709
|
+
.filter((el) => !missedFiles.includes(el));
|
|
3710
|
+
|
|
3711
|
+
// iterate over existent files
|
|
3712
|
+
presentFiles.forEach((fileName) => {
|
|
3713
|
+
const messagesPath = path.join(dirPath, locale, fileName);
|
|
3714
|
+
let messagesData;
|
|
3715
|
+
try {
|
|
3716
|
+
messagesData = JSON.parse(readFile(messagesPath));
|
|
3717
|
+
} catch (e) {
|
|
3718
|
+
localeWarnings.push([
|
|
3719
|
+
// if there is invalid data format, we consider it's critical
|
|
3720
|
+
WARNING_TYPES.CRITICAL,
|
|
3721
|
+
WARNING_REASONS.NO_MESSAGES,
|
|
3722
|
+
[fileName],
|
|
3723
|
+
]);
|
|
3724
|
+
return;
|
|
3725
|
+
}
|
|
3726
|
+
|
|
3727
|
+
if (messagesData.length === 0) {
|
|
3728
|
+
// for some locales there is no translations
|
|
3729
|
+
// so it should bt critical only for required (our) locales
|
|
3730
|
+
const warningType = requiredLocales.includes(locale)
|
|
3731
|
+
? WARNING_TYPES.CRITICAL
|
|
3732
|
+
: WARNING_TYPES.LOW;
|
|
3733
|
+
localeWarnings.push([
|
|
3734
|
+
warningType,
|
|
3735
|
+
WARNING_REASONS.NO_MESSAGES,
|
|
3736
|
+
[fileName],
|
|
3737
|
+
]);
|
|
3738
|
+
}
|
|
3739
|
+
|
|
3740
|
+
if (requiredLocales.includes(locale)) {
|
|
3741
|
+
// check if all keys from base locale are presented in messagesData
|
|
3742
|
+
compareKeys(baseLocaleKeysMap[fileName], messagesData, localeWarnings);
|
|
3743
|
+
}
|
|
3744
|
+
|
|
3745
|
+
messagesData.forEach((obj) => {
|
|
3746
|
+
const messagesKeys = Object.keys(obj);
|
|
3747
|
+
const messagesValues = Object.values(obj);
|
|
3748
|
+
const extensionLength = LOCALES_FILE_EXTENSION.length;
|
|
3749
|
+
const id = fileName.slice(0, -extensionLength);
|
|
3750
|
+
if (!areValidMessagesKeys(messagesKeys, id)
|
|
3751
|
+
|| !areValidMessagesValues(messagesValues)) {
|
|
3752
|
+
localeWarnings.push([
|
|
3753
|
+
// invalid messages data object is always critical
|
|
3754
|
+
WARNING_TYPES.CRITICAL,
|
|
3755
|
+
WARNING_REASONS.INVALID_DATA_OBJ,
|
|
3756
|
+
prepareWarningDetails(obj),
|
|
3757
|
+
]);
|
|
3758
|
+
}
|
|
3759
|
+
});
|
|
3760
|
+
});
|
|
3761
|
+
|
|
3762
|
+
if (localeWarnings.length !== 0) {
|
|
3763
|
+
const warnings = prepareWarnings(localeWarnings);
|
|
3764
|
+
results.push({ locale, warnings });
|
|
3765
|
+
}
|
|
3766
|
+
});
|
|
3767
|
+
|
|
3768
|
+
if (results.length === 0) {
|
|
3769
|
+
logger.info('Validation result: OK');
|
|
3770
|
+
return { ok: true };
|
|
3771
|
+
}
|
|
3772
|
+
|
|
3773
|
+
const isOK = !results
|
|
3774
|
+
.some((res) => {
|
|
3775
|
+
const isCriticalWarning = res.warnings
|
|
3776
|
+
.some((warning) => warning.type === WARNING_TYPES.CRITICAL);
|
|
3777
|
+
return isCriticalWarning;
|
|
3778
|
+
});
|
|
3779
|
+
const resultsLog = createLog(results);
|
|
3780
|
+
if (isOK) {
|
|
3781
|
+
logger.warn(resultsLog);
|
|
3782
|
+
} else {
|
|
3783
|
+
logger.error(resultsLog);
|
|
3784
|
+
}
|
|
3785
|
+
|
|
3786
|
+
return { ok: isOK, data: results, log: resultsLog };
|
|
3787
|
+
};
|
|
3788
|
+
|
|
3789
|
+
const localesValidator = { validate };
|
|
3790
|
+
|
|
3791
|
+
/**
|
|
3792
|
+
* @file Platforms configuration.
|
|
3793
|
+
*
|
|
3794
|
+
* It shall be overridden by custom configuration:
|
|
3795
|
+
* @see {@link https://github.com/AdguardTeam/FiltersRegistry/blob/master/scripts/build/custom_platforms.js}
|
|
3796
|
+
*
|
|
3797
|
+
* IMPORTANT: During making any changes in this file,
|
|
3798
|
+
* the custom_platforms.js should also be updated through PR on GitHub.
|
|
3799
|
+
*/
|
|
3800
|
+
|
|
3801
|
+
/**
|
|
3802
|
+
* Pattern to check if rule contains `$domain` modifier with regular expression
|
|
3803
|
+
*
|
|
3804
|
+
* In Safari, `if-domain` and `unless-domain` do not support regexps, only `*`
|
|
3805
|
+
* https://github.com/AdguardTeam/FiltersRegistry/pull/806
|
|
3806
|
+
*
|
|
3807
|
+
* @example
|
|
3808
|
+
* ```[$domain=/^inattv\d+\.pro$/]#%#//scriptlet('set-constant', 'config.adv', 'emptyObj')```
|
|
3809
|
+
*/
|
|
3810
|
+
const DOMAIN_WITH_REGEXPS_PATTERNS = [
|
|
3811
|
+
'\\$domain=\/',
|
|
3812
|
+
',domain=\/',
|
|
3813
|
+
];
|
|
3814
|
+
|
|
3815
|
+
/**
|
|
3816
|
+
* Pattern to check if rule contains `$all` modifier
|
|
3817
|
+
*
|
|
3818
|
+
* @example
|
|
3819
|
+
* ```/?t=popunder&$all```
|
|
3820
|
+
*/
|
|
3821
|
+
const ALL_MODIFIER_PATTERNS = [
|
|
3822
|
+
'\\$all',
|
|
3823
|
+
];
|
|
3824
|
+
|
|
3825
|
+
/**
|
|
3826
|
+
* Pattern to check if rule contains `$mp4` modifier
|
|
3827
|
+
*
|
|
3828
|
+
* @example
|
|
3829
|
+
* ```Deprecated, use $redirect=noopmp4-1s instead```
|
|
3830
|
+
*/
|
|
3831
|
+
const MP4_MODIFIER_PATTERNS = [
|
|
3832
|
+
'\\$(.*,)?mp4',
|
|
3833
|
+
];
|
|
3834
|
+
|
|
3835
|
+
/**
|
|
3836
|
+
* Pattern to check if rule contains `$network` modifier
|
|
3837
|
+
*
|
|
3838
|
+
* @example
|
|
3839
|
+
* ```57.128.71.215$network```
|
|
3840
|
+
*/
|
|
3841
|
+
const NETWORK_MODIFIER_PATTERNS = [
|
|
3842
|
+
'\\$network',
|
|
3843
|
+
];
|
|
3844
|
+
|
|
3845
|
+
/**
|
|
3846
|
+
* Pattern to check if rule contains `$webrtc` modifier
|
|
3847
|
+
*
|
|
3848
|
+
* @example
|
|
3849
|
+
* ```Removed and no longer supported```
|
|
3850
|
+
*/
|
|
3851
|
+
const WEBRTC_MODIFIER_PATTERNS = [
|
|
3852
|
+
'\\$webrtc',
|
|
3853
|
+
];
|
|
3854
|
+
|
|
3855
|
+
/**
|
|
3856
|
+
* Pattern to check if rule contains `$csp` modifier
|
|
3857
|
+
*
|
|
3858
|
+
* @example
|
|
3859
|
+
* ```||deloplen.com^$csp=script-src 'none'```
|
|
3860
|
+
*/
|
|
3861
|
+
const CSP_MODIFIER_PATTERNS = [
|
|
3862
|
+
'\\$csp',
|
|
3863
|
+
];
|
|
3864
|
+
|
|
3865
|
+
/**
|
|
3866
|
+
* Pattern to check if rule contains `$$` modifier
|
|
3867
|
+
*
|
|
3868
|
+
* Do not exclude scriptlets which contain '$$' when excluding '$$' and '$@$' rules
|
|
3869
|
+
* https://github.com/AdguardTeam/FiltersRegistry/issues/731
|
|
3870
|
+
*
|
|
3871
|
+
* @example
|
|
3872
|
+
* ```mail.com$$script[tag-content="uabp"][min-length="20000"][max-length="300000"]```
|
|
3873
|
+
*/
|
|
3874
|
+
const HTML_FILTERING_MODIFIER_PATTERNS = [
|
|
3875
|
+
'^((?!#%#).)*\\$\\$|\\$\\@\\$',
|
|
3876
|
+
];
|
|
3877
|
+
|
|
3878
|
+
/**
|
|
3879
|
+
* Pattern to check if rule contains `$protobuf` modifier
|
|
3880
|
+
* and does not contain non-basic modifiers like `$domain` or `$path`.
|
|
3881
|
+
*
|
|
3882
|
+
* Pattern parts:
|
|
3883
|
+
* - `\\$` — modifiers divider
|
|
3884
|
+
* - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
|
|
3885
|
+
* `[$path=...]##.textad,protobuf` and rules with `$removeparam` modifier like `$removeparam=protobuf`
|
|
3886
|
+
* - `.*protobuf` — protobuf modifier itself
|
|
3887
|
+
* - `(,|=|$)` — end of line or modifiers divider, as `$protobuf` can be followed by other modifiers (`,`),
|
|
3888
|
+
* it may have a value (`=`), or it may be the last modifier in the rule (`$`).
|
|
3889
|
+
*
|
|
3890
|
+
* Currently it is not supported, but it can be added in the future
|
|
3891
|
+
* https://github.com/AdguardTeam/CoreLibs/issues/1778
|
|
3892
|
+
*/
|
|
3893
|
+
const PROTOBUF_MODIFIER_PATTERNS = [
|
|
3894
|
+
'\\$(?!#|(path|domain)=.*]|.*removeparam=).*protobuf(,|=|$)',
|
|
3895
|
+
];
|
|
3896
|
+
|
|
3897
|
+
/**
|
|
3898
|
+
* Pattern to check if rule contains `$app` modifier
|
|
3899
|
+
* and does not contain non-basic modifiers like `$domain` or `$path`.
|
|
3900
|
+
*
|
|
3901
|
+
* Pattern parts:
|
|
3902
|
+
* - `\\$` — modifiers divider
|
|
3903
|
+
* - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
|
|
3904
|
+
* `[$path=...]##.textad,[app="ads"]` and rules with `$removeparam` modifier like `$removeparam=app=ads`
|
|
3905
|
+
* - `.*app=` — app= modifier itself
|
|
3906
|
+
*
|
|
3907
|
+
* @example
|
|
3908
|
+
* ```@@||imasdk.googleapis.com^$app=tv.htv.app```
|
|
3909
|
+
*/
|
|
3910
|
+
const APP_MODIFIER_PATTERNS = [
|
|
3911
|
+
'\\$(?!#|(path|domain)=.*]|.*removeparam=).*app=',
|
|
3912
|
+
];
|
|
3913
|
+
|
|
3914
|
+
/**
|
|
3915
|
+
* Pattern to check if rule contains `$extension` modifier
|
|
3916
|
+
* and does not contain non-basic modifiers like `$domain` or `$path`.
|
|
3917
|
+
*
|
|
3918
|
+
* Pattern parts:
|
|
3919
|
+
* - `\\$` — modifiers divider
|
|
3920
|
+
* - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
|
|
3921
|
+
* `[$path=...]##.textad,extension` and rules with `$removeparam` modifier like `$removeparam=extension`
|
|
3922
|
+
* - `.*extension` — extension modifier itself
|
|
3923
|
+
* - `(,|=|$)` — end of line or modifiers divider, as `$extension` can be followed by other modifiers (`,`),
|
|
3924
|
+
* it may have a value (`=`), or it may be the last modifier in the rule (`$`).
|
|
3925
|
+
*
|
|
3926
|
+
* @example
|
|
3927
|
+
* ```@@||radar.cloudflare.com^$elemhide,extension,content```
|
|
3928
|
+
*/
|
|
3929
|
+
const EXTENSION_MODIFIER_PATTERNS = [
|
|
3930
|
+
'\\$(?!#|(path|domain)=.*]|.*removeparam=).*extension(,|=|$)',
|
|
3931
|
+
];
|
|
3932
|
+
|
|
3933
|
+
/**
|
|
3934
|
+
* Pattern to check if rule contains only `$content` modifier.
|
|
3935
|
+
*
|
|
3936
|
+
* @example
|
|
3937
|
+
* ```@@||telegram.hr^$content```
|
|
3938
|
+
*/
|
|
3939
|
+
const ONLY_CONTENT_MODIFIER_PATTERNS = [
|
|
3940
|
+
'\\$content$',
|
|
3941
|
+
];
|
|
3942
|
+
|
|
3943
|
+
/**
|
|
3944
|
+
* Pattern to check if rule contains `$content` modifier
|
|
3945
|
+
* and does not contain non-basic modifiers like `$domain` or `$path`.
|
|
3946
|
+
*
|
|
3947
|
+
* Pattern parts:
|
|
3948
|
+
* - `\\$` — modifiers divider
|
|
3949
|
+
* - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
|
|
3950
|
+
* `[$path=...]##.textad,content` and rules with `$removeparam` modifier like `$removeparam=content`
|
|
3951
|
+
* - `.*content` — content modifier itself
|
|
3952
|
+
* - `(,|$)` — end of line or modifiers divider, as `$content` can be followed by other modifiers (`,`).
|
|
3953
|
+
*
|
|
3954
|
+
* @example
|
|
3955
|
+
* ```@@||dnsleaktest.com^$content,elemhide,jsinject```
|
|
3956
|
+
*/
|
|
3957
|
+
const CONTENT_MODIFIER_PATTERNS = [
|
|
3958
|
+
'\\$(?!#|(path|domain)=.*]|.*removeparam=).*content(,|$)',
|
|
3959
|
+
];
|
|
3960
|
+
|
|
3961
|
+
/**
|
|
3962
|
+
* Pattern to check if rule contains `$jsinject` modifier
|
|
3963
|
+
* and does not contain non-basic modifiers like `$domain` or `$path`.
|
|
3964
|
+
*
|
|
3965
|
+
* Pattern parts:
|
|
3966
|
+
* - `\\$` — modifiers divider
|
|
3967
|
+
* - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
|
|
3968
|
+
* `[$path=...]##.textad,jsinject` and rules with `$removeparam` modifier like `$removeparam=jsinject`
|
|
3969
|
+
* - `.*jsinject` — jsinject modifier itself
|
|
3970
|
+
* - `(,|$)` — end of line or modifiers divider, as `$jsinject` can be followed by other modifiers (`,`).
|
|
3971
|
+
*
|
|
3972
|
+
* @example
|
|
3973
|
+
* ```@@://www.atlassian.com^$elemhide,jsinject,extension```
|
|
3974
|
+
*/
|
|
3975
|
+
const JSINJECT_MODIFIER_PATTERNS = [
|
|
3976
|
+
'\\$(?!#|(path|domain)=.*]|.*removeparam=).*jsinject(,|$)',
|
|
3977
|
+
];
|
|
3978
|
+
|
|
3979
|
+
/**
|
|
3980
|
+
* Pattern to check if rule contains `$urlblock` modifier
|
|
3981
|
+
* and does not contain non-basic modifiers like `$domain` or `$path`.
|
|
3982
|
+
*
|
|
3983
|
+
* Pattern parts:
|
|
3984
|
+
* - `\\$` — modifiers divider
|
|
3985
|
+
* - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
|
|
3986
|
+
* `[$path=...]##.textad,urlblock` and rules with `$removeparam` modifier like `$removeparam=urlblock`
|
|
3987
|
+
* - `.*urlblock` — urlblock modifier itself
|
|
3988
|
+
* - `(,|$)` — end of line or modifiers divider, as `$urlblock` can be followed by other modifiers (`,`).
|
|
3989
|
+
*
|
|
3990
|
+
* @example
|
|
3991
|
+
* ```@@||google.com/settings/ads/onweb$urlblock```
|
|
3992
|
+
*/
|
|
3993
|
+
const URLBLOCK_MODIFIER_PATTERNS = [
|
|
3994
|
+
'\\$(?!#|(path|domain)=.*]|.*removeparam=).*urlblock(,|$)',
|
|
3995
|
+
];
|
|
3996
|
+
|
|
3997
|
+
/**
|
|
3998
|
+
* Pattern to check if rule contains `$referrerpolicy` modifier
|
|
3999
|
+
* and does not contain non-basic modifiers like `$domain` or `$path`.
|
|
4000
|
+
*
|
|
4001
|
+
* Pattern parts:
|
|
4002
|
+
* - `\\$` — modifiers divider
|
|
4003
|
+
* - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
|
|
4004
|
+
* `[$path=...]##.textad,referrerpolicy` and rules with `$removeparam` modifier like `$removeparam=referrerpolicy`
|
|
4005
|
+
* - `.*referrerpolicy` — referrerpolicy modifier itself
|
|
4006
|
+
* - `(,|=|$)` — end of line or modifiers divider, as `$referrerpolicy` can be followed by other modifiers (`,`),
|
|
4007
|
+
* it may have a value (`=`), or it may be the last modifier in the rule (`$`).
|
|
4008
|
+
*
|
|
4009
|
+
* @example
|
|
4010
|
+
* ```||yallo.tv^$referrerpolicy=origin```
|
|
4011
|
+
*/
|
|
4012
|
+
const REFERRERPOLICY_MODIFIER_PATTERNS = [
|
|
4013
|
+
'\\$(?!#|(path|domain)=.*]|.*removeparam=).*referrerpolicy(,|=|$)',
|
|
4014
|
+
];
|
|
4015
|
+
|
|
4016
|
+
/**
|
|
4017
|
+
* Pattern to check if rule contains `$replace` modifier
|
|
4018
|
+
* and does not contain non-basic modifiers like `$domain` or `$path`.
|
|
4019
|
+
*
|
|
4020
|
+
* Pattern parts:
|
|
4021
|
+
* - `\\$` — modifiers divider
|
|
4022
|
+
* - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
|
|
4023
|
+
* `[$path=...]##.textad,[replace="ads"]` and rules with `$removeparam` modifier like `$removeparam=replace=ads`
|
|
4024
|
+
* - `.*replace` — replace modifier itself
|
|
4025
|
+
* - `(,|=|$)` — end of line or modifiers divider, as `$replace` can be followed by other modifiers (`,`),
|
|
4026
|
+
* it may have a value (`=`), or it may be the last modifier in the rule (`$`).
|
|
4027
|
+
*
|
|
4028
|
+
* @example
|
|
4029
|
+
* ```||pubads.g.doubleclick.net/gampad/live/ads?correlator=$replace=/(<VAST[\s\S]*?>)[\s\S]*<\/VAST>/\$1<\/VAST>/```
|
|
4030
|
+
*/
|
|
4031
|
+
const REPLACE_MODIFIER_PATTERNS = [
|
|
4032
|
+
'\\$(?!#|(path|domain)=.*]|.*removeparam=).*replace(,|=|$)',
|
|
4033
|
+
];
|
|
4034
|
+
|
|
4035
|
+
/* eslint-disable max-len */
|
|
4036
|
+
|
|
4037
|
+
/**
|
|
4038
|
+
* Pattern to check if rule contains `$hls` modifier
|
|
4039
|
+
* and does not contain non-basic modifiers like `$domain` or `$path`.
|
|
4040
|
+
*
|
|
4041
|
+
* Pattern parts:
|
|
4042
|
+
* - `\\$` — modifiers divider
|
|
4043
|
+
* - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
|
|
4044
|
+
* `[$path=...]##.textad,[hls="ads"]` and rules with `$removeparam` modifier like `$removeparam=hls=ads`
|
|
4045
|
+
* - `.*hls` — hls modifier itself
|
|
4046
|
+
* - `(,|=|$)` — end of line or modifiers divider, as `$hls` can be followed by other modifiers (`,`),
|
|
4047
|
+
* it may have a value (`=`), or it may be the last modifier in the rule (`$`).
|
|
4048
|
+
*
|
|
4049
|
+
* @example
|
|
4050
|
+
* ```||pubads.g.doubleclick.net/ondemand/hls/*.m3u8$hls=/redirector\.googlevideo\.com\/videoplayback[\s\S]*?dclk_video_ads/,domain=10play.com.au```
|
|
4051
|
+
*/
|
|
4052
|
+
const HLS_MODIFIER_PATTERNS = [
|
|
4053
|
+
'\\$(?!#|(path|domain)=.*]|.*removeparam=).*hls(,|=|$)',
|
|
4054
|
+
];
|
|
4055
|
+
|
|
4056
|
+
/**
|
|
4057
|
+
* Pattern to check if rule contains `$jsonprune` modifier
|
|
4058
|
+
* and does not contain non-basic modifiers like `$domain` or `$path`.
|
|
4059
|
+
*
|
|
4060
|
+
* Pattern parts:
|
|
4061
|
+
* - `\\$` — modifiers divider
|
|
4062
|
+
* - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
|
|
4063
|
+
* `[$path=...]##.textad,[jsonprune="ads"]` and rules with `$removeparam` modifier like `$removeparam=jsonprune`
|
|
4064
|
+
* - `.*jsonprune` — jsonprune modifier itself
|
|
4065
|
+
* - `(,|=|$)` — end of line or modifiers divider, as `$jsonprune` can be followed by other modifiers (`,`),
|
|
4066
|
+
* it may have a value (`=`), or it may be the last modifier in the rule (`$`).
|
|
4067
|
+
*
|
|
4068
|
+
* @example
|
|
4069
|
+
* ```.com/watch?v=$xmlhttprequest,jsonprune=\$..[adPlacements\, adSlots\, playerAds],domain=youtubekids.com|youtube-nocookie.com|youtube.com```
|
|
4070
|
+
*/
|
|
4071
|
+
const JSONPRUNE_MODIFIER_PATTERNS = [
|
|
4072
|
+
'\\$(?!#|(path|domain)=.*]|.*removeparam=).*jsonprune(,|=|$)',
|
|
4073
|
+
];
|
|
4074
|
+
|
|
4075
|
+
/* eslint-enable max-len */
|
|
4076
|
+
|
|
4077
|
+
/**
|
|
4078
|
+
* Pattern to check if rule contains `$removeparam` modifier
|
|
4079
|
+
* and does not contain non-basic modifiers like `$domain` or `$path`.
|
|
4080
|
+
*
|
|
4081
|
+
* Pattern parts:
|
|
4082
|
+
* - `\\$` — modifiers divider
|
|
4083
|
+
* - `(?!#|(path|domain)=.*]).*` — negative lookahead to exclude CSS rules (#$#) and rules like
|
|
4084
|
+
* `[$path=...]##.textad,[removeparam="ads"]`
|
|
4085
|
+
* - `.*removeparam` — removeparam modifier itself
|
|
4086
|
+
* - `(,|=|$)` — end of line or modifiers divider, as `$removeparam` can be followed by other modifiers (`,`),
|
|
4087
|
+
* it may have a value (`=`), or it may be the last modifier in the rule (`$`).
|
|
4088
|
+
*
|
|
4089
|
+
* @example
|
|
4090
|
+
* ```$removeparam=fb_ref```
|
|
4091
|
+
*/
|
|
4092
|
+
const REMOVEPARAM_MODIFIER_PATTERNS = [
|
|
4093
|
+
'\\$(?!#|(path|domain)=.*]).*removeparam(,|=|$)',
|
|
4094
|
+
];
|
|
4095
|
+
|
|
4096
|
+
/**
|
|
4097
|
+
* Pattern to check if rule contains `$removeheader` modifier
|
|
4098
|
+
* and does not contain non-basic modifiers like `$domain` or `$path`.
|
|
4099
|
+
*
|
|
4100
|
+
* Pattern parts:
|
|
4101
|
+
* - `\\$` — modifiers divider
|
|
4102
|
+
* - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
|
|
4103
|
+
* `[$path=...]##.textad,[removeheader="ads"]` and rules with `$removeparam` modifier like `$removeparam=removeheader`
|
|
4104
|
+
* - `.*removeheader` — removeheader modifier itself
|
|
4105
|
+
* - `(,|=|$)` — end of line or modifiers divider, as `$removeheader` can be followed by other modifiers (`,`),
|
|
4106
|
+
* it may have a value (`=`), or it may be the last modifier in the rule (`$`).
|
|
4107
|
+
*
|
|
4108
|
+
* @example
|
|
4109
|
+
* ```||dubznetwork.com^$removeheader=refresh```
|
|
4110
|
+
*/
|
|
4111
|
+
const REMOVEHEADER_MODIFIER_PATTERNS = [
|
|
4112
|
+
'\\$(?!#|(path|domain)=.*]|.*removeparam=).*removeheader(,|=|$)',
|
|
4113
|
+
];
|
|
4114
|
+
|
|
4115
|
+
/**
|
|
4116
|
+
* Pattern to check if rule contains `$stealth` modifier
|
|
4117
|
+
* and does not contain non-basic modifiers like `$domain` or `$path`.
|
|
4118
|
+
*
|
|
4119
|
+
* Pattern parts:
|
|
4120
|
+
* - `\\$` — modifiers divider
|
|
4121
|
+
* - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
|
|
4122
|
+
* `[$path=...]##.textad,[stealth="ads"]` and rules with `$removeparam` modifier like `$removeparam=stealth`
|
|
4123
|
+
* - `.*stealth` — stealth modifier itself
|
|
4124
|
+
* - `(,|=|$)` — end of line or modifiers divider, as `$stealth` can be followed by other modifiers (`,`),
|
|
4125
|
+
* it may have a value (`=`), or it may be the last modifier in the rule (`$`).
|
|
4126
|
+
*
|
|
4127
|
+
* @example
|
|
4128
|
+
* ```@@.php?play_vid=$subdocument,stealth=referrer,domain=xyflv.cc```
|
|
4129
|
+
*/
|
|
4130
|
+
const STEALTH_MODIFIER_PATTERNS = [
|
|
4131
|
+
'\\$(?!#|(path|domain)=.*]|.*removeparam=).*stealth(,|=|$)',
|
|
4132
|
+
];
|
|
4133
|
+
|
|
4134
|
+
/**
|
|
4135
|
+
* Pattern to check if rule contains `$cookie` modifier
|
|
4136
|
+
* and does not contain non-basic modifiers like `$domain` or `$path`.
|
|
4137
|
+
*
|
|
4138
|
+
* Pattern parts:
|
|
4139
|
+
* - `\\$` — modifiers divider
|
|
4140
|
+
* - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
|
|
4141
|
+
* `[$path=...]##.textad,cookie` and rules with `$removeparam` modifier like `$removeparam=cookie`
|
|
4142
|
+
* - `.*cookie` — cookie modifier itself
|
|
4143
|
+
* - `(,|=|$)` — end of line or modifiers divider, as `$cookie` can be followed by other modifiers (`,`),
|
|
4144
|
+
* it may have a value (`=`), or it may be the last modifier in the rule (`$`).
|
|
4145
|
+
*
|
|
4146
|
+
* @example
|
|
4147
|
+
* ```$cookie=_ga```
|
|
4148
|
+
*/
|
|
4149
|
+
const COOKIE_MODIFIER_PATTERNS = [
|
|
4150
|
+
'\\$(?!#|(path|domain)=.*]|.*removeparam=).*cookie(,|=|$)',
|
|
4151
|
+
];
|
|
4152
|
+
|
|
4153
|
+
/**
|
|
4154
|
+
* Pattern to check if rule contains `$redirect` modifier
|
|
4155
|
+
* and does not contain non-basic modifiers like `$domain` or `$path`.
|
|
4156
|
+
*
|
|
4157
|
+
* Pattern parts:
|
|
4158
|
+
* - `\\$` — modifiers divider
|
|
4159
|
+
* - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
|
|
4160
|
+
* `[$path=...]##.textad,redirect` and rules with `$removeparam` modifier like `$removeparam=redirect`
|
|
4161
|
+
* - `.*redirect` — redirect modifier itself
|
|
4162
|
+
* - `(,|=|$)` — end of line or modifiers divider, as `$redirect` can be followed by other modifiers (`,`),
|
|
4163
|
+
* it may have a value (`=`), or it may be the last modifier in the rule (`$`).
|
|
4164
|
+
*
|
|
4165
|
+
* @example
|
|
4166
|
+
* ```||google-analytics.com/analytics.js$script,redirect=google-analytics,domain=~olx.*|~banki.ru|~bigc.co.th```
|
|
4167
|
+
*/
|
|
4168
|
+
const REDIRECT_MODIFIER_PATTERNS = [
|
|
4169
|
+
'\\$(?!#|(path|domain)=.*]|.*removeparam=).*redirect(,|=|$)',
|
|
4170
|
+
];
|
|
4171
|
+
|
|
4172
|
+
/**
|
|
4173
|
+
* Pattern to check if rule contains `$redirect-rule` modifier
|
|
4174
|
+
* and does not contain non-basic modifiers like `$domain` or `$path`.
|
|
4175
|
+
*
|
|
4176
|
+
* Pattern parts:
|
|
4177
|
+
* - `\\$` — modifiers divider
|
|
4178
|
+
* - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
|
|
4179
|
+
* `[$path=...]##.textad,redirect-rule` and rules with `$removeparam` modifier like `$removeparam=redirect-rule`
|
|
4180
|
+
* - `.*redirect-rule` — redirect-rule modifier itself
|
|
4181
|
+
* - `(,|=|$)` — end of line or modifiers divider, as `$redirect-rule` can be followed by other modifiers (`,`),
|
|
4182
|
+
* it may have a value (`=`), or it may be the last modifier in the rule (`$`).
|
|
4183
|
+
*
|
|
4184
|
+
* @example
|
|
4185
|
+
* ```$script,third-party,redirect-rule=noopjs,domain=paraphraser.io```
|
|
4186
|
+
*/
|
|
4187
|
+
const REDIRECT_RULE_MODIFIER_PATTERNS = [
|
|
4188
|
+
'\\$(?!#|(path|domain)=.*]|.*removeparam=).*redirect-rule(,|=|$)',
|
|
4189
|
+
];
|
|
4190
|
+
|
|
4191
|
+
/**
|
|
4192
|
+
* Pattern to check if rule contains `$empty` modifier
|
|
4193
|
+
* and does not contain non-basic modifiers like `$domain` or `$path`.
|
|
4194
|
+
*
|
|
4195
|
+
* Pattern parts:
|
|
4196
|
+
* - `\\$` — modifiers divider
|
|
4197
|
+
* - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
|
|
4198
|
+
* `[$path=...]##.textad,empty` and rules with `$removeparam` modifier like `$removeparam=empty`
|
|
4199
|
+
* - `.*empty` — empty modifier itself
|
|
4200
|
+
* - `(,|$)` — end of line or modifiers divider, as `$empty` can be followed by other modifiers (`,`).
|
|
4201
|
+
*
|
|
4202
|
+
* @example
|
|
4203
|
+
* ```Deprecated, use $redirect=nooptext instead```
|
|
4204
|
+
*/
|
|
4205
|
+
const EMPTY_MODIFIER_PATTERNS = [
|
|
4206
|
+
'\\$(?!#|(path|domain)=.*]|.*removeparam=).*empty(,|$)',
|
|
4207
|
+
];
|
|
4208
|
+
|
|
4209
|
+
/**
|
|
4210
|
+
* Pattern to detect scriptlets and JavaScript rules
|
|
4211
|
+
*
|
|
4212
|
+
* @example
|
|
4213
|
+
* ```w3resource.com#%#//scriptlet('prevent-setTimeout', 'ins.adsbygoogle')```
|
|
4214
|
+
* @example
|
|
4215
|
+
* ```meczyki.pl#%#!function(){window.YLHH={bidder:{startAuction:function(){}}};}();```
|
|
4216
|
+
*/
|
|
4217
|
+
const JAVASCRIPT_RULES_PATTERNS = [
|
|
4218
|
+
'#%#',
|
|
4219
|
+
'#@%#',
|
|
4220
|
+
];
|
|
4221
|
+
|
|
4222
|
+
/**
|
|
4223
|
+
* Pattern to detect CSS rules
|
|
4224
|
+
*
|
|
4225
|
+
* @example
|
|
4226
|
+
* ```windowslite.net#$#body { overflow: auto !important; }```
|
|
4227
|
+
*/
|
|
4228
|
+
const CSS_RULES_PATTERNS = [
|
|
4229
|
+
'#\\$#',
|
|
4230
|
+
'#@\\$#',
|
|
4231
|
+
];
|
|
4232
|
+
|
|
4233
|
+
/**
|
|
4234
|
+
* Pattern to detect CSS rules with `@media` queries
|
|
4235
|
+
*
|
|
4236
|
+
* @example
|
|
4237
|
+
* ```windowslite.net#$#body { overflow: auto !important; }```
|
|
4238
|
+
*/
|
|
4239
|
+
const CSS_MEDIA_RULES_PATTERNS = [
|
|
4240
|
+
'#\\$#@media ',
|
|
4241
|
+
];
|
|
4242
|
+
|
|
4243
|
+
/**
|
|
4244
|
+
* Patterns to match unblocking basic rules with `$important` modifier
|
|
4245
|
+
* which is not supported by uBlock Origin.
|
|
4246
|
+
*
|
|
4247
|
+
* @see {@link https://github.com/AdguardTeam/FiltersCompiler/issues/200}
|
|
4248
|
+
*/
|
|
4249
|
+
const UNBLOCKING_IMPORTANT_RULES_PATTERNS = [
|
|
4250
|
+
'@@.*?(\\$|,)important',
|
|
4251
|
+
];
|
|
4252
|
+
|
|
4253
|
+
/**
|
|
4254
|
+
* Pattern to detect Extended CSS rules
|
|
4255
|
+
*
|
|
4256
|
+
* @example
|
|
4257
|
+
* ```xup.in#?##xupab```
|
|
4258
|
+
* @example
|
|
4259
|
+
* ```lunar.az#?#.sagpanel div[class^="yenisb"]:contains(Reklam)```
|
|
4260
|
+
* @example
|
|
4261
|
+
* ```haal.fashion#?#div:has(> div > div > div.dfp-ad-unit)```
|
|
4262
|
+
*/
|
|
4263
|
+
const EXTENDED_CSS_RULES_PATTERNS = [
|
|
4264
|
+
'\\[-ext-',
|
|
4265
|
+
':has\\(',
|
|
4266
|
+
':has-text\\(',
|
|
4267
|
+
':contains\\(',
|
|
4268
|
+
':matches-css\\(',
|
|
4269
|
+
':matches-attr\\(',
|
|
4270
|
+
':matches-property\\(',
|
|
4271
|
+
':xpath\\(',
|
|
4272
|
+
':nth-ancestor\\(',
|
|
4273
|
+
':upward\\(',
|
|
4274
|
+
':remove\\(',
|
|
4275
|
+
':matches-css-before\\(',
|
|
4276
|
+
':matches-css-after\\(',
|
|
4277
|
+
':-abp-has\\(',
|
|
4278
|
+
':-abp-contains\\(',
|
|
4279
|
+
'#\\?#',
|
|
4280
|
+
'#\\$\\?#',
|
|
4281
|
+
'#@\\?#',
|
|
4282
|
+
'#@\\$\\?#',
|
|
4283
|
+
];
|
|
4284
|
+
|
|
4285
|
+
/**
|
|
4286
|
+
* Used for `EXTENSION_CHROMIUM`, `EXTENSION_CHROMIUM_MV3`, `EXTENSION_EDGE`, and `EXTENSION_OPERA` platforms.
|
|
4287
|
+
*/
|
|
4288
|
+
const CHROMIUM_BASED_EXTENSION_PATTERNS = [
|
|
4289
|
+
...HTML_FILTERING_MODIFIER_PATTERNS,
|
|
4290
|
+
...REPLACE_MODIFIER_PATTERNS,
|
|
4291
|
+
...APP_MODIFIER_PATTERNS,
|
|
4292
|
+
...NETWORK_MODIFIER_PATTERNS,
|
|
4293
|
+
...PROTOBUF_MODIFIER_PATTERNS,
|
|
4294
|
+
...EXTENSION_MODIFIER_PATTERNS,
|
|
4295
|
+
...HLS_MODIFIER_PATTERNS,
|
|
4296
|
+
...JSONPRUNE_MODIFIER_PATTERNS,
|
|
4297
|
+
...REFERRERPOLICY_MODIFIER_PATTERNS,
|
|
4298
|
+
...CONTENT_MODIFIER_PATTERNS,
|
|
4299
|
+
];
|
|
4300
|
+
|
|
4301
|
+
/**
|
|
4302
|
+
* Used for `EXTENSION_SAFARI` and `IOS` platforms.
|
|
4303
|
+
*/
|
|
4304
|
+
const SAFARI_BASED_EXTENSION_PATTERNS = [
|
|
4305
|
+
...DOMAIN_WITH_REGEXPS_PATTERNS,
|
|
4306
|
+
...HTML_FILTERING_MODIFIER_PATTERNS,
|
|
4307
|
+
...EXTENSION_MODIFIER_PATTERNS,
|
|
4308
|
+
...REMOVEPARAM_MODIFIER_PATTERNS,
|
|
4309
|
+
...REMOVEHEADER_MODIFIER_PATTERNS,
|
|
4310
|
+
...MP4_MODIFIER_PATTERNS,
|
|
4311
|
+
...REPLACE_MODIFIER_PATTERNS,
|
|
4312
|
+
...STEALTH_MODIFIER_PATTERNS,
|
|
4313
|
+
...COOKIE_MODIFIER_PATTERNS,
|
|
4314
|
+
...APP_MODIFIER_PATTERNS,
|
|
4315
|
+
...PROTOBUF_MODIFIER_PATTERNS,
|
|
4316
|
+
...REDIRECT_MODIFIER_PATTERNS,
|
|
4317
|
+
...REDIRECT_RULE_MODIFIER_PATTERNS,
|
|
4318
|
+
...EMPTY_MODIFIER_PATTERNS,
|
|
4319
|
+
...WEBRTC_MODIFIER_PATTERNS,
|
|
4320
|
+
...CSP_MODIFIER_PATTERNS,
|
|
4321
|
+
...ONLY_CONTENT_MODIFIER_PATTERNS,
|
|
4322
|
+
...NETWORK_MODIFIER_PATTERNS,
|
|
4323
|
+
...REFERRERPOLICY_MODIFIER_PATTERNS,
|
|
4324
|
+
...HLS_MODIFIER_PATTERNS,
|
|
4325
|
+
...JSONPRUNE_MODIFIER_PATTERNS,
|
|
4326
|
+
];
|
|
4327
|
+
|
|
4328
|
+
/* eslint-disable max-len */
|
|
4329
|
+
/**
|
|
4330
|
+
* Pattern to detect Extended CSS `:matches-property()` rules
|
|
4331
|
+
*
|
|
4332
|
+
* @example
|
|
4333
|
+
* ```unsplash.com#?#.ripi6 > div:matches-property(/__reactFiber/.return.return.memoizedProps.ad)```
|
|
4334
|
+
* @example
|
|
4335
|
+
* ```
|
|
4336
|
+
* androidauthority.com#?#main div[class]:has(> div[class]:matches-property(/__reactFiber/.return.memoizedProps.type=med_rect_atf))
|
|
4337
|
+
* ```
|
|
4338
|
+
*/
|
|
4339
|
+
const CSS_MATCHES_PROPERTY_RULES_PATTERNS = [
|
|
4340
|
+
':matches-property\\(',
|
|
4341
|
+
];
|
|
4342
|
+
|
|
4343
|
+
/**
|
|
4344
|
+
* Pattern to detect generic CSS rules
|
|
4345
|
+
*
|
|
4346
|
+
* @example
|
|
4347
|
+
* ```#$#div[class="adsbygoogle"][id="ad-detector"] { display: block !important; }```
|
|
4348
|
+
* @example
|
|
4349
|
+
* ```#$#.pub_728x90.text-ad.textAd.text_ad.text_ads.text-ads.text-ad-links { display: block !important; }```
|
|
4350
|
+
*/
|
|
4351
|
+
const CSS_GENERIC_RULES_PATTERNS = [
|
|
4352
|
+
'^#\\$#',
|
|
4353
|
+
];
|
|
4354
|
+
|
|
4355
|
+
const platformsConfig = {
|
|
4356
|
+
'WINDOWS': {
|
|
4357
|
+
'platform': 'windows',
|
|
4358
|
+
'path': 'windows',
|
|
4359
|
+
'expires': '12 hours',
|
|
4360
|
+
'configuration': {
|
|
4361
|
+
'ignoreRuleHints': false,
|
|
4362
|
+
'replacements': null,
|
|
4363
|
+
},
|
|
4364
|
+
'defines': {
|
|
4365
|
+
'adguard': true,
|
|
4366
|
+
'adguard_app_windows': true,
|
|
4367
|
+
},
|
|
4368
|
+
},
|
|
4369
|
+
'MAC': {
|
|
4370
|
+
'platform': 'mac',
|
|
4371
|
+
'path': 'mac',
|
|
4372
|
+
'configuration': {
|
|
4373
|
+
'ignoreRuleHints': false,
|
|
4374
|
+
'replacements': null,
|
|
4375
|
+
},
|
|
4376
|
+
'defines': {
|
|
4377
|
+
'adguard': true,
|
|
4378
|
+
'adguard_app_mac': true,
|
|
4379
|
+
},
|
|
4380
|
+
},
|
|
4381
|
+
'MAC_V2': {
|
|
4382
|
+
'platform': 'mac',
|
|
4383
|
+
'path': 'mac_v2',
|
|
4384
|
+
'expires': '12 hours',
|
|
4385
|
+
'configuration': {
|
|
4386
|
+
'ignoreRuleHints': false,
|
|
4387
|
+
'replacements': null,
|
|
4388
|
+
},
|
|
4389
|
+
'defines': {
|
|
4390
|
+
'adguard': true,
|
|
4391
|
+
'adguard_app_mac': true,
|
|
4392
|
+
},
|
|
4393
|
+
},
|
|
4394
|
+
'MAC_V3': {
|
|
4395
|
+
'platform': 'mac',
|
|
4396
|
+
'path': 'mac_v3',
|
|
4397
|
+
'expires': '12 hours',
|
|
4398
|
+
'configuration': {
|
|
4399
|
+
'ignoreRuleHints': false,
|
|
4400
|
+
'replacements': null,
|
|
4401
|
+
},
|
|
4402
|
+
'defines': {
|
|
4403
|
+
'adguard': true,
|
|
4404
|
+
'adguard_app_mac': true,
|
|
4405
|
+
},
|
|
4406
|
+
},
|
|
4407
|
+
'ANDROID': {
|
|
4408
|
+
'platform': 'android',
|
|
4409
|
+
'path': 'android',
|
|
4410
|
+
'configuration': {
|
|
4411
|
+
'ignoreRuleHints': false,
|
|
4412
|
+
'replacements': null,
|
|
4413
|
+
},
|
|
4414
|
+
'defines': {
|
|
4415
|
+
'adguard': true,
|
|
4416
|
+
'adguard_app_android': true,
|
|
4417
|
+
},
|
|
4418
|
+
},
|
|
4419
|
+
'CLI': {
|
|
4420
|
+
'platform': 'cli',
|
|
4421
|
+
'path': 'cli',
|
|
4422
|
+
'expires': '12 hours',
|
|
4423
|
+
'configuration': {
|
|
4424
|
+
'ignoreRuleHints': false,
|
|
4425
|
+
'replacements': null,
|
|
4426
|
+
},
|
|
4427
|
+
'defines': {
|
|
4428
|
+
'adguard': true,
|
|
4429
|
+
'adguard_app_cli': true,
|
|
4430
|
+
},
|
|
4431
|
+
},
|
|
4432
|
+
'EXTENSION_CHROMIUM': {
|
|
4433
|
+
'platform': 'ext_chromium',
|
|
4434
|
+
'path': 'extension/chromium',
|
|
4435
|
+
'expires': '10 days',
|
|
4436
|
+
'configuration': {
|
|
4437
|
+
'removeRulePatterns': CHROMIUM_BASED_EXTENSION_PATTERNS,
|
|
4438
|
+
'replacements': null,
|
|
4439
|
+
'ignoreRuleHints': false,
|
|
4440
|
+
},
|
|
4441
|
+
'defines': {
|
|
4442
|
+
'adguard': true,
|
|
4443
|
+
'adguard_ext_chromium': true,
|
|
4444
|
+
},
|
|
4445
|
+
},
|
|
4446
|
+
'EXTENSION_CHROMIUM_MV3': {
|
|
4447
|
+
'platform': 'ext_chromium_mv3',
|
|
4448
|
+
'path': 'extension/chromium-mv3',
|
|
4449
|
+
'expires': '10 days',
|
|
4450
|
+
'configuration': {
|
|
4451
|
+
'removeRulePatterns': [
|
|
4452
|
+
...CHROMIUM_BASED_EXTENSION_PATTERNS,
|
|
4453
|
+
...REDIRECT_MODIFIER_PATTERNS,
|
|
4454
|
+
],
|
|
4455
|
+
'replacements': null,
|
|
4456
|
+
'ignoreRuleHints': false,
|
|
4457
|
+
},
|
|
4458
|
+
'defines': {
|
|
4459
|
+
'adguard': true,
|
|
4460
|
+
'adguard_ext_chromium_mv3': true,
|
|
4461
|
+
},
|
|
4462
|
+
},
|
|
4463
|
+
'EXTENSION_EDGE': {
|
|
4464
|
+
'platform': 'ext_edge',
|
|
4465
|
+
'path': 'extension/edge',
|
|
4466
|
+
'expires': '10 days',
|
|
4467
|
+
'configuration': {
|
|
4468
|
+
'removeRulePatterns': CHROMIUM_BASED_EXTENSION_PATTERNS,
|
|
4469
|
+
'replacements': null,
|
|
4470
|
+
'ignoreRuleHints': false,
|
|
4471
|
+
},
|
|
4472
|
+
'defines': {
|
|
4473
|
+
'adguard': true,
|
|
4474
|
+
'adguard_ext_edge': true,
|
|
4475
|
+
'adguard_ext_chromium': true,
|
|
4476
|
+
},
|
|
4477
|
+
},
|
|
4478
|
+
'EXTENSION_OPERA': {
|
|
4479
|
+
'platform': 'ext_opera',
|
|
4480
|
+
'path': 'extension/opera',
|
|
4481
|
+
'expires': '10 days',
|
|
4482
|
+
'configuration': {
|
|
4483
|
+
'removeRulePatterns': CHROMIUM_BASED_EXTENSION_PATTERNS,
|
|
4484
|
+
'replacements': null,
|
|
4485
|
+
'ignoreRuleHints': false,
|
|
4486
|
+
},
|
|
4487
|
+
'defines': {
|
|
4488
|
+
'adguard': true,
|
|
4489
|
+
'adguard_ext_opera': true,
|
|
4490
|
+
'adguard_ext_chromium': true,
|
|
4491
|
+
},
|
|
4492
|
+
},
|
|
4493
|
+
'EXTENSION_FIREFOX': {
|
|
4494
|
+
'platform': 'ext_ff',
|
|
4495
|
+
'path': 'extension/firefox',
|
|
4496
|
+
'expires': '10 days',
|
|
4497
|
+
'configuration': {
|
|
4498
|
+
'removeRulePatterns': [
|
|
4499
|
+
...HTML_FILTERING_MODIFIER_PATTERNS,
|
|
4500
|
+
...APP_MODIFIER_PATTERNS,
|
|
4501
|
+
...NETWORK_MODIFIER_PATTERNS,
|
|
4502
|
+
...PROTOBUF_MODIFIER_PATTERNS,
|
|
4503
|
+
...EXTENSION_MODIFIER_PATTERNS,
|
|
4504
|
+
...HLS_MODIFIER_PATTERNS,
|
|
4505
|
+
...JSONPRUNE_MODIFIER_PATTERNS,
|
|
4506
|
+
...REFERRERPOLICY_MODIFIER_PATTERNS,
|
|
4507
|
+
|
|
4508
|
+
],
|
|
4509
|
+
'replacements': null,
|
|
4510
|
+
'ignoreRuleHints': false,
|
|
4511
|
+
},
|
|
4512
|
+
'defines': {
|
|
4513
|
+
'adguard': true,
|
|
4514
|
+
'adguard_ext_firefox': true,
|
|
4515
|
+
},
|
|
4516
|
+
},
|
|
4517
|
+
'EXTENSION_SAFARI': {
|
|
4518
|
+
'platform': 'ext_safari',
|
|
4519
|
+
'path': 'extension/safari',
|
|
4520
|
+
'configuration': {
|
|
4521
|
+
'removeRulePatterns': SAFARI_BASED_EXTENSION_PATTERNS,
|
|
4522
|
+
'replacements': null,
|
|
4523
|
+
'ignoreRuleHints': false,
|
|
4524
|
+
},
|
|
4525
|
+
'defines': {
|
|
4526
|
+
'adguard': true,
|
|
4527
|
+
'adguard_ext_safari': true,
|
|
4528
|
+
},
|
|
4529
|
+
},
|
|
4530
|
+
'IOS': {
|
|
4531
|
+
'platform': 'ios',
|
|
4532
|
+
'path': 'ios',
|
|
4533
|
+
'configuration': {
|
|
4534
|
+
'removeRulePatterns': SAFARI_BASED_EXTENSION_PATTERNS,
|
|
4535
|
+
'replacements': null,
|
|
4536
|
+
'ignoreRuleHints': false,
|
|
4537
|
+
},
|
|
4538
|
+
'defines': {
|
|
4539
|
+
'adguard': true,
|
|
4540
|
+
'adguard_app_ios': true,
|
|
4541
|
+
},
|
|
4542
|
+
},
|
|
4543
|
+
'EXTENSION_ANDROID_CONTENT_BLOCKER': {
|
|
4544
|
+
'platform': 'ext_android_cb',
|
|
4545
|
+
'path': 'extension/android-content-blocker',
|
|
4546
|
+
'configuration': {
|
|
4547
|
+
'removeRulePatterns': [
|
|
4548
|
+
...DOMAIN_WITH_REGEXPS_PATTERNS,
|
|
4549
|
+
...HTML_FILTERING_MODIFIER_PATTERNS,
|
|
4550
|
+
...EXTENSION_MODIFIER_PATTERNS,
|
|
4551
|
+
...REMOVEPARAM_MODIFIER_PATTERNS,
|
|
4552
|
+
...REMOVEHEADER_MODIFIER_PATTERNS,
|
|
4553
|
+
...JAVASCRIPT_RULES_PATTERNS,
|
|
4554
|
+
...CSS_RULES_PATTERNS,
|
|
4555
|
+
...MP4_MODIFIER_PATTERNS,
|
|
4556
|
+
...REPLACE_MODIFIER_PATTERNS,
|
|
4557
|
+
...STEALTH_MODIFIER_PATTERNS,
|
|
4558
|
+
...COOKIE_MODIFIER_PATTERNS,
|
|
4559
|
+
...EMPTY_MODIFIER_PATTERNS,
|
|
4560
|
+
...APP_MODIFIER_PATTERNS,
|
|
4561
|
+
...PROTOBUF_MODIFIER_PATTERNS,
|
|
4562
|
+
...CSP_MODIFIER_PATTERNS,
|
|
4563
|
+
...EXTENDED_CSS_RULES_PATTERNS,
|
|
4564
|
+
...REDIRECT_MODIFIER_PATTERNS,
|
|
4565
|
+
...REDIRECT_RULE_MODIFIER_PATTERNS,
|
|
4566
|
+
...ONLY_CONTENT_MODIFIER_PATTERNS,
|
|
4567
|
+
...ALL_MODIFIER_PATTERNS,
|
|
4568
|
+
...NETWORK_MODIFIER_PATTERNS,
|
|
4569
|
+
...REFERRERPOLICY_MODIFIER_PATTERNS,
|
|
4570
|
+
...HLS_MODIFIER_PATTERNS,
|
|
4571
|
+
...JSONPRUNE_MODIFIER_PATTERNS,
|
|
4572
|
+
...JSINJECT_MODIFIER_PATTERNS,
|
|
4573
|
+
...URLBLOCK_MODIFIER_PATTERNS,
|
|
4574
|
+
],
|
|
4575
|
+
'ignoreRuleHints': false,
|
|
4576
|
+
},
|
|
4577
|
+
'defines': {
|
|
4578
|
+
'adguard': true,
|
|
4579
|
+
'adguard_ext_android_cb': true,
|
|
4580
|
+
},
|
|
4581
|
+
},
|
|
4582
|
+
'EXTENSION_UBLOCK': {
|
|
4583
|
+
'platform': 'ext_ublock',
|
|
4584
|
+
'path': 'extension/ublock',
|
|
4585
|
+
'configuration': {
|
|
4586
|
+
'removeRulePatterns': [
|
|
4587
|
+
...HTML_FILTERING_MODIFIER_PATTERNS,
|
|
4588
|
+
...MP4_MODIFIER_PATTERNS,
|
|
4589
|
+
...REPLACE_MODIFIER_PATTERNS,
|
|
4590
|
+
...STEALTH_MODIFIER_PATTERNS,
|
|
4591
|
+
...COOKIE_MODIFIER_PATTERNS,
|
|
4592
|
+
...APP_MODIFIER_PATTERNS,
|
|
4593
|
+
...NETWORK_MODIFIER_PATTERNS,
|
|
4594
|
+
...PROTOBUF_MODIFIER_PATTERNS,
|
|
4595
|
+
...EXTENSION_MODIFIER_PATTERNS,
|
|
4596
|
+
...JSINJECT_MODIFIER_PATTERNS,
|
|
4597
|
+
...URLBLOCK_MODIFIER_PATTERNS,
|
|
4598
|
+
...CONTENT_MODIFIER_PATTERNS,
|
|
4599
|
+
...WEBRTC_MODIFIER_PATTERNS,
|
|
4600
|
+
...CSS_MEDIA_RULES_PATTERNS,
|
|
4601
|
+
...HLS_MODIFIER_PATTERNS,
|
|
4602
|
+
...REFERRERPOLICY_MODIFIER_PATTERNS,
|
|
4603
|
+
...JSONPRUNE_MODIFIER_PATTERNS,
|
|
4604
|
+
...UNBLOCKING_IMPORTANT_RULES_PATTERNS,
|
|
4605
|
+
...REMOVEHEADER_MODIFIER_PATTERNS,
|
|
4606
|
+
...CSS_MATCHES_PROPERTY_RULES_PATTERNS, // TODO: remove when this issue is fixed - https://github.com/AdguardTeam/FiltersCompiler/issues/252
|
|
4607
|
+
...CSS_GENERIC_RULES_PATTERNS,
|
|
4608
|
+
],
|
|
4609
|
+
'ignoreRuleHints': false,
|
|
4610
|
+
'adbHeader': '![Adblock Plus 2.0]',
|
|
4611
|
+
},
|
|
4612
|
+
'defines': {
|
|
4613
|
+
'ext_ublock': true,
|
|
4614
|
+
},
|
|
4615
|
+
},
|
|
4616
|
+
};
|
|
4617
|
+
|
|
4618
|
+
// Sets RuleConverter to use logger of current library
|
|
4619
|
+
tsurlfilter.setLogger(logger);
|
|
4620
|
+
|
|
4621
|
+
// Sets configuration compatibility
|
|
4622
|
+
tsurlfilter.setConfiguration({ compatibility: tsurlfilter.CompatibilityTypes.Corelibs });
|
|
4623
|
+
|
|
4624
|
+
const __dirname$1 = path.dirname(new URL((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('index.cjs', document.baseURI).href))).pathname);
|
|
4625
|
+
|
|
4626
|
+
const jsonSchemasConfigDir = path.join(__dirname$1, './schemas/');
|
|
4627
|
+
|
|
4628
|
+
process.on('unhandledRejection', (error) => {
|
|
4629
|
+
throw error;
|
|
4630
|
+
});
|
|
4631
|
+
|
|
4632
|
+
const compile = (path, logPath, reportFile, platformsPath, whitelist, blacklist, customPlatformsConfig) => {
|
|
4633
|
+
if (customPlatformsConfig) {
|
|
4634
|
+
logger.info('Using custom platforms configuration');
|
|
4635
|
+
// eslint-disable-next-line no-restricted-syntax, guard-for-in
|
|
4636
|
+
for (const platform in customPlatformsConfig) {
|
|
4637
|
+
logger.info(`Redefining platform ${platform}`);
|
|
4638
|
+
platformsConfig[platform] = customPlatformsConfig[platform];
|
|
4639
|
+
}
|
|
4640
|
+
}
|
|
4641
|
+
|
|
4642
|
+
return build(
|
|
4643
|
+
path,
|
|
4644
|
+
logPath,
|
|
4645
|
+
reportFile,
|
|
4646
|
+
platformsPath,
|
|
4647
|
+
platformsConfig,
|
|
4648
|
+
whitelist,
|
|
4649
|
+
blacklist,
|
|
4650
|
+
);
|
|
4651
|
+
};
|
|
4652
|
+
|
|
4653
|
+
const validateJSONSchema = (platformsPath, requiredFiltersAmount) => {
|
|
4654
|
+
return schemaValidator.validate(platformsPath, jsonSchemasConfigDir, requiredFiltersAmount);
|
|
4655
|
+
};
|
|
4656
|
+
|
|
4657
|
+
const validateLocales = (localesDirPath, requiredLocales) => {
|
|
4658
|
+
return localesValidator.validate(localesDirPath, requiredLocales);
|
|
4659
|
+
};
|
|
4660
|
+
|
|
4661
|
+
exports.compile = compile;
|
|
4662
|
+
exports.validateJSONSchema = validateJSONSchema;
|
|
4663
|
+
exports.validateLocales = validateLocales;
|