@adguard/filters-compiler 3.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js ADDED
@@ -0,0 +1,4658 @@
1
+ import path from 'path';
2
+ import { RuleFactory, CosmeticRule, NetworkRule, setLogger, setConfiguration, CompatibilityTypes } from '@adguard/tsurlfilter';
3
+ import fs from 'fs';
4
+ import md5 from 'md5';
5
+ import { FiltersDownloader } from '@adguard/filters-downloader';
6
+ import { RuleParser, RuleConverter, RuleGenerator, CosmeticRuleType, CommentParser, RuleCategory, RegExpUtils, ADG_SCRIPTLET_MASK } from '@adguard/agtree';
7
+ import { Logger } from '@adguard/logger';
8
+ import { defaultParserOptions } from '@adguard/agtree/parser';
9
+ import { parse } from '@adguard/ecss-tree';
10
+ import { TextEncoder, TextDecoder } from 'util';
11
+ import { createRequire } from 'module';
12
+ import { JSDOM } from 'jsdom';
13
+ import crypto from 'crypto';
14
+ import moment from 'moment';
15
+ import { parse as parse$1 } from 'tldts';
16
+ import Ajv from 'ajv';
17
+
18
+ /**
19
+ * Version utility functions
20
+ */
21
+ const version = {
22
+ /**
23
+ * Parses version from string
24
+ *
25
+ * @param v version string
26
+ * @returns {Array}
27
+ */
28
+ parse(v) {
29
+ const version = [];
30
+ const parts = String(v || '').split('.');
31
+
32
+ const parseVersionPart = (part) => {
33
+ if (Number.isNaN(part)) {
34
+ return 0;
35
+ }
36
+ return Math.max(part - 0, 0);
37
+ };
38
+
39
+ // eslint-disable-next-line no-restricted-syntax
40
+ for (const part of parts) {
41
+ version.push(parseVersionPart(part));
42
+ }
43
+
44
+ return version;
45
+ },
46
+
47
+ /**
48
+ * Increments build part of version '0.0.0.0'
49
+ *
50
+ * @param v version string
51
+ * @returns {string}
52
+ */
53
+ increment(v) {
54
+ const version = this.parse(v);
55
+
56
+ if (version.length > 0) {
57
+ version[version.length - 1] = version[version.length - 1] + 1;
58
+ }
59
+
60
+ for (let i = version.length; i > 0; i -= 1) {
61
+ if (version[i] === 100) {
62
+ version[i] = 0;
63
+ version[i - 1] += 1;
64
+ }
65
+ }
66
+
67
+ return version.join('.');
68
+ },
69
+ };
70
+
71
+ /**
72
+ * Export logger implementation.
73
+ */
74
+ const logger = new Logger(console);
75
+
76
+ /**
77
+ * Initializes logger
78
+ *
79
+ * @param path log file
80
+ * @param level log lvl
81
+ */
82
+ logger.initialize = (path) => {
83
+ if (!path) {
84
+ /* eslint-disable-next-line no-console */
85
+ console.warn('Log file is not specified');
86
+ return;
87
+ }
88
+ fs.openSync(path, 'w');
89
+ };
90
+
91
+ // TODO: a lot of these masks can be imported from @adguard/agtree
92
+ /**
93
+ * Rule masks constants
94
+ */
95
+ const RuleMasks = {
96
+ MASK_REGEX_RULE: '/',
97
+ MASK_RULE_SEPARATOR: '^',
98
+ MASK_WHITE_LIST: '@@',
99
+ MASK_BASE_RULE: '||',
100
+ MASK_ELEMENT_HIDING: '##',
101
+ MASK_ELEMENT_HIDING_EXCEPTION: '#@#',
102
+ MASK_CSS: '#$#',
103
+ MASK_CSS_EXCEPTION: '#@$#',
104
+ MASK_CSS_EXTENDED_CSS_RULE: '#?#',
105
+ MASK_CSS_EXCEPTION_EXTENDED_CSS_RULE: '#@?#',
106
+ MASK_CSS_INJECT_EXTENDED_CSS_RULE: '#$?#',
107
+ MASK_CSS_EXCEPTION_INJECT_EXTENDED_CSS_RULE: '#@$?#',
108
+ MASK_SCRIPT: '#%#',
109
+ MASK_SCRIPT_EXCEPTION: '#@%#',
110
+ MASK_CONTENT: '$$',
111
+ MASK_CONTENT_EXCEPTION: '$@$',
112
+ MASK_COMMENT: '!',
113
+ MASK_HOST_FILE_COMMENT: '#',
114
+ MASK_HINT: '!+',
115
+ MASK_DIRECTIVES: '!#',
116
+ MASK_SCRIPTLET: '#%#//scriptlet',
117
+ MASK_SCRIPTLET_EXCEPTION: '#@%#//scriptlet',
118
+ MASK_TRUSTED_SCRIPTLET: 'trusted-',
119
+ };
120
+
121
+ /**
122
+ * Excludes rule
123
+ * @param {string} rule
124
+ * @param {array} excluded
125
+ * @param {string} message
126
+ */
127
+ const excludeRule$1 = (rule, excluded, message) => {
128
+ if (excluded) {
129
+ excluded.push(`! ${message}`);
130
+ excluded.push(rule);
131
+ }
132
+ };
133
+
134
+ /**
135
+ * Converts rules to AdGuard syntax
136
+ * @param {array} rulesList
137
+ * @param {array} [excluded]
138
+ * @return {array} result
139
+ */
140
+ const convertRulesToAdgSyntax = (rulesList, excluded) => {
141
+ const result = [];
142
+
143
+ for (let i = 0; i < rulesList.length; i += 1) {
144
+ const rule = rulesList[i];
145
+ try {
146
+ const ruleNode = RuleParser.parse(rule);
147
+ const conversionResult = RuleConverter.convertToAdg(ruleNode);
148
+ const convertedRules = conversionResult.result.map((r) => RuleGenerator.generate(r));
149
+ result.push(...convertedRules);
150
+
151
+ if (conversionResult.isConverted) {
152
+ const message = `Rule "${rule}" converted to: "${[...convertedRules]}"`;
153
+ excludeRule$1(rule, excluded, message);
154
+ }
155
+ } catch (e) {
156
+ const message = `Unable to convert rule to AdGuard syntax: "${rule}" due to error: ${e.message}`;
157
+ logger.info(message);
158
+ excludeRule$1(rule, excluded, message);
159
+ }
160
+ }
161
+
162
+ return result;
163
+ };
164
+
165
+ const SINGLE_QUOTE = "'";
166
+ const DOUBLE_QUOTE = '"';
167
+
168
+ /**
169
+ * Trims quotes (single or double) from the start and end of a string if they exist.
170
+ *
171
+ * @param {string} str - The string to trim.
172
+ * @returns {string} The trimmed string.
173
+ */
174
+ const trimQuotes = (str) => {
175
+ if (
176
+ (str.startsWith(SINGLE_QUOTE) && str.endsWith(SINGLE_QUOTE))
177
+ || (str.startsWith(DOUBLE_QUOTE) && str.endsWith(DOUBLE_QUOTE))
178
+ ) {
179
+ return str.slice(1, -1);
180
+ }
181
+ return str;
182
+ };
183
+
184
+ /**
185
+ * Converts a list of rules into uBO (uBlock Origin) syntax.
186
+ *
187
+ * @param {string[]} rules - An array of rules to be converted.
188
+ * @returns {string[]} An array of converted rules in uBO syntax. If no rules are provided, an empty array is returned.
189
+ *
190
+ * @throws {Error} Logs an error message if a rule cannot be converted due to a parsing or conversion issue.
191
+ */
192
+ const convertToUbo = (rules) => {
193
+ const modified = [];
194
+ if (!rules) {
195
+ return modified;
196
+ }
197
+ rules.forEach((rule) => {
198
+ if (rule) {
199
+ try {
200
+ const ruleNode = RuleParser.parse(rule);
201
+ // js injection rules are not supported in uBO
202
+ if (ruleNode.type === CosmeticRuleType.JsInjectionRule) {
203
+ return;
204
+ }
205
+ if (ruleNode.type === CosmeticRuleType.ScriptletInjectionRule) {
206
+ const scriptletNode = ruleNode.body.children[0].children[0];
207
+ const scriptletNameString = trimQuotes(scriptletNode.value);
208
+
209
+ if (!scriptletNameString) {
210
+ // If the scriptlet name is missing, skip processing this rule
211
+ return;
212
+ }
213
+
214
+ const scriptletName = trimQuotes(scriptletNameString);
215
+
216
+ // TODO: move this check to AGTree AG-41266
217
+ if (scriptletName.startsWith(RuleMasks.MASK_TRUSTED_SCRIPTLET)) {
218
+ // https://github.com/AdguardTeam/Scriptlets#trusted-scriptlets-restriction
219
+ // does not work in other blockers
220
+ const message = `Trusted scriptlets should not be converted to uBO syntax. Rule: "${rule}"`;
221
+ logger.info(message);
222
+ modified.push('');
223
+ return;
224
+ }
225
+ }
226
+ const conversionResult = RuleConverter.convertToUbo(ruleNode);
227
+ const convertedRules = conversionResult.result.map((r) => RuleGenerator.generate(r));
228
+ modified.push(...convertedRules);
229
+ } catch (e) {
230
+ const message = `Unable to convert rule to Ubo syntax: "${rule}" due to error: ${e.message}`;
231
+ logger.info(message);
232
+ }
233
+ } else {
234
+ modified.push('');
235
+ }
236
+ });
237
+ return modified;
238
+ };
239
+
240
+ /**
241
+ * ExtendedCss is not supposed to work without window environment,
242
+ * so we pass some wrapper dummy.
243
+ */
244
+ // TODO: switch to aglint in compiler
245
+
246
+ global.TextEncoder = TextEncoder;
247
+ global.TextDecoder = TextDecoder;
248
+
249
+ const dom = new JSDOM('<!DOCTYPE html><p>Empty</p>');
250
+
251
+ global.window = dom.window;
252
+ global.document = global.window.document;
253
+ if (!global.navigator) {
254
+ global.navigator = global.window.navigator;
255
+ }
256
+ global.Element = global.window.Element;
257
+
258
+ const require$1 = createRequire(import.meta.url);
259
+ const { ExtendedCss } = require$1('@adguard/extended-css');
260
+
261
+ // TODO: in future SelectorValidationResult from ExtendedCss may be imported and used instead of it
262
+ /**
263
+ * @typedef {Object} SelectorValidationResult
264
+ * @property {boolean} ok selector validation status
265
+ * @property {string|null} error reason of invalid selector for invalid selector
266
+ * and `null` for valid one
267
+ */
268
+
269
+ /**
270
+ * Related to the bug — pseudo-class arg with combinators.
271
+ *
272
+ * @see {@link https://github.com/dperini/nwsapi/issues/55}
273
+ *
274
+ * @example
275
+ * '*:not(div > span)'
276
+ */
277
+ const VALID_PSEUDO_CLASS_COMBINATOR_ARG_REGEXP = /(.+)?:(not|is)\((.+)?(~|>|\+)(.+)?\)(.+)?/;
278
+
279
+ /**
280
+ * Related to the bug — pseudo-class arg with parenthesis in attribute value.
281
+ *
282
+ * @see {@link https://github.com/dperini/nwsapi/issues/71}
283
+ *
284
+ * @example
285
+ * 'div:not([right=")"])'
286
+ * 'body *:not([left="("])'
287
+ */
288
+ const VALID_PSEUDO_CLASS_PARENTHESIS_ARG_REGEXP = /(.+)?:(not|is)\((.+)?\[.+=("|')(.+)?(\(|\))(.+)?("|')\]\)/;
289
+
290
+ // TODO: remove backupValidate() after the bugs are fixed
291
+ /**
292
+ * Validates `selector` by its matching with specific regular expressions due to nwsapi bugs:
293
+ * @see {@link https://github.com/dperini/nwsapi/issues/55},
294
+ * @see {@link https://github.com/dperini/nwsapi/issues/71}.
295
+ *
296
+ * @param selector Selector to validate.
297
+ * @param originalError Previous validation error for selectors which are non-related to the bugs.
298
+ *
299
+ * @returns {SelectorValidationResult}
300
+ */
301
+ const backupValidate = (selector, originalError) => {
302
+ const isValidArgBugRelated = VALID_PSEUDO_CLASS_COMBINATOR_ARG_REGEXP.test(selector)
303
+ || VALID_PSEUDO_CLASS_PARENTHESIS_ARG_REGEXP.test(selector);
304
+ // if selector is not matched by the regexp specific to the bug
305
+ // original validate error should be returned
306
+ if (!isValidArgBugRelated) {
307
+ return { ok: false, error: originalError };
308
+ }
309
+ return { ok: true, error: null };
310
+ };
311
+
312
+ /**
313
+ * Validates css selector, uses ExtendedCss.validate() for it.
314
+ *
315
+ * @param selectorText
316
+ *
317
+ * @returns {SelectorValidationResult}
318
+ */
319
+ const validateCssSelector = (selectorText) => {
320
+ // jsdom is crashing when selector is a script
321
+ if (selectorText.indexOf('##script:contains') !== -1
322
+ || selectorText.indexOf('##script:inject') !== -1) {
323
+ return {
324
+ ok: false,
325
+ error: 'Selector as a script is not supported.',
326
+ };
327
+ }
328
+
329
+ // skip :before and :after selectors
330
+ if (selectorText.match(/[^:\s]([:]{1,2}before(\s|,|$))|[^:\s]([:]{1,2}after(\s|,|$))/ig)) {
331
+ return {
332
+ ok: true,
333
+ error: null,
334
+ };
335
+ }
336
+
337
+ // skip selectors with case-insensitive attribute, for example: div[class^="Abc_123" i]
338
+ if (selectorText.match(/\[[a-z\d-_]+[\^$*]?=['"]?[^'"]+['"]?\si]/g)) {
339
+ return {
340
+ ok: true,
341
+ error: null,
342
+ };
343
+ }
344
+
345
+ let validation = ExtendedCss.validate(selectorText);
346
+ // TODO: remove later when the bug is fixed
347
+ // https://github.com/dperini/nwsapi/issues/55
348
+ // ExtendedCss.validate() should be enough for selector validation
349
+ if (!validation.ok) {
350
+ validation = backupValidate(selectorText, validation.error);
351
+ }
352
+ return validation;
353
+ };
354
+
355
+ /**
356
+ * @typedef {import('@adguard/agtree').AnyRule} AnyRule
357
+ */
358
+
359
+ const AFFINITY_DIRECTIVE = '!#safari_cb_affinity'; // used as closing directive
360
+ const AFFINITY_DIRECTIVE_OPEN = `${AFFINITY_DIRECTIVE}(`;
361
+
362
+ const NOT_VALIDATE_HINT = 'NOT_VALIDATE';
363
+ const SPACE$1 = ' ';
364
+
365
+ /**
366
+ * Push rule with warning message to excluded
367
+ * @param {array} excluded
368
+ * @param {string} warning
369
+ * @param {string} rule
370
+ */
371
+ const excludeRule = (excluded, warning, rule) => {
372
+ if (excluded) {
373
+ excluded.push(warning);
374
+ excluded.push(rule);
375
+ }
376
+ };
377
+
378
+ /**
379
+ * @typedef {object} ValidationResult
380
+ * @property {boolean} valid Whether the rule is valid.
381
+ * @property {string|null} error Error message if the rule is invalid.
382
+ */
383
+
384
+ /**
385
+ * Class to validate filter rules.
386
+ */
387
+ class RuleValidator {
388
+ /**
389
+ * Creates validation result for rule.
390
+ *
391
+ * @param {boolean} valid Whether the rule is valid.
392
+ * @param {string} [error] Error message if the rule is invalid.
393
+ */
394
+ static createValidationResult(valid, error) {
395
+ if (error) {
396
+ return { valid, error };
397
+ }
398
+
399
+ return { valid, error: null };
400
+ }
401
+
402
+ /**
403
+ * Validates regexp pattern.
404
+ *
405
+ * @param {string} pattern Regexp pattern to validate.
406
+ * @param {string} ruleText Rule text.
407
+ *
408
+ * @throws {SyntaxError} If the pattern is invalid, otherwise nothing.
409
+ */
410
+ static validateRegexp(pattern, ruleText) {
411
+ if (!RegExpUtils.isRegexPattern(pattern)) {
412
+ return;
413
+ }
414
+
415
+ try {
416
+ // eslint-disable-next-line no-new
417
+ new RegExp(pattern.slice(1, -1));
418
+ } catch (e) {
419
+ throw new SyntaxError(`Rule has invalid regex pattern: "${ruleText}"`);
420
+ }
421
+ }
422
+
423
+ /**
424
+ * Validates rule node.
425
+ *
426
+ * @param {AnyRule} ruleNode Rule node to validate.
427
+ *
428
+ * @returns {ValidationResult} Validation result.
429
+ */
430
+ static validate(ruleNode) {
431
+ if (ruleNode.category === RuleCategory.Invalid) {
432
+ return RuleValidator.createValidationResult(
433
+ false,
434
+ ruleNode.error.message,
435
+ );
436
+ }
437
+
438
+ if (ruleNode.category === RuleCategory.Empty || ruleNode.category === RuleCategory.Comment) {
439
+ return RuleValidator.createValidationResult(true);
440
+ }
441
+
442
+ const ruleText = RuleGenerator.generate(ruleNode);
443
+
444
+ try {
445
+ // Validate cosmetic rules
446
+ if (ruleNode.category === RuleCategory.Cosmetic) {
447
+ // eslint-disable-next-line no-new
448
+ new CosmeticRule(ruleNode, 0);
449
+ return RuleValidator.createValidationResult(true);
450
+ }
451
+
452
+ // Validate network rules
453
+ const rule = new NetworkRule(ruleNode, 0);
454
+ RuleValidator.validateRegexp(rule.getPattern(), ruleText);
455
+ } catch (error) {
456
+ // TODO: add getErrorMessage as a helper
457
+ const message = error instanceof Error ? error.message : String(error);
458
+ const errorMessage = `Error: "${message}" in the rule: "${ruleText}"`;
459
+ return RuleValidator.createValidationResult(false, errorMessage);
460
+ }
461
+
462
+ return RuleValidator.createValidationResult(true);
463
+
464
+ // TODO: validate host rules
465
+ }
466
+ }
467
+
468
+ /**
469
+ * Universal function for validating CSS context.
470
+ *
471
+ * @param {string} input - string to be validated.
472
+ * @param {string} contextName - context (e.g., 'selectorList', 'declarationList', 'mediaQueryList').
473
+ * @throws {Error} Throws an error if parsing fails.
474
+ */
475
+ const validateCssContext = (input, contextName) => {
476
+ parse(input, {
477
+ context: contextName,
478
+ onParseError(error) {
479
+ throw error;
480
+ },
481
+ });
482
+ };
483
+
484
+ /**
485
+ * Removes invalid rules from the list of rules
486
+ * and logs process in the excluded list.
487
+ *
488
+ * @param {string[]} list List of rule texts.
489
+ * @param {string[]} excluded List of messages with validation results.
490
+ * @param {string[]} invalid List of messages with validation errors.
491
+ * @param {string} filterName Name of the filter.
492
+ *
493
+ * @returns {string[]} List of valid rules.
494
+ */
495
+ // eslint-disable-next-line default-param-last
496
+ const validateAndFilterRules = (list, excluded, invalid = [], filterName) => {
497
+ if (!list) {
498
+ return [];
499
+ }
500
+
501
+ return list.filter((ruleText, index, array) => {
502
+ if (CommentParser.isCommentRule(ruleText)) {
503
+ return true;
504
+ }
505
+
506
+ const previousRule = index > 0 ? array[index - 1] : null;
507
+ // Skip validation if "ruleText" is preceded by "NOT_VALIDATE" hint
508
+ // https://github.com/AdguardTeam/FiltersCompiler/issues/245
509
+ if (
510
+ previousRule
511
+ && previousRule.startsWith(RuleMasks.MASK_HINT)
512
+ && (
513
+ previousRule.includes(`${SPACE$1}${NOT_VALIDATE_HINT}${SPACE$1}`)
514
+ || previousRule.endsWith(`${SPACE$1}${NOT_VALIDATE_HINT}`)
515
+ )
516
+
517
+ ) {
518
+ return true;
519
+ }
520
+
521
+ let convertedRuleNodes;
522
+ try {
523
+ const ruleNode = RuleParser.parse(ruleText, {
524
+ ...defaultParserOptions,
525
+ // tolerant mode is used for rather quick syntax validation
526
+ tolerant: true,
527
+ isLocIncluded: false,
528
+ includeRaws: false,
529
+ parseAbpSpecificRules: false,
530
+ parseUboSpecificRules: false,
531
+ });
532
+ const conversionResult = RuleConverter.convertToAdg(ruleNode);
533
+ convertedRuleNodes = conversionResult.result;
534
+ // eslint-disable-next-line no-restricted-syntax
535
+ for (const convertedRuleNode of convertedRuleNodes) {
536
+ if (convertedRuleNode.category !== RuleCategory.Cosmetic) {
537
+ continue;
538
+ }
539
+ switch (convertedRuleNode.type) {
540
+ case CosmeticRuleType.ElementHidingRule: {
541
+ validateCssContext(convertedRuleNode.body.selectorList.value, 'selectorList');
542
+ break;
543
+ }
544
+ case CosmeticRuleType.CssInjectionRule: {
545
+ validateCssContext(convertedRuleNode.body.selectorList.value, 'selectorList');
546
+ if (convertedRuleNode.body.remove === true) {
547
+ break;
548
+ }
549
+ validateCssContext(convertedRuleNode.body.declarationList.value, 'declarationList');
550
+ if (convertedRuleNode.body.mediaQueryList && convertedRuleNode.body.mediaQueryList.value) {
551
+ validateCssContext(convertedRuleNode.body.mediaQueryList.value, 'mediaQueryList');
552
+ }
553
+ break;
554
+ }
555
+ default:
556
+ break;
557
+ }
558
+ }
559
+ } catch (e) {
560
+ logger.error(`Invalid rule in ${filterName}: ${ruleText}`);
561
+ excludeRule(excluded, e.message, ruleText);
562
+ invalid.push(e.message);
563
+ return false;
564
+ }
565
+
566
+ // optional chaining is needed for the length property because convertedRules can be undefined
567
+ // if RuleParser.parse() or RuleConverter.convertToAdg() throws an error
568
+ if (!convertedRuleNodes || convertedRuleNodes.length === 0) {
569
+ return false;
570
+ }
571
+
572
+ for (let i = 0; i < convertedRuleNodes.length; i += 1) {
573
+ const convertedRuleNode = convertedRuleNodes[i];
574
+
575
+ let validationResult = RuleValidator.validate(convertedRuleNode);
576
+
577
+ // TODO: remove this checking when $header is fixed
578
+ // https://github.com/AdguardTeam/AdguardBrowserExtension/issues/2942
579
+ if (
580
+ !validationResult.valid
581
+ && validationResult.error.includes('$header rules are not compatible with some other modifiers')
582
+ ) {
583
+ // $header rules are not compatible with other modifiers ONLY in the tsurlfilter
584
+ // but it is fine for corelibs
585
+ validationResult = { valid: true };
586
+ }
587
+
588
+ if (!validationResult.valid) {
589
+ // log source rule text to the excluded log
590
+ logger.error(`Invalid rule in ${filterName}: ${ruleText}`);
591
+ // save warning as comment
592
+ excludeRule(excluded, `! ${validationResult.error}`, ruleText);
593
+ // ruleText should be already included into the error text
594
+ invalid.push(validationResult.error);
595
+ return false;
596
+ }
597
+
598
+ const rule = RuleFactory.createRule(convertedRuleNode);
599
+
600
+ // It is impossible to bundle jsdom into tsurlfilter, so we check if rules are valid in the compiler
601
+ if (rule instanceof CosmeticRule && rule.getType() === CosmeticRuleType.ElementHidingRule) {
602
+ const validationResult = validateCssSelector(rule.getContent());
603
+ if (!validationResult.ok) {
604
+ // TODO: rule selector can be validated by agtree
605
+ logger.error(`Invalid rule selector in ${filterName}: ${ruleText}`);
606
+ // log source rule text to the excluded log
607
+ excludeRule(excluded, `! ${validationResult.error} in rule:`, ruleText);
608
+ invalid.push(`${validationResult.error} in rule: ${ruleText}`);
609
+ return false;
610
+ }
611
+ }
612
+ }
613
+
614
+ return true;
615
+ });
616
+ };
617
+
618
+ /**
619
+ * Validates !#safari_cb_affinity directives
620
+ *
621
+ * @param lines
622
+ */
623
+ const checkAffinityDirectives = (lines) => {
624
+ if (!(lines && lines.length)) {
625
+ // skip empty filter
626
+ return true;
627
+ }
628
+ const stack = [];
629
+
630
+ for (let i = 0; i < lines.length; i += 1) {
631
+ const line = lines[i];
632
+ if (line.startsWith(AFFINITY_DIRECTIVE_OPEN)) {
633
+ stack.push(line);
634
+ continue;
635
+ }
636
+ if (line === AFFINITY_DIRECTIVE) {
637
+ const pop = stack.pop();
638
+ if (!(pop && pop.startsWith(AFFINITY_DIRECTIVE_OPEN))) {
639
+ return false;
640
+ }
641
+ }
642
+ }
643
+
644
+ return !stack.length;
645
+ };
646
+
647
+ // TODO: consider refactoring this file
648
+ // because agtree provides modern approach to solve such problems
649
+
650
+ // Based on: https://github.com/github/linguist/pull/5968/commits/f7c5c39139945576a5f9ff0b41c990e6b6019232
651
+ // eslint-disable-next-line max-len
652
+ const ADBLOCK_AGENT_PATTERN = /^(?:!|#)?\s*\[(?<AdblockInfo>\s*(?:[Aa]d[Bb]lock(?:\s+[Pp]lus)?|u[Bb]lock(?:\s+[Oo]rigin)?|[Aa]d[Gg]uard)(?:\s+(?:\d\.?)+)?\s*)(?:;\g<AdblockInfo>)*\]\s*$/;
653
+
654
+ /**
655
+ * CSS rules with width and height attributes break SVG rendering
656
+ * https://github.com/AdguardTeam/AdguardBrowserExtension/issues/683
657
+ *
658
+ * @param ruleText Rule text
659
+ */
660
+ const fixCssRuleAttributesForEdge = function (ruleText) {
661
+ if (ruleText.includes(RuleMasks.MASK_CSS)
662
+ || ruleText.includes(RuleMasks.MASK_CSS_EXCEPTION)
663
+ || ruleText.includes(RuleMasks.MASK_ELEMENT_HIDING)
664
+ || ruleText.includes(RuleMasks.MASK_ELEMENT_HIDING_EXCEPTION)) {
665
+ ruleText = ruleText.replace(/\[width=/gi, '[Width=');
666
+ ruleText = ruleText.replace('/[height=/gi', '[Height=');
667
+ }
668
+
669
+ return ruleText;
670
+ };
671
+
672
+ /**
673
+ * Updates rule text
674
+ */
675
+ const overrideRule = function (ruleText, platform) {
676
+ if (platform === 'ext_edge') {
677
+ ruleText = fixCssRuleAttributesForEdge(ruleText);
678
+ }
679
+
680
+ return ruleText;
681
+ };
682
+
683
+ /**
684
+ * Modifies header for AdGuard Base filter
685
+ * https://github.com/AdguardTeam/FiltersCompiler/issues/78
686
+ * @param {array} header
687
+ * @param {boolean} optimized
688
+ */
689
+ const modifyBaseFilterHeader = (header, optimized) => {
690
+ header[0] = `! Title: AdGuard Base filter + EasyList${optimized ? ' (Optimized)' : ''}`;
691
+ };
692
+
693
+ /**
694
+ * Rewrites title and description
695
+ * https://github.com/AdguardTeam/AdguardFilters/issues/5138#issuecomment-328847738
696
+ */
697
+ const rewriteHeader = function (header) {
698
+ const result = [];
699
+ header.forEach((line) => {
700
+ if (line.startsWith('! Title: ')) {
701
+ line = '! Title: AdGuard Base filter';
702
+ } else if (line.startsWith('! Description: ')) {
703
+ line = '! Description: This filter is necessary for quality ad blocking.';
704
+ }
705
+
706
+ result.push(line);
707
+ });
708
+
709
+ return result;
710
+ };
711
+
712
+ /**
713
+ * Filters easylist block from list of rules
714
+ * https://github.com/AdguardTeam/AdguardFilters/issues/5138#issuecomment-328847738
715
+ */
716
+ const rewriteRules = function (rules) {
717
+ const filtered = [];
718
+ let flag = -1;
719
+ for (let i = 0; i < rules.length; i += 1) {
720
+ const rule = rules[i];
721
+
722
+ if (flag >= 0 && rule.startsWith('!------------------')) {
723
+ if (flag !== i - 1) {
724
+ // we skip next line after block header
725
+ // looking for the end of easylist block
726
+ flag = -1;
727
+ }
728
+
729
+ continue;
730
+ }
731
+
732
+ if (rule.startsWith('!------------------ EasyList rules')) {
733
+ flag = i;
734
+ continue;
735
+ }
736
+
737
+ if (flag < 0) {
738
+ filtered.push(rule);
739
+ }
740
+ }
741
+
742
+ return filtered;
743
+ };
744
+
745
+ /**
746
+ * Replaces Version: with OriginalVersion: comments in case of some client cannot afford it.
747
+ *
748
+ * @param rules
749
+ */
750
+ const fixVersionComments = (rules) => {
751
+ return rules.map((x) => {
752
+ if (x.startsWith('! Version:')) {
753
+ return x.replace('! Version:', '! OriginalVersion:');
754
+ }
755
+
756
+ return x;
757
+ });
758
+ };
759
+
760
+ // TODO: use agtree to remove adblock agent strings
761
+ /**
762
+ * Removes adblock agent strings, for example:
763
+ * - [AdBlock],
764
+ * - [Adblock Plus],
765
+ * - [Adblock Plus 2.0],
766
+ * - [AdGuard],
767
+ * - [uBlock] / [uBlock Origin], etc.
768
+ *
769
+ * @param lines
770
+ */
771
+ const removeAdblockVersion = (lines) => {
772
+ return lines.filter((line) => !line.trim().match(ADBLOCK_AGENT_PATTERN));
773
+ };
774
+
775
+ /**
776
+ * Removes scriptlet rules
777
+ * @param {array} rules
778
+ * @return {array} rules
779
+ */
780
+ const removeScriptletRules = (rules) => rules.filter((rule) => !rule.script.startsWith(ADG_SCRIPTLET_MASK));
781
+
782
+ /**
783
+ * Removes `groupDescription` field from `groups`.
784
+ *
785
+ * @param rawGroups
786
+ * @returns Corrected groups
787
+ */
788
+ const removeGroupDescriptions$1 = function (rawGroups) {
789
+ const groups = [];
790
+
791
+ // eslint-disable-next-line no-restricted-syntax
792
+ for (const g of rawGroups) {
793
+ const copy = { ...g };
794
+
795
+ delete copy.groupDescription;
796
+
797
+ groups.push(copy);
798
+ }
799
+
800
+ return groups;
801
+ };
802
+
803
+ /**
804
+ * Corrects metadata for backward compatibility with old clients on MAC (v1) platform
805
+ * Hides tag fields
806
+ *
807
+ * @param metadata
808
+ * @returns Corrected metadata
809
+ */
810
+ const rewriteMetadataForOldMacV1 = function (metadata) {
811
+ const result = {
812
+ groups: removeGroupDescriptions$1(metadata.groups.slice(0)),
813
+ filters: [],
814
+ };
815
+
816
+ // eslint-disable-next-line no-restricted-syntax
817
+ for (const f of metadata.filters) {
818
+ const copy = { ...f };
819
+ delete copy.tags;
820
+ delete copy.timeAdded;
821
+ delete copy.trustLevel;
822
+ delete copy.downloadUrl;
823
+ delete copy.deprecated;
824
+
825
+ result.filters.push(copy);
826
+ }
827
+
828
+ return result;
829
+ };
830
+
831
+ /**
832
+ * Corrects metadata for backward compatibility with old clients on MAC_V2 platform —
833
+ * removed `groupDescription` field from `groups`.
834
+ *
835
+ * @param metadata
836
+ * @returns Corrected metadata
837
+ */
838
+ const rewriteMetadataForOldMacV2 = function (metadata) {
839
+ const result = { ...metadata };
840
+ result.groups = removeGroupDescriptions$1(result.groups.slice(0));
841
+ return result;
842
+ };
843
+
844
+ /* eslint-disable global-require */
845
+
846
+ const require = createRequire(import.meta.url);
847
+
848
+ /**
849
+ * Some sources require proper user-agents and forbid downloading without.
850
+ */
851
+ const USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko)'
852
+ + 'Chrome/63.0.3239.132 Mobile Safari/537.36';
853
+
854
+ /**
855
+ * Sync downloads file from url
856
+ *
857
+ * @param url
858
+ * @param {number} [retryNum=0] number of times to retry downloading, defaults to 0
859
+ * @returns {*}
860
+ */
861
+ const tryDownloadFile = function (url, retryNum = 0) {
862
+ let args = ['--fail', '--silent', '--user-agent', USER_AGENT, '-L', url];
863
+ if (retryNum) {
864
+ args.push('--retry');
865
+ args.push(retryNum);
866
+ }
867
+ const options = { encoding: 'utf8', maxBuffer: Infinity };
868
+ const tlsCheck = process.env.TLS;
869
+ if (tlsCheck === 'insecure') {
870
+ args = ['--insecure'].concat(args);
871
+ }
872
+ return require('child_process')
873
+ .execFileSync('curl', args, options);
874
+ };
875
+
876
+ /**
877
+ * Sync downloads file from url with two attempts
878
+ *
879
+ * @param url
880
+ * @returns {*}
881
+ */
882
+ const downloadFile = (url) => {
883
+ logger.info(`Downloading: ${url}`);
884
+
885
+ // 5 times to retry after first fail attempt:
886
+ // 1 sec for first time, double for every forthcoming attempts
887
+ // so it will take: 1 + 2 + 4 + 8 + 16 = 31 seconds
888
+ // https://curl.se/docs/manpage.html#--retry
889
+ const RETRY_NUM = 5;
890
+
891
+ try {
892
+ return tryDownloadFile(url);
893
+ } catch (e) {
894
+ logger.warn(e);
895
+ logger.warn(`Retry downloading: ${url}`);
896
+ return tryDownloadFile(url, RETRY_NUM);
897
+ }
898
+ };
899
+
900
+ /* eslint-disable global-require */
901
+
902
+ // Here we can access optimizable filters and its optimization percentages
903
+ // eslint-disable-next-line max-len
904
+ const OPTIMIZATION_PERCENT_URL = 'https://chrome.adtidy.org/optimization_config/percent.json?key=4DDBE80A3DA94D819A00523252FB6380';
905
+ // eslint-disable-next-line max-len
906
+ const OPTIMIZATION_STATS_URL = 'https://chrome.adtidy.org/filters/{0}/stats.json?key=4DDBE80A3DA94D819A00523252FB6380';
907
+
908
+ let filtersOptimizationPercent = null;
909
+
910
+ /**
911
+ * Downloads and caches filters optimization percentages configuration
912
+ */
913
+ const getFiltersOptimizationPercent = () => {
914
+
915
+ if (filtersOptimizationPercent === null) {
916
+ filtersOptimizationPercent = JSON.parse(downloadFile(OPTIMIZATION_PERCENT_URL));
917
+ }
918
+
919
+ if (filtersOptimizationPercent.config.length === 0) {
920
+ // eslint-disable-next-line no-throw-literal
921
+ throw 'Invalid configuration';
922
+ }
923
+
924
+ return filtersOptimizationPercent;
925
+ };
926
+
927
+ /**
928
+ * Downloads filter optimization config for the filter
929
+ */
930
+ const getFilterOptimizationConfig = (filterId) => {
931
+
932
+ // config: [{filterId: 1, percent: 45}, ...]
933
+ const filterOptimizationPercent = getFiltersOptimizationPercent().config
934
+ .find((config) => config.filterId === filterId);
935
+
936
+ let optimizationConfig = null;
937
+ if (filterOptimizationPercent) {
938
+ optimizationConfig = JSON.parse(downloadFile(OPTIMIZATION_STATS_URL.replace('{0}', filterId)));
939
+ if (!optimizationConfig || !optimizationConfig.groups || optimizationConfig.groups.length === 0) {
940
+ throw new Error(`Unable to retrieve optimization stats for ${filterId}`);
941
+ }
942
+ }
943
+
944
+ return optimizationConfig;
945
+ };
946
+
947
+ /**
948
+ * Checks if rule should be skipped, because optimization is enabled for this filter
949
+ * and hits of this rule is lower than some value
950
+ * @param ruleText Rule text
951
+ * @param optimizationConfig Optimization config for this filter (retrieved with getFilterOptimizationConfig)
952
+ */
953
+ const skipRuleWithOptimization = (ruleText, optimizationConfig) => {
954
+ if (!optimizationConfig) {
955
+ return false;
956
+ }
957
+
958
+ // eslint-disable-next-line no-restricted-syntax
959
+ for (const group of optimizationConfig.groups) {
960
+ const hits = group.rules[ruleText];
961
+ if (hits !== undefined && hits < group.config.hits) {
962
+ return true;
963
+ }
964
+ }
965
+
966
+ return false;
967
+ };
968
+
969
+ /**
970
+ * @typedef {object} OptimizationConfig
971
+ * @property {number} filterId - Filter identifier
972
+ * @property {number} percent - Expected optimization percent
973
+ * `~= (rules count in optimized filter) / (rules count in original filter) * 100`
974
+ * @property {number} minPercent - Lower bound of `percent` value
975
+ * @property {number} maxPercent - Upper bound of `percent` value
976
+ * @property {boolean} strict - If `percent < minPercent || percent > maxPercent`
977
+ * and strict mode is on then filter compilation should fail, otherwise original rules must be used
978
+ */
979
+
980
+
981
+ const HINT_MASK = `${RuleMasks.MASK_HINT} `;
982
+ const COMMENT_REGEXP = '^\\!($|[^#])';
983
+
984
+ const PLATFORM_HINT_REGEXP = /(^| )PLATFORM\(([^)]+)\)/g;
985
+ const NOT_PLATFORM_HINT_REGEXP = /(.*)NOT_PLATFORM\(([^)]+)\)/g;
986
+
987
+ const NOT_OPTIMIZED_HINT$1 = 'NOT_OPTIMIZED';
988
+
989
+ /**
990
+ * Parses rule hints
991
+ *
992
+ * @param rules rules
993
+ * @param platform Platform
994
+ */
995
+ const splitRuleHintLines = function (rules, platform) {
996
+ const result = [];
997
+ if (rules) {
998
+ for (let i = 0; i < rules.length; i += 1) {
999
+ let rule = rules[i].trim();
1000
+ if (rule.startsWith(HINT_MASK)) {
1001
+ continue;
1002
+ }
1003
+
1004
+ rule = overrideRule(rule, platform);
1005
+
1006
+ const hint = i > 0 ? rules[i - 1] : null;
1007
+ result.push({
1008
+ rule,
1009
+ hint: (hint && hint.startsWith(HINT_MASK)) ? hint : null,
1010
+ });
1011
+ }
1012
+ }
1013
+
1014
+ return result;
1015
+ };
1016
+
1017
+ /**
1018
+ * Joins hint rules
1019
+ *
1020
+ * @param hintLines
1021
+ * @returns {Array}
1022
+ */
1023
+ const joinRuleHintLines = function (hintLines) {
1024
+ const result = [];
1025
+ hintLines.forEach((f) => {
1026
+ if (f.hint) {
1027
+ result.push(f.hint);
1028
+ }
1029
+ result.push(f.rule);
1030
+ });
1031
+
1032
+ return result;
1033
+ };
1034
+
1035
+ /**
1036
+ * Parses platforms by pattern
1037
+ *
1038
+ *
1039
+ * @param hint stripped hint
1040
+ * @param pattern regexp
1041
+ */
1042
+ const parsePlatforms = function (hint, pattern) {
1043
+ const result = [];
1044
+
1045
+ if (!hint) {
1046
+ return result;
1047
+ }
1048
+
1049
+ let match = pattern.exec(hint);
1050
+ while (match !== null) {
1051
+ const group = match[2];
1052
+ const split = group.split(',');
1053
+ // eslint-disable-next-line no-restricted-syntax
1054
+ for (let s of split) {
1055
+ s = s.trim();
1056
+ result.push(s);
1057
+ }
1058
+
1059
+ match = pattern.exec(hint);
1060
+ }
1061
+
1062
+ return result;
1063
+ };
1064
+
1065
+ /**
1066
+ * Is rule supported with platform hint
1067
+ *
1068
+ * @param rule
1069
+ * @param platform
1070
+ */
1071
+ const isPlatformSupported = function (rule, platform) {
1072
+ const { hint } = rule;
1073
+ if (!hint || !hint.startsWith(HINT_MASK)) {
1074
+ return true;
1075
+ }
1076
+
1077
+ if (!platform) {
1078
+ return true;
1079
+ }
1080
+
1081
+ const stripped = hint.substring(HINT_MASK.length).trim();
1082
+
1083
+ const supportedPlatforms = parsePlatforms(stripped, PLATFORM_HINT_REGEXP);
1084
+ const unsupportedPlatforms = parsePlatforms(stripped, NOT_PLATFORM_HINT_REGEXP);
1085
+
1086
+ const supported = supportedPlatforms.length === 0 || supportedPlatforms.indexOf(platform) >= 0;
1087
+ const unsupported = unsupportedPlatforms.length > 0 && unsupportedPlatforms.indexOf(platform) >= 0;
1088
+
1089
+ return supported && !unsupported;
1090
+ };
1091
+
1092
+ /**
1093
+ * Checks if rule supports optimization
1094
+ *
1095
+ * @param rule
1096
+ * @returns {boolean}
1097
+ */
1098
+ const isOptimizationSupported = function (rule) {
1099
+ const { hint } = rule;
1100
+ if (!hint || !hint.startsWith(HINT_MASK)) {
1101
+ return true;
1102
+ }
1103
+
1104
+ const stripped = hint.substring(HINT_MASK.length).trim();
1105
+
1106
+ return !stripped.includes(NOT_OPTIMIZED_HINT$1);
1107
+ };
1108
+
1109
+ /**
1110
+ * Checks if rule should be omitted with specified configuration
1111
+ *
1112
+ * @param rule
1113
+ * @param config
1114
+ * @param filterId
1115
+ * @returns {boolean}
1116
+ */
1117
+ const shouldOmitRule = function (rule, config, filterId) {
1118
+ const ruleText = rule.rule;
1119
+
1120
+ if (!ruleText) {
1121
+ return true;
1122
+ }
1123
+
1124
+ // Omit rules by filtration settings
1125
+ if (!config.configuration.ignoreRuleHints && !isPlatformSupported(rule, config.platform)) {
1126
+ logger.info(`${ruleText} removed with platform hint ${rule.hint}`);
1127
+ return true;
1128
+ }
1129
+
1130
+ if (config.configuration.removeRulePatterns) {
1131
+ // eslint-disable-next-line no-restricted-syntax
1132
+ for (const pattern of config.configuration.removeRulePatterns) {
1133
+ if (ruleText.match(new RegExp(pattern))) {
1134
+ // eslint-disable-next-line max-len
1135
+ logger.info(`${ruleText} removed with removeRulePattern ${pattern} in filter ${filterId} for ${config.platform} platform`);
1136
+ return true;
1137
+ }
1138
+ }
1139
+ }
1140
+
1141
+ return false;
1142
+ };
1143
+
1144
+ /**
1145
+ * Checks if rule should be omitted with specified configuration
1146
+ *
1147
+ * @param ruleLine
1148
+ * @param {OptimizationConfig} optimizationConfig
1149
+ */
1150
+ const shouldOmitRuleWithOptimization = function (ruleLine, optimizationConfig) {
1151
+ const ruleText = ruleLine.rule;
1152
+
1153
+ if (!ruleText) {
1154
+ return true;
1155
+ }
1156
+
1157
+ if (!isOptimizationSupported(ruleLine)) {
1158
+ return false;
1159
+ }
1160
+
1161
+ return skipRuleWithOptimization(ruleText, optimizationConfig);
1162
+ };
1163
+
1164
+ /**
1165
+ * We want to be sure that our mobile optimization is correct and didn't remove valuable rules
1166
+ *
1167
+ * @param filterId
1168
+ * @param rules
1169
+ * @param optimizedRules
1170
+ * @param {OptimizationConfig} optimizationConfig
1171
+ */
1172
+ const isOptimizationCorrect = function (filterId, rules, optimizedRules, optimizationConfig) {
1173
+ const filterRulesCount = rules.length;
1174
+ const optimizedRulesCount = optimizedRules.length;
1175
+
1176
+ // do not count decimal part of number
1177
+ const resultOptimizationPercent = Math.floor((optimizedRulesCount / filterRulesCount) * 100);
1178
+
1179
+ const expectedOptimizationPercent = optimizationConfig.percent;
1180
+ const minOptimizationPercent = optimizationConfig.minPercent;
1181
+ const maxOptimizationPercent = optimizationConfig.maxPercent;
1182
+ const { strict } = optimizationConfig;
1183
+
1184
+ const tooLow = resultOptimizationPercent < minOptimizationPercent;
1185
+ const tooHigh = resultOptimizationPercent > maxOptimizationPercent;
1186
+
1187
+ const incorrect = tooLow || tooHigh;
1188
+
1189
+ if (incorrect) {
1190
+ const message = `Unable to optimize filter ${filterId} with configuration`
1191
+ + `[~=${expectedOptimizationPercent}%, min=${minOptimizationPercent}%, max=${maxOptimizationPercent}%],`
1192
+ + `calculated = ${resultOptimizationPercent.toFixed(2)}%! `
1193
+ + `Filter rules count: ${filterRulesCount}. Optimized rules count: ${optimizedRulesCount}.`;
1194
+ if (strict) {
1195
+ throw new Error(message);
1196
+ } else {
1197
+ logger.error(message);
1198
+ }
1199
+ }
1200
+
1201
+ logger.info(`Filter ${filterId} optimization: ${filterRulesCount} => ${optimizedRulesCount},`
1202
+ + `${expectedOptimizationPercent}% => ${resultOptimizationPercent}%.`);
1203
+
1204
+ return !incorrect;
1205
+ };
1206
+
1207
+ /**
1208
+ * Filters set of rules with configuration
1209
+ *
1210
+ * @param rules
1211
+ * @param filterId
1212
+ * @param config
1213
+ */
1214
+ const cleanupRules = (rules, config, filterId) => {
1215
+ const ruleLines = splitRuleHintLines(rules, config.platform);
1216
+
1217
+ const filtered = ruleLines.filter((r) => !shouldOmitRule(r, config, filterId));
1218
+
1219
+ return joinRuleHintLines(filtered);
1220
+ };
1221
+
1222
+ /**
1223
+ * Filters set of rules with configuration and optimization
1224
+ *
1225
+ * @param rules
1226
+ * @param config
1227
+ * @param {OptimizationConfig} optimizationConfig
1228
+ * @param filterId
1229
+ */
1230
+ const cleanupAndOptimizeRules = function (rules, config, optimizationConfig, filterId) {
1231
+ config.configuration.removeRulePatterns = config.configuration.removeRulePatterns || [];
1232
+ config.configuration.removeRulePatterns.push(COMMENT_REGEXP);
1233
+
1234
+ const ruleLines = splitRuleHintLines(rules, config.platform);
1235
+
1236
+ const filtered = ruleLines.filter((r) => !shouldOmitRule(r, config, filterId));
1237
+
1238
+ const optimized = filtered.filter((r) => !shouldOmitRuleWithOptimization(r, optimizationConfig));
1239
+
1240
+ let result;
1241
+ // We check that our optimization is correct and didn't remove valuable rules
1242
+ // We do it via comparing expected optimization percent
1243
+ // with real (ratio between optimized rules number and all rules number)
1244
+ if (optimizationConfig && !isOptimizationCorrect(filterId, filtered, optimized, optimizationConfig)) {
1245
+ // Back to default OPTIMIZATION
1246
+ result = joinRuleHintLines(filtered);
1247
+ } else {
1248
+ result = joinRuleHintLines(optimized);
1249
+ }
1250
+
1251
+ config.configuration.removeRulePatterns.pop();
1252
+ return result;
1253
+ };
1254
+
1255
+ /* eslint-disable global-require */
1256
+
1257
+ const __dirname$3 = path.dirname(new URL(import.meta.url).pathname);
1258
+
1259
+ const RULES_SEPARATOR = '\r\n';
1260
+ let filterIdsPool = [];
1261
+ const metadataFilterIdsPool = [];
1262
+
1263
+ const OPTIMIZED_PLATFORMS_LIST = ['ext_safari', 'android', 'ios'];
1264
+
1265
+ const PLATFORM_FILTERS_DIR = 'filters';
1266
+ const FILTERS_METADATA_FILE_JSON = 'filters.json';
1267
+ const FILTERS_I18N_METADATA_FILE_JSON = 'filters_i18n.json';
1268
+
1269
+ // AG-20175
1270
+ const FILTERS_METADATA_FILE_JS = 'filters.js';
1271
+ const FILTERS_I18N_METADATA_FILE_JS = 'filters_i18n.js';
1272
+
1273
+ /**
1274
+ * Tag id for obsolete filters.
1275
+ *
1276
+ * @see {@link https://github.com/AdguardTeam/FiltersRegistry/blob/85f2e9f6f5f7c04797017e713dc5986b22a78840/tags/metadata.json#L184}
1277
+ */
1278
+ const OBSOLETE_TAG_ID = 46;
1279
+
1280
+ /**
1281
+ * From 1 to 99 we have AdGuard filters
1282
+ *
1283
+ * @type {number}
1284
+ */
1285
+ const LAST_ADGUARD_FILTER_ID = 99;
1286
+ const LOCAL_SCRIPT_RULES_FILE = 'local_script_rules.txt';
1287
+ const LOCAL_SCRIPT_RULES_FILE_JSON = 'local_script_rules.json';
1288
+
1289
+ const LOCAL_SCRIPT_RULES_COMMENT = 'By the rules of AMO and addons.opera.com we cannot use remote scripts'
1290
+ + '(and our JS injection rules could be counted as remote scripts).\r\n'
1291
+ + 'So what we do:\r\n'
1292
+ + '1. We gather all current JS rules in the DEFAULT_SCRIPT_RULES object'
1293
+ + '(see lib/utils/local-script-rules.js)\r\n'
1294
+ + '2. We disable JS rules got from remote server\r\n'
1295
+ + '3. We allow only custom rules got from the User filter (which user creates manually)'
1296
+ + 'or from this DEFAULT_SCRIPT_RULES object';
1297
+
1298
+ const ONE_HOUR_SEC = 60 * 60;
1299
+ const ONE_DAY_SEC = 24 * ONE_HOUR_SEC;
1300
+
1301
+ /**
1302
+ * Default value of filter expiration time
1303
+ * if impossible to parse a value specified in platforms.json or in filter metadata.
1304
+ *
1305
+ * Defaults to 1 day (or 86400 in seconds).
1306
+ */
1307
+ const DEFAULT_EXPIRES_SEC = 1 * ONE_DAY_SEC;
1308
+
1309
+ /**
1310
+ * Platforms configurations
1311
+ */
1312
+ let platformPathsConfig = null;
1313
+ let filterFile = null;
1314
+ let metadataFile = null;
1315
+ let revisionFile = null;
1316
+ let adguardFiltersServerUrl = null;
1317
+
1318
+ /**
1319
+ * Sync reads file content
1320
+ *
1321
+ * @param path
1322
+ * @returns {*}
1323
+ */
1324
+ const readFile$2 = function (path) {
1325
+ try {
1326
+ return fs.readFileSync(path, { encoding: 'utf-8' });
1327
+ } catch (e) {
1328
+ return null;
1329
+ }
1330
+ };
1331
+
1332
+ /**
1333
+ * Creates header contents
1334
+ *
1335
+ * @param metadataFile
1336
+ * @param revisionFile
1337
+ * @param platformsJsonExpires
1338
+ * @returns {[*,*,*,*,string]}
1339
+ */
1340
+ const makeHeader = function (metadataFile, revisionFile, platformsJsonExpires) {
1341
+ const metadataString = readFile$2(metadataFile);
1342
+ if (!metadataString) {
1343
+ throw new Error('Error reading metadata');
1344
+ }
1345
+
1346
+ const metadata = JSON.parse(metadataString);
1347
+
1348
+ const revisionString = readFile$2(revisionFile);
1349
+ if (!revisionString) {
1350
+ throw new Error('Error reading revision');
1351
+ }
1352
+
1353
+ const revision = JSON.parse(revisionString);
1354
+
1355
+ const expires = typeof platformsJsonExpires !== 'undefined'
1356
+ ? platformsJsonExpires
1357
+ : metadata.expires;
1358
+
1359
+ return [
1360
+ `! Title: ${metadata.name}`,
1361
+ `! Description: ${metadata.description}`,
1362
+ `! Version: ${revision.version}`,
1363
+ `! TimeUpdated: ${moment(revision.timeUpdated).format()}`,
1364
+ `! Expires: ${expires} (update frequency)`,
1365
+ ];
1366
+ };
1367
+
1368
+ /**
1369
+ * Strips spec character from end of string
1370
+ *
1371
+ * @param string
1372
+ * @param char
1373
+ */
1374
+ const stripEnd = function (string, char) {
1375
+ if (string.endsWith(char)) {
1376
+ return stripEnd(string.substring(0, string.length - 1), char);
1377
+ }
1378
+ return string;
1379
+ };
1380
+
1381
+ /**
1382
+ * Checks if filter id is unique
1383
+ *
1384
+ * @param pool
1385
+ * @param filterId
1386
+ */
1387
+ const checkFilterId = function (pool, filterId) {
1388
+ if (pool.indexOf(filterId) >= 0) {
1389
+ throw new Error(`Invalid filters: Filter identifier is not unique: ${filterId}`);
1390
+ }
1391
+
1392
+ pool.push(filterId);
1393
+ };
1394
+
1395
+ /**
1396
+ * Replaces newlines
1397
+ *
1398
+ * @param message
1399
+ * @returns {XML|string|*}
1400
+ */
1401
+ const normalizeData = function (message) {
1402
+ message = message.replace(/\r/g, '');
1403
+ message = message.replace(/\n+/g, '\n');
1404
+ return message;
1405
+ };
1406
+
1407
+ /**
1408
+ * Calculates checksum
1409
+ * See:
1410
+ * https://adblockplus.org/en/filters#special-comments
1411
+ * https://hg.adblockplus.org/adblockplus/file/tip/addChecksum.py
1412
+ *
1413
+ * @param header Header lines
1414
+ * @param rules Rules lines
1415
+ */
1416
+ const calculateChecksum = function (header, rules) {
1417
+ let content = header.concat(rules).join('\n');
1418
+ content = normalizeData(content);
1419
+ const checksum = crypto.createHash('md5').update(content).digest('base64');
1420
+
1421
+ return `! Checksum: ${stripEnd(checksum.trim(), '=')}`;
1422
+ };
1423
+
1424
+ /**
1425
+ * This will create a dir given a path such as './folder/subfolder'
1426
+ * @param {string} dir - The path of the directory to create. Can be absolute or relative.
1427
+ * @returns {string} The path of the last directory created or the existing directory.
1428
+ * @throws {Error} Throws an error if there is a permission issue or if the directory cannot be created.
1429
+ */
1430
+ const createDir = (dir) => {
1431
+ const { sep } = path;
1432
+ const initDir = path.isAbsolute(dir) ? sep : '';
1433
+ // eslint-disable-next-line no-undef
1434
+ const baseDir = __dirname$3;
1435
+
1436
+ return dir.split(sep).reduce((parentDir, childDir) => {
1437
+ const curDir = path.resolve(baseDir, parentDir, childDir);
1438
+ try {
1439
+ fs.mkdirSync(curDir);
1440
+ } catch (err) {
1441
+ if (err.code === 'EEXIST') { // curDir already exists!
1442
+ return curDir;
1443
+ }
1444
+
1445
+ // To avoid `EISDIR` error on Mac and `EACCES`-->`ENOENT` and `EPERM` on Windows.
1446
+ if (err.code === 'ENOENT') { // Throw the original parentDir error on curDir `ENOENT` failure.
1447
+ throw new Error(`EACCES: permission denied, mkdir '${parentDir}'`);
1448
+ }
1449
+
1450
+ const caughtErr = ['EACCES', 'EPERM', 'EISDIR'].indexOf(err.code) > -1;
1451
+ if ((!caughtErr || caughtErr) && (curDir === path.resolve(dir))) {
1452
+ throw err; // Throw if it's just the last created dir.
1453
+ }
1454
+ }
1455
+
1456
+ return curDir;
1457
+ }, initDir);
1458
+ };
1459
+
1460
+ /**
1461
+ * Replaces tag keywords in the provided filters with their corresponding tag IDs.
1462
+ *
1463
+ * @param {Array<Object>} rawFilters - The array of filter objects to process.
1464
+ * @param {Array<Object>} tags - The array of tag objects.
1465
+ * @returns {Array<Object>} A new array of filter objects with tag keywords replaced by their corresponding tag IDs.
1466
+ * @throws {Error} If any tag keyword in the filters does not have a corresponding tag in the `tags` array.
1467
+ */
1468
+ const replaceTagKeywords = function (rawFilters, tags) {
1469
+ const tagsMap = new Map();
1470
+
1471
+ tags.forEach((tag) => {
1472
+ tagsMap.set(tag.keyword, tag.tagId);
1473
+ });
1474
+
1475
+ // create new variable to avoid mutation of input parameters
1476
+ const filters = [];
1477
+ const lostTags = [];
1478
+ // eslint-disable-next-line no-restricted-syntax
1479
+ for (const filter of rawFilters) {
1480
+ const newFilter = { ...filter };
1481
+ if (newFilter.tags) {
1482
+ const ids = [];
1483
+ // eslint-disable-next-line no-restricted-syntax
1484
+ for (const t of newFilter.tags) {
1485
+ const id = tagsMap.get(t);
1486
+ if (id) {
1487
+ ids.push(id);
1488
+ } else {
1489
+ logger.error(`Missing tag with keyword: ${t}`);
1490
+ lostTags.push(t);
1491
+ }
1492
+ }
1493
+
1494
+ delete newFilter.tags;
1495
+ newFilter.tags = ids;
1496
+ }
1497
+ filters.push(newFilter);
1498
+ }
1499
+
1500
+ if (lostTags.length > 0) {
1501
+ throw new Error(`Missing tag with keyword: ${lostTags.join(', ')}`);
1502
+ }
1503
+
1504
+ return filters;
1505
+ };
1506
+
1507
+ /**
1508
+ * Converts `rawExpires` with `day` marker into **seconds**.
1509
+ *
1510
+ * @param {any} rawExpires Raw `expires` value from filter metadata or platforms.json.
1511
+ *
1512
+ * @returns {number} Parsed expires value from days to seconds,
1513
+ * or {@link DEFAULT_EXPIRES_SEC} if it cannot be parsed.
1514
+ */
1515
+ const convertExpiresDaysToSeconds = (rawExpires) => {
1516
+ const expiresDays = parseInt(rawExpires, 10);
1517
+ if (Number.isNaN(expiresDays)) {
1518
+ return DEFAULT_EXPIRES_SEC;
1519
+ }
1520
+ return expiresDays * ONE_DAY_SEC;
1521
+ };
1522
+
1523
+ /**
1524
+ * Converts `rawExpires` with `hour` marker into **seconds**.
1525
+ *
1526
+ * @param {any} rawExpires Raw `expires` value from filter metadata or platforms.json.
1527
+ *
1528
+ * @returns {number} Parsed expires value from hours to seconds,
1529
+ * or {@link DEFAULT_EXPIRES_SEC} if it cannot be parsed.
1530
+ */
1531
+ const convertExpiresHoursToSeconds = (rawExpires) => {
1532
+ const expiresHours = parseInt(rawExpires, 10);
1533
+ if (Number.isNaN(expiresHours)) {
1534
+ return DEFAULT_EXPIRES_SEC;
1535
+ }
1536
+ return expiresHours * ONE_HOUR_SEC;
1537
+ };
1538
+
1539
+ /**
1540
+ * Overrides filter metadata `expires` property with platforms.json's `expires` property.
1541
+ *
1542
+ * Then parses the value and converts it to **seconds**.
1543
+ * If it cannot be parsed or not set at all, the default value {@link DEFAULT_EXPIRES_SEC} is used.
1544
+ *
1545
+ * @example
1546
+ * `12 hours` → 43200
1547
+ * `1 day` → 86400
1548
+ * `2 days` → 172800
1549
+ *
1550
+ * @param rawFilters Input filters' metadata.
1551
+ * @param platformsJsonExpires Platforms.json's `expires` property to override filters' `expires`.
1552
+ *
1553
+ * @returns {Array<object>} Updated filters' metadata with `expires` property in **seconds**.
1554
+ */
1555
+ const replaceExpires = function (rawFilters, platformsJsonExpires) {
1556
+ const filters = [];
1557
+ // eslint-disable-next-line no-restricted-syntax
1558
+ for (const filter of rawFilters) {
1559
+ // do not mutate input parameters
1560
+ const newFilter = { ...filter };
1561
+ // expires value which is set in platforms.json has higher priority
1562
+ // https://github.com/AdguardTeam/FiltersCompiler/issues/198
1563
+ if (typeof platformsJsonExpires !== 'undefined') {
1564
+ newFilter.expires = platformsJsonExpires;
1565
+ }
1566
+
1567
+ if (newFilter.expires) {
1568
+ if (newFilter.expires.indexOf('day') > 0) {
1569
+ newFilter.expires = convertExpiresDaysToSeconds(newFilter.expires);
1570
+ } else if (newFilter.expires.indexOf('hour') > 0) {
1571
+ newFilter.expires = convertExpiresHoursToSeconds(newFilter.expires);
1572
+ }
1573
+ } else {
1574
+ // use default value if 'expires' is not set either in platforms.json or in filter metadata
1575
+ newFilter.expires = DEFAULT_EXPIRES_SEC;
1576
+ }
1577
+ filters.push(newFilter);
1578
+ }
1579
+
1580
+ return filters;
1581
+ };
1582
+
1583
+ /**
1584
+ * First step of processing filters metadata
1585
+ *
1586
+ * @param filtersMetadata
1587
+ */
1588
+ const processFiltersFromMetadata = function (filtersMetadata) {
1589
+ // do not mutate input parameters
1590
+ const filters = [];
1591
+ // eslint-disable-next-line no-restricted-syntax
1592
+ for (const filter of filtersMetadata) {
1593
+ const newFilter = {
1594
+ ...filter,
1595
+ deprecated: Boolean(filter.deprecated),
1596
+ };
1597
+
1598
+ /**
1599
+ * In case of backward compatibility
1600
+ * Adds 'languages' metadata field parsed from 'lang:' tags
1601
+ */
1602
+ if (newFilter.tags) {
1603
+ const filterLanguages = [];
1604
+ let hasRecommended = false;
1605
+ // eslint-disable-next-line no-restricted-syntax
1606
+ for (const t of newFilter.tags) {
1607
+ if (!hasRecommended && t === 'recommended') {
1608
+ hasRecommended = true;
1609
+ }
1610
+
1611
+ if (t.startsWith('lang:')) {
1612
+ filterLanguages.push(t.substring(5));
1613
+ }
1614
+ }
1615
+
1616
+ // Languages will be added for recommended filters only
1617
+ newFilter.languages = hasRecommended ? filterLanguages : [];
1618
+ }
1619
+
1620
+ filters.push(newFilter);
1621
+ }
1622
+
1623
+ return filters;
1624
+ };
1625
+
1626
+ /**
1627
+ * Does several things with filters URLs:
1628
+ * 1. Rewrites subscription urls for specified platform config
1629
+ * 2. Adds downloadUrl field to the filter
1630
+ *
1631
+ * @param metadata
1632
+ * @param config
1633
+ */
1634
+ const postProcessUrls = (metadata, config) => {
1635
+ const useOptimized = OPTIMIZED_PLATFORMS_LIST.indexOf(config.platform) >= 0;
1636
+
1637
+ const result = {};
1638
+
1639
+ result.groups = metadata.groups.slice(0);
1640
+ result.tags = metadata.tags.slice(0);
1641
+ result.filters = [];
1642
+
1643
+ const platformPath = config.path;
1644
+ // eslint-disable-next-line no-restricted-syntax
1645
+ for (const f of metadata.filters) {
1646
+ const fileName = `${f.filterId}${useOptimized ? '_optimized' : ''}.txt`;
1647
+ const downloadUrl = `${adguardFiltersServerUrl}${platformPath}/filters/${fileName}`;
1648
+
1649
+ const copy = { ...f, downloadUrl };
1650
+
1651
+ if (copy.subscriptionUrl && copy.subscriptionUrl.startsWith(adguardFiltersServerUrl)) {
1652
+ copy.subscriptionUrl = downloadUrl;
1653
+ }
1654
+
1655
+ result.filters.push(copy);
1656
+ }
1657
+
1658
+ return result;
1659
+ };
1660
+
1661
+ /**
1662
+ * Removes redundant metadata for included or excluded filters for the current platform
1663
+ *
1664
+ * @param metadata
1665
+ * @param platform
1666
+ *
1667
+ * @returns {object} Metadata with filtered `filters` array due to the `platform`.
1668
+ */
1669
+ const removeRedundantFiltersMetadata = (metadata, platform) => {
1670
+ // leaves only included filters metadata
1671
+ metadata.filters = metadata.filters.filter((filter) => !filter.platformsIncluded
1672
+ || (filter.platformsIncluded && filter.platformsIncluded.includes(platform)));
1673
+ // removes excluded filters metadata
1674
+ metadata.filters = metadata.filters.filter((filter) => !filter.platformsExcluded
1675
+ || (filter.platformsExcluded && !filter.platformsExcluded.includes(platform)));
1676
+
1677
+ return metadata;
1678
+ };
1679
+
1680
+ /**
1681
+ * Parses object info
1682
+ * Splits string {mask}{id}.{message} like "group.1.name" etc.
1683
+ *
1684
+ * @param string
1685
+ * @param mask
1686
+ * @returns {{id: *, message: *}}
1687
+ */
1688
+ const parseInfo = (string, mask) => {
1689
+ const searchIndex = string.indexOf(mask) + mask.length;
1690
+
1691
+ return {
1692
+ id: string.substring(searchIndex, string.indexOf('.', searchIndex)),
1693
+ message: string.substring(string.lastIndexOf('.') + 1),
1694
+ };
1695
+ };
1696
+
1697
+ /**
1698
+ * Loads locale data from a specified directory and organizes it into groups, tags, and filters.
1699
+ *
1700
+ * @param {string} dir - The directory containing locale subdirectories with JSON files.
1701
+ * @returns {Object} An object containing the loaded locale data.
1702
+ */
1703
+ const loadLocales = function (dir) {
1704
+ const result = {
1705
+ groups: {},
1706
+ tags: {},
1707
+ filters: {},
1708
+ };
1709
+
1710
+ const locales = fs.readdirSync(dir);
1711
+ // eslint-disable-next-line no-restricted-syntax
1712
+ for (const directory of locales) {
1713
+ const localeDir = path.join(dir, directory);
1714
+ if (fs.lstatSync(localeDir).isDirectory()) {
1715
+ const groups = JSON.parse(readFile$2(path.join(localeDir, 'groups.json')));
1716
+ if (groups) {
1717
+ // eslint-disable-next-line no-restricted-syntax
1718
+ for (const group of groups) {
1719
+ // eslint-disable-next-line guard-for-in,no-restricted-syntax
1720
+ for (const p in group) {
1721
+ const info = parseInfo(p, 'group.');
1722
+ if (!info || !info.id) {
1723
+ continue;
1724
+ }
1725
+
1726
+ const { id } = info;
1727
+ result.groups[id] = result.groups[id] || {};
1728
+ result.groups[id][directory] = result.groups[id][directory] || {};
1729
+ result.groups[id][directory][info.message] = group[p];
1730
+ }
1731
+ }
1732
+ }
1733
+
1734
+ const tags = JSON.parse(readFile$2(path.join(localeDir, 'tags.json')));
1735
+ if (tags) {
1736
+ // eslint-disable-next-line no-restricted-syntax
1737
+ for (const tag of tags) {
1738
+ // eslint-disable-next-line guard-for-in,no-restricted-syntax
1739
+ for (const p in tag) {
1740
+ const info = parseInfo(p, 'tag.');
1741
+ if (!info || !info.id) {
1742
+ continue;
1743
+ }
1744
+
1745
+ const { id } = info;
1746
+ result.tags[id] = result.tags[id] || {};
1747
+ result.tags[id][directory] = result.tags[id][directory] || {};
1748
+ result.tags[id][directory][info.message] = tag[p];
1749
+ }
1750
+ }
1751
+ }
1752
+
1753
+ const filters = JSON.parse(readFile$2(path.join(localeDir, 'filters.json')));
1754
+ if (filters) {
1755
+ // eslint-disable-next-line no-restricted-syntax
1756
+ for (const filter of filters) {
1757
+ // eslint-disable-next-line guard-for-in,no-restricted-syntax
1758
+ for (const p in filter) {
1759
+ const info = parseInfo(p, 'filter.');
1760
+ if (!info || !info.id) {
1761
+ continue;
1762
+ }
1763
+
1764
+ const { id } = info;
1765
+ result.filters[id] = result.filters[id] || {};
1766
+ result.filters[id][directory] = result.filters[id][directory] || {};
1767
+ result.filters[id][directory][info.message] = filter[p];
1768
+ }
1769
+ }
1770
+ }
1771
+ }
1772
+ }
1773
+
1774
+ return result;
1775
+ };
1776
+
1777
+ /**
1778
+ * Excludes obsolete filters data from localizations
1779
+ * @param {object} localizationsFilters
1780
+ * @param {array} obsoleteFilters
1781
+ * @return {object} result
1782
+ */
1783
+ const excludeObsoleteFilters = (localizationsFilters, obsoleteFilters) => {
1784
+ const result = { ...localizationsFilters };
1785
+ obsoleteFilters.forEach((filter) => {
1786
+ delete result[filter.filterId];
1787
+ });
1788
+ return result;
1789
+ };
1790
+
1791
+ /**
1792
+ * Removes obsolete filters metadata
1793
+ * @param {object} metadata
1794
+ * @return {object} result
1795
+ */
1796
+ const removeObsoleteFilters = (metadata) => {
1797
+ const result = { ...metadata };
1798
+ result.filters = metadata.filters.filter((filter) => !filter.tags.includes(OBSOLETE_TAG_ID));
1799
+ return result;
1800
+ };
1801
+
1802
+ /**
1803
+ * Sorts metadata's filters by `filterId` property.
1804
+ *
1805
+ * @param {object} metadata Metadata to sort filters for.
1806
+ * @returns {object} Metadata with sorted filters.
1807
+ */
1808
+ const sortMetadataFilters = (metadata) => {
1809
+ const result = { ...metadata };
1810
+ result.filters.sort((a, b) => a.filterId - b.filterId);
1811
+ return result;
1812
+ };
1813
+
1814
+ /**
1815
+ * Checks whether the filter should be built for the specified platform.
1816
+ *
1817
+ * @param {object} metadata Filter metadata.
1818
+ * @param {string} platform Platform.
1819
+ *
1820
+ * @returns True if
1821
+ * - both `platformsIncluded` and `platformsExcluded` properties are not defined in the `metadata`,
1822
+ * - `platformsExcluded` does not contain the specified `platform`,
1823
+ * - `platformsIncluded` contains the specified `platform`.
1824
+ *
1825
+ * @throws An error if both `platformsIncluded` and `platformsExcluded` are defined.
1826
+ */
1827
+ const shouldBuildFilterForPlatform = (metadata, platform) => {
1828
+ const { filterId, platformsExcluded, platformsIncluded } = metadata;
1829
+
1830
+ if (platformsExcluded && platformsIncluded) {
1831
+ let errorMessage = 'Both platformsIncluded and platformsExcluded cannot be defined simultaneously';
1832
+ if (filterId) {
1833
+ errorMessage += ` for filter ${filterId}`;
1834
+ }
1835
+ throw new Error(errorMessage);
1836
+ }
1837
+
1838
+ if (platformsExcluded && platformsExcluded.includes(platform)) {
1839
+ return false;
1840
+ }
1841
+
1842
+ if (!platformsIncluded) {
1843
+ return true;
1844
+ }
1845
+
1846
+ return platformsIncluded.includes(platform);
1847
+ };
1848
+
1849
+ /**
1850
+ * Removes group descriptions from the input groups.
1851
+ *
1852
+ * @param {Object} inputGroups Input groups metadata.
1853
+ * @returns {Object} Output groups metadata.
1854
+ */
1855
+ const removeGroupDescriptions = (inputGroups) => {
1856
+ const result = {};
1857
+ Object.keys(inputGroups).forEach((groupId) => {
1858
+ result[groupId] = {};
1859
+
1860
+ Object.keys(inputGroups[groupId]).forEach((locale) => {
1861
+ const localeData = inputGroups[groupId][locale];
1862
+ const cleanedLocaleData = { ...localeData };
1863
+
1864
+ if (cleanedLocaleData.description) {
1865
+ delete cleanedLocaleData.description;
1866
+ }
1867
+
1868
+ result[groupId][locale] = cleanedLocaleData;
1869
+ });
1870
+ });
1871
+
1872
+ return result;
1873
+ };
1874
+
1875
+ /**
1876
+ * Writes filters metadata and localizations for different platforms.
1877
+ *
1878
+ * @param {string} platformsPath - The base path where platform-specific directories are located.
1879
+ * @param {string} filtersDir - The directory containing filters metadata and related files.
1880
+ * @param {Array<Object>} filtersMetadata - The metadata for filters to be processed.
1881
+ * @param {Array<string>} obsoleteFilters - A list of obsolete filters to be excluded.
1882
+ *
1883
+ * @returns {void}
1884
+ * @throws {Error} If reading or parsing metadata files fails.
1885
+ */
1886
+ const writeFiltersMetadata = function (platformsPath, filtersDir, filtersMetadata, obsoleteFilters) {
1887
+ logger.info('Writing filters metadata');
1888
+
1889
+ const groups = JSON.parse(readFile$2(path.join(filtersDir, '../groups', 'metadata.json')));
1890
+ if (!groups) {
1891
+ logger.error('Error reading groups metadata');
1892
+ return;
1893
+ }
1894
+
1895
+ const tags = JSON.parse(readFile$2(path.join(filtersDir, '../tags', 'metadata.json')));
1896
+ if (!tags) {
1897
+ logger.error('Error reading tags metadata');
1898
+ return;
1899
+ }
1900
+
1901
+ // do not mutate input parameters
1902
+ const parsedLangTagsFiltersMetadata = processFiltersFromMetadata(filtersMetadata);
1903
+ const replacedTagKeywordsFiltersMetadata = replaceTagKeywords(parsedLangTagsFiltersMetadata, tags);
1904
+
1905
+ const localizations = loadLocales(path.join(filtersDir, '../locales'));
1906
+
1907
+ // eslint-disable-next-line guard-for-in,no-restricted-syntax
1908
+ for (const platform in platformPathsConfig) {
1909
+ const config = platformPathsConfig[platform];
1910
+ const platformDir = path.join(platformsPath, config.path);
1911
+ createDir(platformDir);
1912
+
1913
+ logger.info(`Writing filters metadata: ${config.path}`);
1914
+ const filtersFileJson = path.join(platformDir, FILTERS_METADATA_FILE_JSON);
1915
+ const filtersFileJs = path.join(platformDir, FILTERS_METADATA_FILE_JS);
1916
+
1917
+ const replacedExpiresFiltersMetadata = replaceExpires(replacedTagKeywordsFiltersMetadata, config.expires);
1918
+
1919
+ let metadata = {
1920
+ groups,
1921
+ tags,
1922
+ filters: replacedExpiresFiltersMetadata,
1923
+ };
1924
+
1925
+ metadata = postProcessUrls(metadata, config);
1926
+ metadata = removeRedundantFiltersMetadata(metadata, config.platform);
1927
+
1928
+ if (platform === 'MAC') {
1929
+ metadata = rewriteMetadataForOldMacV1(metadata);
1930
+ } else if (platform === 'MAC_V2') {
1931
+ metadata = rewriteMetadataForOldMacV2(metadata);
1932
+ metadata = removeObsoleteFilters(metadata);
1933
+ } else {
1934
+ metadata = removeObsoleteFilters(metadata);
1935
+ }
1936
+
1937
+ const filtersContent = JSON.stringify(sortMetadataFilters(metadata), null, '\t');
1938
+
1939
+ fs.writeFileSync(filtersFileJson, filtersContent, 'utf8');
1940
+ fs.writeFileSync(filtersFileJs, filtersContent, 'utf8');
1941
+
1942
+ logger.info(`Writing filters localizations: ${config.path}`);
1943
+ const filtersI18nFileJson = path.join(platformDir, FILTERS_I18N_METADATA_FILE_JSON);
1944
+ const filtersI18nFileJs = path.join(platformDir, FILTERS_I18N_METADATA_FILE_JS);
1945
+
1946
+ let localizedFilters = { ...localizations.filters };
1947
+
1948
+ filtersMetadata.forEach((metadata) => {
1949
+ if (!shouldBuildFilterForPlatform(metadata, config.platform)) {
1950
+ const { filterId } = metadata;
1951
+ // eslint-disable-next-line max-len
1952
+ logger.info(`Adding localization for filter ${filterId} skipped for platform '${config.platform}' due to platformsExcluded or platformsIncluded`);
1953
+ delete localizedFilters[filterId];
1954
+ }
1955
+ });
1956
+
1957
+ // old MAC platform may not support absence some filters i18n metadata,
1958
+ // for all other platforms we can exclude obsolete filters
1959
+ if (platform !== 'MAC') {
1960
+ localizedFilters = excludeObsoleteFilters(localizedFilters, obsoleteFilters);
1961
+ }
1962
+
1963
+ const i18nMetadata = {
1964
+ groups: localizations.groups,
1965
+ tags: localizations.tags,
1966
+ filters: localizedFilters,
1967
+ };
1968
+
1969
+ let i18nGroups = localizations.groups;
1970
+
1971
+ // no new fields should be added for old 'MAC' platform
1972
+ if (platform === 'MAC') {
1973
+ delete i18nMetadata.tags;
1974
+ i18nGroups = removeGroupDescriptions(localizations.groups);
1975
+ }
1976
+
1977
+ i18nMetadata.groups = i18nGroups;
1978
+
1979
+ const i18nContent = JSON.stringify(i18nMetadata, null, '\t');
1980
+
1981
+ fs.writeFileSync(filtersI18nFileJson, i18nContent, 'utf8');
1982
+ fs.writeFileSync(filtersI18nFileJs, i18nContent, 'utf8');
1983
+ }
1984
+
1985
+ logger.info('Writing filters metadata done');
1986
+ };
1987
+
1988
+ /**
1989
+ * Separates script rules from AG filters into specified file.
1990
+ *
1991
+ * @param platformsPath - Path to platforms folder
1992
+ */
1993
+ const writeLocalScriptRules = function (platformsPath) {
1994
+ logger.info('Writing local script rules');
1995
+
1996
+ // eslint-disable-next-line guard-for-in,no-restricted-syntax
1997
+ for (const platform in platformPathsConfig) {
1998
+ const config = platformPathsConfig[platform];
1999
+ const platformDir = path.join(platformsPath, config.path);
2000
+
2001
+ const rulesTxt = [];
2002
+ const rulesJson = {
2003
+ comment: LOCAL_SCRIPT_RULES_COMMENT,
2004
+ rules: [],
2005
+ };
2006
+
2007
+ // TODO: find a better way to iterate over ag filters
2008
+ // because AdGuard Chinese filter has id 224
2009
+ // https://github.com/AdguardTeam/FiltersRegistry/blob/master/filters/filter_224_Chinese/metadata.json
2010
+ for (let i = 1; i <= LAST_ADGUARD_FILTER_ID; i += 1) {
2011
+ const filterRules = readFile$2(path.join(platformDir, PLATFORM_FILTERS_DIR, `${i}.txt`));
2012
+ if (!filterRules) {
2013
+ continue;
2014
+ }
2015
+
2016
+ const lines = filterRules.split('\n');
2017
+ // eslint-disable-next-line no-restricted-syntax
2018
+ for (let rule of lines) {
2019
+ rule = rule.trim();
2020
+
2021
+ if (!rule
2022
+ || rule.startsWith(RuleMasks.MASK_COMMENT)) {
2023
+ continue;
2024
+ }
2025
+
2026
+ if (rule.includes(RuleMasks.MASK_SCRIPT)) {
2027
+ rulesTxt.push(rule);
2028
+
2029
+ const m = rule.split(RuleMasks.MASK_SCRIPT);
2030
+ rulesJson.rules.push({
2031
+ domains: m[0],
2032
+ script: m[1],
2033
+ });
2034
+ }
2035
+ }
2036
+ }
2037
+
2038
+ // https://github.com/AdguardTeam/AdguardBrowserExtension/issues/1847
2039
+ // remove scriptlet rules in local_script_rules.json
2040
+ rulesJson.rules = removeScriptletRules(rulesJson.rules);
2041
+
2042
+ fs.writeFileSync(
2043
+ path.join(platformDir, LOCAL_SCRIPT_RULES_FILE),
2044
+ rulesTxt.join(RULES_SEPARATOR),
2045
+ 'utf8',
2046
+ );
2047
+ fs.writeFileSync(
2048
+ path.join(platformDir, LOCAL_SCRIPT_RULES_FILE_JSON),
2049
+ JSON.stringify(rulesJson, null, 4),
2050
+ 'utf8',
2051
+ );
2052
+ }
2053
+
2054
+ logger.info('Writing local script rules done');
2055
+ };
2056
+
2057
+ /**
2058
+ * Loads and processes filter metadata from the specified directory.
2059
+ *
2060
+ * @param {string} filterDir - The directory containing the filter metadata and revision files.
2061
+ * @param {number[]} [whitelist] - An optional array of whitelist filter IDs.
2062
+ * @param {number[]} [blacklist] - An optional array of blacklist filter IDs.
2063
+ * @returns {Object} The processed filter metadata.
2064
+ * @throws {Error} If the metadata or revision file cannot be read.
2065
+ */
2066
+ const loadFilterMetadata = function (filterDir, whitelist, blacklist) {
2067
+ const metadataFilePath = path.join(filterDir, metadataFile);
2068
+ const metadataString = readFile$2(metadataFilePath);
2069
+ if (!metadataString) {
2070
+ throw new Error(`Error reading filter metadata:${filterDir}`);
2071
+ }
2072
+
2073
+ const revisionFilePath = path.join(filterDir, revisionFile);
2074
+ const revisionString = readFile$2(revisionFilePath);
2075
+ if (!revisionString) {
2076
+ throw new Error(`Error reading filter revision:${filterDir}`);
2077
+ }
2078
+
2079
+ const revision = JSON.parse(revisionString);
2080
+
2081
+ const result = JSON.parse(metadataString);
2082
+ result.version = revision.version;
2083
+ result.timeUpdated = moment(revision.timeUpdated).format('YYYY-MM-DDTHH:mm:ssZZ');
2084
+ result.timeAdded = moment(result.timeAdded).format('YYYY-MM-DDTHH:mm:ssZZ');
2085
+ delete result.disabled;
2086
+
2087
+ const { filterId } = result;
2088
+ if (
2089
+ (whitelist && whitelist.includes(filterId))
2090
+ || (blacklist && !blacklist.includes(filterId))
2091
+ ) {
2092
+ checkFilterId(metadataFilterIdsPool, filterId);
2093
+ }
2094
+
2095
+ return result;
2096
+ };
2097
+
2098
+ /**
2099
+ * Exclude `#%#` and `#@%#` rules from rules list.
2100
+ *
2101
+ * @param {Array<any>} rules Input list of rules.
2102
+ * @return {Array<any>} Filtered list of rules.
2103
+ */
2104
+ const excludeScriptRules = (rules) => {
2105
+ return rules.filter((rule) => {
2106
+ return rule
2107
+ && !rule.includes(RuleMasks.MASK_SCRIPT)
2108
+ && !rule.includes(RuleMasks.MASK_SCRIPT_EXCEPTION);
2109
+ });
2110
+ };
2111
+
2112
+ /**
2113
+ * Calculates checksum and writes filter file
2114
+ *
2115
+ * @param filterFile
2116
+ * @param adbHeader
2117
+ * @param rulesHeader
2118
+ * @param rules
2119
+ */
2120
+ const writeFilterFile = function (filterFile, adbHeader, rulesHeader, rules) {
2121
+ let header = rulesHeader;
2122
+ if (adbHeader) {
2123
+ // Add Adb Plus compatibility header
2124
+ header = [adbHeader].concat(rulesHeader);
2125
+ }
2126
+
2127
+ const checksum = calculateChecksum(header, rules);
2128
+
2129
+ let data = [checksum].concat(rulesHeader).concat(rules);
2130
+ if (adbHeader) {
2131
+ data = [adbHeader].concat(data);
2132
+ }
2133
+
2134
+ fs.writeFileSync(filterFile, data.join(RULES_SEPARATOR), 'utf8');
2135
+ };
2136
+
2137
+ /**
2138
+ * Writes filter platform build
2139
+ */
2140
+ const writeFilterRules = function (filterId, dir, config, rulesHeader, rules, optimized) {
2141
+ createDir(dir);
2142
+
2143
+ const filterFile = path.join(dir, `${filterId}${optimized ? '_optimized' : ''}.txt`);
2144
+ let rulesList = rules;
2145
+
2146
+ // Convert Adguard scriptlets and redirect rules to UBlock syntax.
2147
+ // Exclude script rules
2148
+ // and script rules exceptions https://github.com/AdguardTeam/FiltersCompiler/issues/199
2149
+ // Modify title for base filter
2150
+ if (config.platform === 'ext_ublock') {
2151
+ rulesList = convertToUbo(rulesList);
2152
+ rulesList = excludeScriptRules(rulesList);
2153
+ if (filterId === 2) {
2154
+ modifyBaseFilterHeader(rulesHeader, optimized);
2155
+ }
2156
+ }
2157
+
2158
+ writeFilterFile(filterFile, config.configuration.adbHeader, rulesHeader, rulesList);
2159
+
2160
+ // For English filter only we should provide additional filter version.
2161
+ if (filterId === 2 && config.platform === 'ext_ublock' && !optimized) {
2162
+ const correctedHeader = rewriteHeader(rulesHeader);
2163
+ const correctedRules = rewriteRules(rulesList);
2164
+
2165
+ const correctedFile = path.join(dir, `${filterId}_without_easylist.txt`);
2166
+ writeFilterFile(correctedFile, config.configuration.adbHeader, correctedHeader, correctedRules);
2167
+ }
2168
+ };
2169
+
2170
+ /**
2171
+ * Removes rules array duplicates,
2172
+ * ignores comments and hinted rules
2173
+ *
2174
+ * @param list a list of rules to filter
2175
+ * @returns {*} a list of rules without duplicate
2176
+ */
2177
+ const removeRuleDuplicates = function (list) {
2178
+ logger.info('Removing duplicates..');
2179
+
2180
+ return list.filter((item, pos) => {
2181
+ // Do not remove hinted duplicates
2182
+ if (pos > 0) {
2183
+ const previous = list[pos - 1];
2184
+ if (previous && previous.startsWith(RuleMasks.MASK_HINT)) {
2185
+ return true;
2186
+ }
2187
+ }
2188
+
2189
+ // Do not remove hinted duplicates
2190
+ const duplicatePosition = list.indexOf(item, pos > 0 ? pos - 1 : pos);
2191
+ if (duplicatePosition !== pos && duplicatePosition > 0) {
2192
+ const duplicate = list[duplicatePosition - 1];
2193
+ if (duplicate && duplicate.startsWith(RuleMasks.MASK_HINT)) {
2194
+ return true;
2195
+ }
2196
+ }
2197
+
2198
+ // Do not remove commented duplicates
2199
+ const result = item.startsWith(RuleMasks.MASK_COMMENT) || duplicatePosition === pos;
2200
+
2201
+ if (!result) {
2202
+ logger.info(`${item} removed as duplicate`);
2203
+ }
2204
+
2205
+ return result;
2206
+ });
2207
+ };
2208
+
2209
+ /**
2210
+ * Builds platforms for filter
2211
+ *
2212
+ * @param filterDir - Path to filter directory
2213
+ * @param platformsPath - Path to platforms folder
2214
+ * @param whitelist - Array of filter ids to whitelist
2215
+ * @param blacklist - Array of filter ids to blacklist
2216
+ */
2217
+ const buildFilter$1 = async (filterDir, platformsPath, whitelist, blacklist) => {
2218
+ const originalRules = readFile$2(path.join(filterDir, filterFile)).split('\r\n');
2219
+
2220
+ const metadataFilePath = path.join(filterDir, metadataFile);
2221
+ const revisionFilePath = path.join(filterDir, revisionFile);
2222
+
2223
+ const metadata = JSON.parse(readFile$2(metadataFilePath));
2224
+ const { filterId } = metadata;
2225
+ checkFilterId(filterIdsPool, filterId);
2226
+
2227
+ if (whitelist && whitelist.length > 0 && whitelist.indexOf(filterId) < 0) {
2228
+ logger.info(`Filter ${filterId} skipped with whitelist`);
2229
+ return;
2230
+ }
2231
+
2232
+ if (blacklist && blacklist.length > 0 && blacklist.indexOf(filterId) >= 0) {
2233
+ logger.info(`Filter ${filterId} skipped with blacklist`);
2234
+ return;
2235
+ }
2236
+
2237
+ const optimizationConfig = getFilterOptimizationConfig(filterId);
2238
+
2239
+ // eslint-disable-next-line guard-for-in,no-restricted-syntax
2240
+ for (const platform in platformPathsConfig) {
2241
+ const config = platformPathsConfig[platform];
2242
+
2243
+ if (!shouldBuildFilterForPlatform(metadata, config.platform)) {
2244
+ // eslint-disable-next-line max-len
2245
+ logger.info(`Build of filter ${filterId} skipped for platform '${config.platform}' due to platformsExcluded or platformsIncluded`);
2246
+ continue;
2247
+ }
2248
+
2249
+ let rules = FiltersDownloader.resolveConditions(originalRules, config.defines);
2250
+
2251
+ // handle includes after resolving conditions:
2252
+ // if there is a bad include after resolving conditions, the generator should be terminated
2253
+ // https://github.com/AdguardTeam/FiltersCompiler/issues/84
2254
+ // eslint-disable-next-line no-await-in-loop
2255
+ rules = await FiltersDownloader.resolveIncludes(rules, filterDir, config.defines);
2256
+
2257
+ rules = cleanupRules(rules, config, filterId);
2258
+ rules = removeRuleDuplicates(rules);
2259
+
2260
+ // Apply replacement rules
2261
+ if (config.configuration?.replacements) {
2262
+ rules = rules.map((rule) => {
2263
+ // eslint-disable-next-line no-restricted-syntax
2264
+ for (const repl of config.configuration.replacements) {
2265
+ rule = rule.replace(new RegExp(repl.from, 'g'), repl.to);
2266
+ }
2267
+ return rule;
2268
+ });
2269
+ }
2270
+
2271
+ const optimizedRules = cleanupAndOptimizeRules(rules, config, optimizationConfig, filterId);
2272
+ // eslint-disable-next-line max-len
2273
+ logger.info(`Filter ${filterId}. Rules ${originalRules.length} => ${rules.length} => ${optimizedRules.length}. PlatformPath: '${config.path}'`);
2274
+
2275
+ const header = makeHeader(metadataFilePath, revisionFilePath, config.expires);
2276
+
2277
+ const platformDir = path.join(platformsPath, config.path, PLATFORM_FILTERS_DIR);
2278
+ writeFilterRules(filterId, platformDir, config, header, rules, false);
2279
+
2280
+ // add '(Optimized)' to the '! Title:' for optimized filters
2281
+ // https://github.com/AdguardTeam/FiltersCompiler/issues/78
2282
+ const optimizedHeader = [...header];
2283
+ optimizedHeader[0] += ' (Optimized)';
2284
+
2285
+ writeFilterRules(filterId, platformDir, config, optimizedHeader, optimizedRules, true);
2286
+ }
2287
+ };
2288
+
2289
+ /**
2290
+ * Initializes service
2291
+ *
2292
+ * @param filterFileName output filter file name.
2293
+ * @param metadataFileName output metadata file name.
2294
+ * @param revisionFileName output revision file name.
2295
+ * @param platformsConfig platforms configuration object.
2296
+ * @param adguardFiltersServer server that will serve the filters that're being built.
2297
+ */
2298
+ const init = function (
2299
+ filterFileName,
2300
+ metadataFileName,
2301
+ revisionFileName,
2302
+ platformsConfig,
2303
+ adguardFiltersServer,
2304
+ ) {
2305
+ filterFile = filterFileName;
2306
+ metadataFile = metadataFileName;
2307
+ revisionFile = revisionFileName;
2308
+
2309
+ adguardFiltersServerUrl = adguardFiltersServer;
2310
+
2311
+ if (!platformsConfig) {
2312
+ throw new Error('Platforms config is not defined');
2313
+ }
2314
+
2315
+ platformPathsConfig = platformsConfig;
2316
+ };
2317
+
2318
+ /**
2319
+ * Checks if filter has 'obsolete' tag
2320
+ * @param {object} metadata
2321
+ * @returns {boolean}
2322
+ */
2323
+ const isObsoleteFilter = (metadata) => metadata.tags && metadata.tags.some((tag) => tag === 'obsolete');
2324
+
2325
+ /**
2326
+ * Parses directory recursive
2327
+ *
2328
+ * @param filtersDir
2329
+ * @param filtersMetadata
2330
+ * @param platformsPath
2331
+ * @param whitelist
2332
+ * @param blacklist
2333
+ * @param obsoleteFiltersMetadata
2334
+ */
2335
+ const parseDirectory$1 = async (
2336
+ filtersDir,
2337
+ filtersMetadata,
2338
+ platformsPath,
2339
+ whitelist,
2340
+ blacklist,
2341
+ obsoleteFiltersMetadata,
2342
+ ) => {
2343
+ const items = fs.readdirSync(filtersDir);
2344
+ // eslint-disable-next-line no-restricted-syntax
2345
+ for (const directory of items) {
2346
+ const filterDir = path.join(filtersDir, directory);
2347
+ if (fs.lstatSync(filterDir).isDirectory()) {
2348
+ const metadataFilePath = path.join(filterDir, metadataFile);
2349
+ if (fs.existsSync(metadataFilePath)) {
2350
+ logger.info(`Building filter platforms: ${directory}`);
2351
+ // eslint-disable-next-line no-await-in-loop
2352
+ await buildFilter$1(filterDir, platformsPath, whitelist, blacklist);
2353
+ logger.info(`Building filter platforms: ${directory} done`);
2354
+ const filterMetadata = loadFilterMetadata(filterDir, whitelist, blacklist);
2355
+ filtersMetadata.push(filterMetadata);
2356
+ if (isObsoleteFilter(filterMetadata)) {
2357
+ obsoleteFiltersMetadata.push(filterMetadata);
2358
+ }
2359
+ } else {
2360
+ // eslint-disable-next-line no-await-in-loop
2361
+ await parseDirectory$1(
2362
+ filterDir,
2363
+ filtersMetadata,
2364
+ platformsPath,
2365
+ whitelist,
2366
+ blacklist,
2367
+ obsoleteFiltersMetadata,
2368
+ );
2369
+ }
2370
+ }
2371
+ }
2372
+ };
2373
+
2374
+ /**
2375
+ * Generates platform-specific files and metadata based on the provided filters and configurations.
2376
+ *
2377
+ * @async
2378
+ * @param {string} filtersDir - The directory containing filter files to be processed.
2379
+ * @param {string} platformsPath - The output path where the generated platform files will be stored.
2380
+ * @param {Array<number>} whitelist - A list of whitelist filter IDs.
2381
+ * @param {Array<number>} blacklist - A list of blacklist filter IDs.
2382
+ * @returns {Promise<void>} Resolves when the generation process is complete.
2383
+ *
2384
+ * @throws {Error} If `platformsPath` or `platformPathsConfig` is not specified.
2385
+ */
2386
+ const generate = async (filtersDir, platformsPath, whitelist, blacklist) => {
2387
+ if (!platformsPath) {
2388
+ logger.warn('Platforms build output path is not specified');
2389
+ return;
2390
+ }
2391
+
2392
+ if (!platformPathsConfig) {
2393
+ logger.warn('Platforms configuration is not specified');
2394
+ return;
2395
+ }
2396
+
2397
+ createDir(platformsPath);
2398
+
2399
+ const filtersMetadata = [];
2400
+ const obsoleteFiltersMetadata = [];
2401
+
2402
+ await parseDirectory$1(filtersDir, filtersMetadata, platformsPath, whitelist, blacklist, obsoleteFiltersMetadata);
2403
+
2404
+ writeFiltersMetadata(platformsPath, filtersDir, filtersMetadata, obsoleteFiltersMetadata);
2405
+ writeLocalScriptRules(platformsPath);
2406
+
2407
+ // reset af the end
2408
+ // TODO: find out better way to reset
2409
+ filterIdsPool = [];
2410
+ };
2411
+
2412
+ const { log } = console;
2413
+ const reportDate = new Date();
2414
+
2415
+ let reportData = `\nFiltersCompiler report ${reportDate.toLocaleDateString()} ${reportDate.toLocaleTimeString()}:\n\n`;
2416
+
2417
+ /**
2418
+ * Adds filters data to report
2419
+ * @param {object} metadata
2420
+ * @param {object} filterRules
2421
+ * @param {string[]} invalidRules
2422
+ */
2423
+ const addFilter = (metadata, filterRules, invalidRules) => {
2424
+ if (metadata && filterRules) {
2425
+ const { filterId, name, subscriptionUrl } = metadata;
2426
+ const filterLength = filterRules.lines.length;
2427
+ const excludedLength = filterRules.excluded.length;
2428
+
2429
+ reportData += `Filter ID: ${filterId}\n`
2430
+ + `Filter name: ${name}\n`
2431
+ + `Compiled rules: ${filterLength}\n`
2432
+ + `Excluded rules: ${excludedLength}\n`
2433
+ + `URL: ${subscriptionUrl}\n`;
2434
+ // log list of invalid rules only if they exist
2435
+ if (invalidRules.length > 0) {
2436
+ reportData += 'INVALID RULES:\n'
2437
+ + `${invalidRules.join('\n')}\n`;
2438
+ } else {
2439
+ reportData += 'All rules are valid.\n';
2440
+ }
2441
+ reportData += '---------------------------\n';
2442
+ }
2443
+ };
2444
+
2445
+ /**
2446
+ * Adds disabled filters data to report
2447
+ * @param {object} metadata
2448
+ */
2449
+ const skipFilter = (metadata) => {
2450
+ if (metadata) {
2451
+ const { filterId, name, subscriptionUrl } = metadata;
2452
+
2453
+ reportData += `Filter ID: ${filterId}\n`
2454
+ + `Filter name: ${name}\n`
2455
+ + 'Filter is DISABLED!\n'
2456
+ + `URL: ${subscriptionUrl}\n`
2457
+ + '---------------------------\n';
2458
+ }
2459
+ };
2460
+
2461
+ /**
2462
+ * Creates report file or outputs report
2463
+ * @param {string} reportPath
2464
+ */
2465
+ const create = (reportPath) => {
2466
+ if (reportPath) {
2467
+ fs.writeFileSync(reportPath, reportData, 'utf8');
2468
+ return;
2469
+ }
2470
+ log(reportData);
2471
+ };
2472
+
2473
+ /**
2474
+ * Returns filter ID from directory name if it has a number id,
2475
+ * otherwise returns 0.
2476
+ *
2477
+ * @param {string} str Directory name.
2478
+ *
2479
+ * @returns {number} Filter ID for filter directory, otherwise 0.
2480
+ */
2481
+ const getFilterIdFromDirName = (str) => {
2482
+ const chunks = str.split('_');
2483
+ const rawId = chunks.length > 1 ? Number(chunks[1]) : 0;
2484
+ return !Number.isNaN(rawId) ? rawId : 0;
2485
+ };
2486
+
2487
+ const DOT = '.';
2488
+
2489
+ /**
2490
+ * Modifies string to handle domains without rule markers
2491
+ *
2492
+ * @param {string} rule - Rule in base adblock syntax.
2493
+ * @returns {string} - Domain without base rule markers.
2494
+ */
2495
+ const removeRuleMarkers = (rule) => rule
2496
+ .replace(RuleMasks.MASK_BASE_RULE, '')
2497
+ .replace(RuleMasks.MASK_RULE_SEPARATOR, '');
2498
+
2499
+ /**
2500
+ * Checks if the line is in base rule style syntax with no modifier, i.e.,
2501
+ * starts with `||` and ends with `^`.
2502
+ *
2503
+ * @param {string} rule - Rule to check.
2504
+ * @returns {boolean} - True if the rule is in base rule style syntax.
2505
+ */
2506
+ const shouldOptimize = (rule) => {
2507
+ return rule.startsWith(RuleMasks.MASK_BASE_RULE)
2508
+ && rule.endsWith(RuleMasks.MASK_RULE_SEPARATOR);
2509
+ };
2510
+
2511
+ /**
2512
+ * Returns the top level domain of the given domain.
2513
+ *
2514
+ * @param {string} domain Domain to get the top level domain from.
2515
+ *
2516
+ * @returns {string} Top level domain.
2517
+ */
2518
+ const getTopLevelDomain = (domain) => {
2519
+ const parsedDomain = parse$1(domain).domain;
2520
+ return typeof parsedDomain === 'string' ? parsedDomain : domain;
2521
+ };
2522
+
2523
+ /**
2524
+ * Finds the widest domains in the given list.
2525
+ *
2526
+ * @param {Set<string>} domains Set of domains to find the widest domains from.
2527
+ *
2528
+ * @returns {Set<string>} Set of widest domains.
2529
+ *
2530
+ * @example
2531
+ * - example.com, sub1.example.com, abc.sub2.example.com -> example.com
2532
+ * - example.org, example.com -> example.org, example.com
2533
+ */
2534
+ const findWidestDomains = (domains) => {
2535
+ const sortedDomains = [...domains].sort((a, b) => {
2536
+ return a.split(DOT).length - b.split(DOT).length;
2537
+ });
2538
+
2539
+ const result = new Set();
2540
+
2541
+ sortedDomains.forEach((domain) => {
2542
+ let isSubdomain = false;
2543
+ result.forEach((parent) => {
2544
+ if (domain.endsWith(`${DOT}${parent}`)) {
2545
+ isSubdomain = true;
2546
+ }
2547
+ });
2548
+
2549
+ if (!isSubdomain) {
2550
+ result.add(domain);
2551
+ }
2552
+ });
2553
+
2554
+ return result;
2555
+ };
2556
+
2557
+ /**
2558
+ * Removes redundant rules from lines
2559
+ * @param {string[]} lines - An array of text lines.
2560
+ * @returns {string[]} - An array of of text lines with redundant rules removed.
2561
+ */
2562
+ const optimizeDomainBlockingRules = async (lines) => {
2563
+ const linesToSkipOptimization = new Set();
2564
+ const rawDomainsToOptimize = new Set();
2565
+
2566
+ lines.forEach((line) => {
2567
+ if (!shouldOptimize(line)) {
2568
+ linesToSkipOptimization.add(line);
2569
+ return;
2570
+ }
2571
+
2572
+ const rawDomain = removeRuleMarkers(line);
2573
+ rawDomainsToOptimize.add(rawDomain);
2574
+ });
2575
+
2576
+ /**
2577
+ * Map of tld for all raw domains
2578
+ * @type {Map<string, Set<string>>}
2579
+ *
2580
+ * It is needed to group rawDomains by top level domain
2581
+ * so groups of related rawDomains can be optimized in parallel.
2582
+ */
2583
+ const topDomainsMap = new Map();
2584
+
2585
+ rawDomainsToOptimize.forEach((rawDomain) => {
2586
+ const topLevelDomain = getTopLevelDomain(rawDomain);
2587
+
2588
+ if (!topDomainsMap.has(topLevelDomain)) {
2589
+ topDomainsMap.set(topLevelDomain, new Set([rawDomain]));
2590
+ } else {
2591
+ topDomainsMap.get(topLevelDomain).add(rawDomain);
2592
+ }
2593
+ });
2594
+
2595
+ const widerDomains = new Set();
2596
+
2597
+ /**
2598
+ * Runs optimization for a group of related domains -
2599
+ * finds the widest domains in the group and adds them to the result list of wider domains.
2600
+ *
2601
+ * @param {Set<string>} rawDomains Set of related raw domains to optimize.
2602
+ */
2603
+ const optimizeDomains = (rawDomains) => {
2604
+ const widestDomains = findWidestDomains(rawDomains);
2605
+ widestDomains.forEach((domain) => {
2606
+ widerDomains.add(domain);
2607
+ });
2608
+ };
2609
+
2610
+ topDomainsMap.forEach((rawDomains) => {
2611
+ optimizeDomains(rawDomains);
2612
+ });
2613
+
2614
+ // return lines in the order they were given
2615
+ return lines.filter((line) => {
2616
+ if (linesToSkipOptimization.has(line)) {
2617
+ return true;
2618
+ }
2619
+
2620
+ const domain = removeRuleMarkers(line);
2621
+
2622
+ return widerDomains.has(domain);
2623
+ });
2624
+ };
2625
+
2626
+ /* eslint-disable global-require */
2627
+
2628
+
2629
+ const __dirname$2 = path.dirname(new URL(import.meta.url).pathname);
2630
+
2631
+ const TEMPLATE_FILE = 'template.txt';
2632
+ const FILTER_FILE = 'filter.txt';
2633
+ const REVISION_FILE = 'revision.json';
2634
+ const EXCLUDE_FILE = 'exclude.txt';
2635
+ const EXCLUDED_LINES_FILE = 'diff.txt';
2636
+ const METADATA_FILE = 'metadata.json';
2637
+ const ADGUARD_FILTERS_SERVER_URL = 'https://filters.adtidy.org/';
2638
+ const TRUST_LEVEL_DIR = './utils/trust-levels';
2639
+ const DEFAULT_TRUST_LEVEL = 'low';
2640
+
2641
+ const SPACE = ' ';
2642
+ const SLASH = '/';
2643
+ const COMMA = ',';
2644
+ const EQUAL_SIGN = '=';
2645
+ const MODIFIERS_SEPARATOR = '$';
2646
+ const INCLUDE_DIRECTIVE = '@include ';
2647
+ const STRIP_COMMENTS_OPTION = 'stripComments';
2648
+ const OPTIMIZE_DOMAIN_BLOCKING_RULES = 'optimizeDomainBlockingRules';
2649
+ const NOT_OPTIMIZED_OPTION = 'notOptimized';
2650
+ const EXCLUDE_OPTION = 'exclude';
2651
+ const ADD_MODIFIERS_OPTION = 'addModifiers';
2652
+ const IGNORE_TRUST_LEVEL_OPTION = 'ignoreTrustLevel';
2653
+
2654
+ const NOT_OPTIMIZED_HINT = '!+ NOT_OPTIMIZED';
2655
+
2656
+ /**
2657
+ * Filter header tag for diff path which is needed for patch updates.
2658
+ */
2659
+ const DIFF_PATH_TAG = 'Diff-Path:';
2660
+
2661
+ // TODO: Move include directive option functions to utils/builder-utils.js
2662
+
2663
+ /**
2664
+ * Sync reads file content
2665
+ *
2666
+ * @param path
2667
+ * @returns {*}
2668
+ */
2669
+ const readFile$1 = function (path) {
2670
+ if (!fs.existsSync(path)) {
2671
+ return null;
2672
+ }
2673
+
2674
+ return fs.readFileSync(path, { encoding: 'utf-8' });
2675
+ };
2676
+
2677
+ /**
2678
+ * Sync writes content to file
2679
+ *
2680
+ * @param path
2681
+ * @param data
2682
+ */
2683
+ const writeFile = function (path, data) {
2684
+ fs.writeFileSync(path, data, 'utf8');
2685
+ };
2686
+
2687
+ /**
2688
+ * Splits lines
2689
+ *
2690
+ * @param string
2691
+ */
2692
+ const splitLines = function (string) {
2693
+ return string.split(/\r?\n/);
2694
+ };
2695
+
2696
+ /**
2697
+ * Removes comments from lines
2698
+ *
2699
+ * @param lines
2700
+ */
2701
+ const stripComments = function (lines) {
2702
+ logger.info('Stripping comments..');
2703
+
2704
+ return lines.filter((line, pos) => {
2705
+ if (pos > 0 && lines[pos - 1].startsWith(RuleMasks.MASK_HINT)) {
2706
+ return true;
2707
+ }
2708
+
2709
+ if (
2710
+ line.startsWith(RuleMasks.MASK_HINT)
2711
+ || line.startsWith(RuleMasks.MASK_DIRECTIVES)
2712
+ ) {
2713
+ return true;
2714
+ }
2715
+
2716
+ return !line.startsWith(RuleMasks.MASK_COMMENT);
2717
+ });
2718
+ };
2719
+
2720
+ /**
2721
+ * Adds not optimized hints
2722
+ *
2723
+ * @param lines
2724
+ */
2725
+ const addNotOptimizedHints = function (lines) {
2726
+ logger.info('Adding hints..');
2727
+
2728
+ const result = [];
2729
+
2730
+ lines.forEach((v) => {
2731
+ if (!v) {
2732
+ return;
2733
+ }
2734
+
2735
+ if (!v.startsWith(RuleMasks.MASK_COMMENT) && !v.startsWith(`${RuleMasks.MASK_HINT}${SPACE}`)) {
2736
+ result.push(NOT_OPTIMIZED_HINT);
2737
+ }
2738
+
2739
+ result.push(v);
2740
+ });
2741
+
2742
+ return result;
2743
+ };
2744
+ /**
2745
+ * Adds or updates modifiers in each line of the given array of lines.
2746
+ *
2747
+ * @param {string[]} lines - An array of text lines.
2748
+ * @param {string} modifiersStr - Modifiers as a string to add.
2749
+ * @returns {string[]} - An array of modified lines.
2750
+ */
2751
+ const addModifiers = (lines, modifiersStr) => {
2752
+ return lines.map((line) => {
2753
+ // If the line is empty or contains only whitespace, it returns a comment mask
2754
+ if (!line || line.trim() === '') {
2755
+ return RuleMasks.MASK_COMMENT;
2756
+ }
2757
+ // If the line starts with a host file comment mask, it replaces it with a comment mask
2758
+ if (line.startsWith(RuleMasks.MASK_HOST_FILE_COMMENT)) {
2759
+ return line.replace(RuleMasks.MASK_HOST_FILE_COMMENT, RuleMasks.MASK_COMMENT);
2760
+ }
2761
+ // If the line starts with a comment mask, it returns the line as is.
2762
+ if (line.startsWith(RuleMasks.MASK_COMMENT)) {
2763
+ return line;
2764
+ }
2765
+ // If the line does not contain a modifiers separator, it appends the given modifiers string
2766
+ if (!line.includes(MODIFIERS_SEPARATOR)) {
2767
+ return `${line}${MODIFIERS_SEPARATOR}${modifiersStr}`;
2768
+ }
2769
+ // If the line already contains modifiers, combine the existing modifiers with the new ones,
2770
+ // ensuring no duplicates, and return the line with the updated modifiers.
2771
+ const [rule, existingModifiersStr] = line.split(MODIFIERS_SEPARATOR);
2772
+ const existingModifiers = existingModifiersStr.split(COMMA);
2773
+ const newModifiers = modifiersStr.split(COMMA);
2774
+ const combinedModifiers = [...new Set([...existingModifiers, ...newModifiers])].join(COMMA);
2775
+
2776
+ return `${rule}${MODIFIERS_SEPARATOR}${combinedModifiers}`;
2777
+ });
2778
+ };
2779
+
2780
+ /**
2781
+ * Checks case when '#%#' rules are excluded to DON'T exclude '#%#//scriptlet' rules
2782
+ *
2783
+ * @param {string} line
2784
+ * @param {string} exclusion
2785
+ * @return {boolean}
2786
+ */
2787
+ const scriptletException = (line, exclusion) => (exclusion === '#%#' && line.includes('#%#//scriptlet'))
2788
+ || (exclusion === '#@%#' && line.includes('#@%#//scriptlet'));
2789
+
2790
+ /**
2791
+ * Checks if line is excluded with specified set of exclusions
2792
+ *
2793
+ * @param line
2794
+ * @param exclusions
2795
+ * @param excluded
2796
+ * @param reason
2797
+ */
2798
+ const isExcluded = function (line, exclusions, excluded, reason) {
2799
+ // eslint-disable-next-line no-restricted-syntax
2800
+ for (let exclusion of exclusions) {
2801
+ exclusion = exclusion.trim();
2802
+
2803
+ if (exclusion && !exclusion.startsWith(RuleMasks.MASK_COMMENT)) {
2804
+ const message = `${line} is excluded by "${exclusion}" in ${reason}`;
2805
+
2806
+ const isExcludedByRegexp = exclusion.endsWith(SLASH) && exclusion.startsWith(SLASH)
2807
+ && line.match(new RegExp(exclusion.substring(1, exclusion.length - 1)));
2808
+
2809
+ if ((isExcludedByRegexp || line.includes(exclusion)) && !scriptletException(line, exclusion)) {
2810
+ logger.info(message);
2811
+ excluded.push(`${RuleMasks.MASK_COMMENT}${SPACE}${message}`);
2812
+ excluded.push(line);
2813
+ return exclusion;
2814
+ }
2815
+ }
2816
+ }
2817
+
2818
+ return null;
2819
+ };
2820
+
2821
+ /**
2822
+ * Applies exclusion from exclusions file
2823
+ *
2824
+ * @param lines
2825
+ * @param exclusionsFile
2826
+ * @param excluded
2827
+ * @returns {*}
2828
+ */
2829
+ const exclude = function (lines, exclusionsFile, excluded) {
2830
+ logger.info('Applying exclusions..');
2831
+
2832
+ let exclusions = readFile$1(exclusionsFile);
2833
+ if (!exclusions) {
2834
+ return lines;
2835
+ }
2836
+
2837
+ exclusions = splitLines(exclusions);
2838
+
2839
+ const exclusionsFileName = path.parse(exclusionsFile).base;
2840
+ const result = [];
2841
+
2842
+ lines.forEach((line, pos) => {
2843
+ const exclusion = isExcluded(line, exclusions, excluded, exclusionsFileName);
2844
+ if (exclusion) {
2845
+ if (pos > 0 && lines[pos - 1].startsWith(RuleMasks.MASK_HINT)) {
2846
+ result.push(`${RuleMasks.MASK_COMMENT} [excluded by ${exclusion}] ${line}`);
2847
+ }
2848
+ } else {
2849
+ result.push(line);
2850
+ }
2851
+ });
2852
+
2853
+ return result;
2854
+ };
2855
+
2856
+ /**
2857
+ * Strips leading and trailing quotes from string
2858
+ *
2859
+ * @param s
2860
+ * @returns {*}
2861
+ */
2862
+ const stripEndQuotes = function (s) {
2863
+ let t = s.length;
2864
+ if (s.charAt(0) === '"') {
2865
+ s = s.substring(1, t -= 1);
2866
+ }
2867
+ // eslint-disable-next-line no-plusplus
2868
+ if (s.charAt(--t) === '"') {
2869
+ s = s.substring(0, t);
2870
+ }
2871
+ return s;
2872
+ };
2873
+
2874
+ /**
2875
+ * Extracts the value of an attribute.
2876
+ *
2877
+ * @param {string} attribute - The input attribute string.
2878
+ * @returns {string} The extracted attribute value without quotes if it is not empty.
2879
+ * @throws {Error} If the attribute value is empty.
2880
+ */
2881
+ const getOptionValue = (attribute) => {
2882
+ const quotedValue = attribute.substring(attribute.indexOf(EQUAL_SIGN) + 1);
2883
+ const value = stripEndQuotes(quotedValue).trim();
2884
+ if (value.length === 0) {
2885
+ throw new Error(`Include directive value cannot be empty: '${attribute}'`);
2886
+ }
2887
+ return value;
2888
+ };
2889
+
2890
+ /**
2891
+ * @typedef {object} IncludeOption
2892
+ * @property {string} name Option name
2893
+ * @property {boolean|string} value Option value
2894
+ */
2895
+
2896
+ /**
2897
+ * @typedef {object} ParsedIncludeData
2898
+ * @property {string} url Parsed url of file to include
2899
+ * @property {IncludeOption[]} options Array of parsed include directive options
2900
+ */
2901
+
2902
+ /**
2903
+ * Parses an include directive line to extract the URL and options.
2904
+ *
2905
+ * @param {string} lineWithDirective - The input line with directive to parse.
2906
+ * @returns {ParsedIncludeData} Parsed result containing the URL and options.
2907
+ */
2908
+ const parseIncludeDirective = function (lineWithDirective) {
2909
+ const parts = lineWithDirective.trim().split(SPACE);
2910
+ let url = parts[1].trim();
2911
+ url = stripEndQuotes(url);
2912
+ // Initialize an options array to store the parsed options.
2913
+ const options = [];
2914
+ // Stack options into the array in the sequence in which they found in the string
2915
+ for (let i = 1; i < parts.length; i += 1) {
2916
+ const attribute = parts[i].trim();
2917
+ if (attribute.startsWith(`${SLASH}${STRIP_COMMENTS_OPTION}`)) {
2918
+ options.push({ name: STRIP_COMMENTS_OPTION, value: true });
2919
+ } else if (attribute.startsWith(`${SLASH}${OPTIMIZE_DOMAIN_BLOCKING_RULES}`)) {
2920
+ options.push({ name: OPTIMIZE_DOMAIN_BLOCKING_RULES, value: true });
2921
+ } else if (attribute.startsWith(`${SLASH}${NOT_OPTIMIZED_OPTION}`)) {
2922
+ options.push({ name: NOT_OPTIMIZED_OPTION, value: true });
2923
+ } else if (attribute.startsWith(`${SLASH}${EXCLUDE_OPTION}${EQUAL_SIGN}`)) {
2924
+ options.push({ name: EXCLUDE_OPTION, value: getOptionValue(attribute) });
2925
+ } else if (attribute.startsWith(`${SLASH}${ADD_MODIFIERS_OPTION}${EQUAL_SIGN}`)) {
2926
+ options.push({ name: ADD_MODIFIERS_OPTION, value: getOptionValue(attribute) });
2927
+ } else if (attribute.startsWith(`${SLASH}${IGNORE_TRUST_LEVEL_OPTION}`)) {
2928
+ options.push({ name: IGNORE_TRUST_LEVEL_OPTION, value: true });
2929
+ }
2930
+ }
2931
+ return { url, options };
2932
+ };
2933
+
2934
+ /**
2935
+ * Checks if lines contains invalid redirect directives
2936
+ *
2937
+ * @param lines
2938
+ * @param url
2939
+ */
2940
+ const checkRedirects = function (lines, url) {
2941
+ // eslint-disable-next-line no-restricted-syntax
2942
+ for (const line of lines) {
2943
+ if (/^!\s?[Rr]edirect:/.test(line)) {
2944
+ throw new Error(`Error: include ${url} contains redirect directive: ${line}`);
2945
+ }
2946
+ }
2947
+ };
2948
+
2949
+ /**
2950
+ * @typedef {object} ParsedIncludeResult
2951
+ * @property {string[]} includedLines Included rules.
2952
+ * @property {boolean} shouldIgnoreTrustLevel Indicates whether the metadata's trust level should be ignored.
2953
+ */
2954
+
2955
+ /**
2956
+ * Creates content from compiler's `@include` directive, not `!#include` preprocessor directive.
2957
+ *
2958
+ * @param {string} filterDir Filter directory.
2959
+ * @param {string} directiveLine The include line containing the URL or file path and optional options.
2960
+ * @param {Array<string>} excluded An array of strings representing excluded content.
2961
+ *
2962
+ * @returns {Promise<ParsedIncludeResult>} Parsed include data.
2963
+ * @throws {Error} Throws an error if there is an issue handling the include operation.
2964
+ */
2965
+ const include = async (filterDir, directiveLine, excluded) => {
2966
+ let includedLines = [];
2967
+ let shouldIgnoreTrustLevel = false;
2968
+
2969
+ const { url, options } = parseIncludeDirective(directiveLine);
2970
+
2971
+ if (!url) {
2972
+ logger.warn('Invalid include url');
2973
+ return includedLines;
2974
+ }
2975
+
2976
+ logger.info(`Applying inclusion from: ${url}`);
2977
+
2978
+ const externalInclude = url.includes(':');
2979
+
2980
+ const included = externalInclude
2981
+ ? downloadFile(url)
2982
+ : readFile$1(path.join(filterDir, url));
2983
+
2984
+ if (included) {
2985
+ includedLines = splitLines(included);
2986
+
2987
+ checkRedirects(includedLines, url);
2988
+
2989
+ // resolved `@include` directive url
2990
+ const originUrl = externalInclude
2991
+ ? FiltersDownloader.getFilterUrlOrigin(url)
2992
+ : filterDir;
2993
+
2994
+ includedLines = await FiltersDownloader.resolveIncludes(includedLines, originUrl);
2995
+
2996
+ includedLines = removeAdblockVersion(includedLines);
2997
+
2998
+ // eslint-disable-next-line no-restricted-syntax
2999
+ for (const { name, value } of options) {
3000
+ let optionsExcludePath;
3001
+ switch (name) {
3002
+ case EXCLUDE_OPTION:
3003
+ optionsExcludePath = path.join(filterDir, value);
3004
+ includedLines = exclude(includedLines, optionsExcludePath, excluded);
3005
+ break;
3006
+ case STRIP_COMMENTS_OPTION:
3007
+ includedLines = stripComments(includedLines);
3008
+ break;
3009
+ case OPTIMIZE_DOMAIN_BLOCKING_RULES:
3010
+ // eslint-disable-next-line no-await-in-loop
3011
+ includedLines = await optimizeDomainBlockingRules(includedLines);
3012
+ break;
3013
+ case NOT_OPTIMIZED_OPTION:
3014
+ includedLines = addNotOptimizedHints(includedLines);
3015
+ break;
3016
+ case ADD_MODIFIERS_OPTION:
3017
+ includedLines = addModifiers(includedLines, value);
3018
+ break;
3019
+ case IGNORE_TRUST_LEVEL_OPTION:
3020
+ if (externalInclude) {
3021
+ // eslint-disable-next-line max-len
3022
+ throw new Error(`Trust level ignoring option is not supported for external includes: ${directiveLine}`);
3023
+ }
3024
+ shouldIgnoreTrustLevel = true;
3025
+ break;
3026
+ }
3027
+ }
3028
+
3029
+ includedLines = fixVersionComments(includedLines);
3030
+ } else {
3031
+ throw new Error(`Error handling include from: ${options.url}`);
3032
+ }
3033
+
3034
+ logger.info(`Inclusion lines: ${includedLines.length}`);
3035
+ logger.info(`Should be filtered due to trust level: ${shouldIgnoreTrustLevel}`);
3036
+
3037
+ return {
3038
+ includedLines,
3039
+ shouldIgnoreTrustLevel,
3040
+ };
3041
+ };
3042
+
3043
+ /**
3044
+ * Resolves preprocessor `!#include` directives by FiltersDownloader.
3045
+ *
3046
+ * @param {string} filterDir Filter directory.
3047
+ * @param {string} filterName Filter name.
3048
+ * @param {string[]} lines Array of lines.
3049
+ *
3050
+ * @returns {Promise<string[]>} Promise which resolves
3051
+ * to array of rules as a result of resolving preprocessor `!#include` directives.
3052
+ *
3053
+ * @throws {Error} Throws an error if unable to resolve `!#include` directives.
3054
+ */
3055
+ const getResolvedPreprocessorIncludes = async function (filterDir, filterName, lines) {
3056
+ let rules = lines;
3057
+ // bad includes are ignored here because they'll be handled in generator after resolving conditions
3058
+ // https://github.com/AdguardTeam/FiltersCompiler/issues/84
3059
+ try {
3060
+ rules = await FiltersDownloader.resolveIncludes(lines, filterDir);
3061
+ } catch (e) {
3062
+ logger.warn(`Error resolving includes in ${filterName}: ${e.message}`);
3063
+ }
3064
+ return rules;
3065
+ };
3066
+
3067
+ /**
3068
+ * Resolved preprocessor `!#include` directives and converts rules to Adguard syntax.
3069
+ *
3070
+ * @param {string} filterDir Filter directory.
3071
+ * @param {string} filterName Filter name.
3072
+ * @param {string[]} lines Array of raw rules and possibly preprocessor `!#include` directives.
3073
+ * @param {string} excluded Array of rules to exclude.
3074
+ *
3075
+ * @returns {Promise<string[]>} Promise which resolves to array of AdGuard syntax rules.
3076
+ */
3077
+ const prepareAdgRules = async function (filterDir, filterName, lines, excluded) {
3078
+ const resolvedIncludes = await getResolvedPreprocessorIncludes(filterDir, filterName, lines);
3079
+ return convertRulesToAdgSyntax(resolvedIncludes, excluded);
3080
+ };
3081
+
3082
+ /**
3083
+ * Checks whether the line contain `Diff-Path` header tag.
3084
+ *
3085
+ * `Diff-Path` header tag should be removed from the rules
3086
+ * because some third-party filters may contain it
3087
+ * but @adguard/diff-builder does not support third-party patches.
3088
+ * Anyway the `Diff-Path` header tag should be added by the diff-builder during patch building
3089
+ * and should not be present in the filter file.
3090
+ *
3091
+ * @param {string} line Line to check.
3092
+ *
3093
+ * @returns {boolean} `true` if the line contains `Diff-Path` header tag, `false` otherwise.
3094
+ */
3095
+ const isDiffPathHeaderTag = (line) => {
3096
+ // non-comment lines should be kept
3097
+ if (
3098
+ !line.startsWith(RuleMasks.MASK_COMMENT)
3099
+ // ubo supports `#` for comments
3100
+ && !line.startsWith(RuleMasks.MASK_HOST_FILE_COMMENT)
3101
+ ) {
3102
+ return false;
3103
+ }
3104
+
3105
+ return line.includes(DIFF_PATH_TAG);
3106
+ };
3107
+
3108
+ /**
3109
+ * @typedef {object} CompileResult
3110
+ * @property {string[]} lines Compiled rules.
3111
+ * @property {string[]} excluded Excluded rules.
3112
+ * @property {string[]} invalid Invalid rules.
3113
+ */
3114
+
3115
+ /**
3116
+ * Compiles filter lines.
3117
+ *
3118
+ * @param filterDir Filter directory.
3119
+ * @param filterName Filter name.
3120
+ * @param templateContent Content of template.txt file.
3121
+ * @param trustLevelSettings Trust level settings for the filter.
3122
+ *
3123
+ * @returns {Promise<CompileResult>} Promise which resolves to compiled data for the filter.
3124
+ */
3125
+ const compile$1 = async function (filterDir, filterName, templateContent, trustLevelSettings) {
3126
+ let result = [];
3127
+ const excluded = [];
3128
+
3129
+ // collect invalid rules for report
3130
+ // https://github.com/AdguardTeam/FiltersCompiler/issues/87
3131
+ const invalid = [];
3132
+
3133
+ const lines = splitLines(templateContent);
3134
+ // eslint-disable-next-line no-restricted-syntax
3135
+ for (const line of lines) {
3136
+ if (line.startsWith(INCLUDE_DIRECTIVE)) {
3137
+ // eslint-disable-next-line no-await-in-loop
3138
+ const { includedLines, shouldIgnoreTrustLevel } = await include(filterDir, line, excluded);
3139
+
3140
+ let includedRules = [];
3141
+
3142
+ includedLines.forEach((line) => includedRules.push(line.trim()));
3143
+
3144
+ // eslint-disable-next-line no-await-in-loop
3145
+ includedRules = await prepareAdgRules(filterDir, filterName, includedRules, excluded);
3146
+
3147
+ if (shouldIgnoreTrustLevel) {
3148
+ logger.info(`Ignoring trust level for ${filterName} due to @include directive: ${line}`);
3149
+ } else {
3150
+ logger.info(`Applying trust-level exclusions to ${filterName} @include directive: ${line}`);
3151
+ includedRules = exclude(includedRules, trustLevelSettings, excluded);
3152
+ }
3153
+
3154
+ // 'for' loop is used in purpose instead of spread operator
3155
+ // to avoid 'Maximum call stack size exceeded' error on large number of rules
3156
+ for (let i = 0; i < includedRules.length; i += 1) {
3157
+ result.push(includedRules[i]);
3158
+ }
3159
+ } else {
3160
+ let inlineRules = [line.trim()];
3161
+
3162
+ // eslint-disable-next-line no-await-in-loop
3163
+ inlineRules = await prepareAdgRules(filterDir, filterName, inlineRules, excluded);
3164
+
3165
+ logger.info('Applying trust-level exclusions to inline template.txt rules...');
3166
+ inlineRules = exclude(inlineRules, trustLevelSettings, excluded);
3167
+
3168
+ // 'for' loop is used in purpose instead of spread operator
3169
+ // to avoid 'Maximum call stack size exceeded' error on large number of rules
3170
+ for (let i = 0; i < inlineRules.length; i += 1) {
3171
+ result.push(inlineRules[i]);
3172
+ }
3173
+ }
3174
+ }
3175
+
3176
+ result = result.filter((line) => !isDiffPathHeaderTag(line));
3177
+
3178
+ const excludeFilePath = path.join(filterDir, EXCLUDE_FILE);
3179
+ result = exclude(result, excludeFilePath, excluded);
3180
+
3181
+ result = validateAndFilterRules(result, excluded, invalid, filterName);
3182
+
3183
+ return {
3184
+ lines: result,
3185
+ excluded,
3186
+ invalid,
3187
+ };
3188
+ };
3189
+
3190
+ /**
3191
+ * Creates revision object,
3192
+ * doesn't increment version if hash is not changed
3193
+ *
3194
+ * @param {string} path - The path to the revision file.
3195
+ * @param {string} hash - The hash of the filter file.
3196
+ * @returns {{version: string, timeUpdated: number}}
3197
+ */
3198
+ const makeRevision = function (path, hash) {
3199
+ const result = {
3200
+ version: '1.0.0.0',
3201
+ timeUpdated: new Date().getTime(),
3202
+ hash,
3203
+ };
3204
+
3205
+ const current = readFile$1(path);
3206
+ if (current) {
3207
+ const currentRevision = JSON.parse(current);
3208
+ if (currentRevision.version) {
3209
+ result.version = currentRevision.version;
3210
+
3211
+ if (currentRevision.timeUpdated) {
3212
+ result.timeUpdated = currentRevision.timeUpdated;
3213
+ }
3214
+
3215
+ if (!currentRevision.hash || currentRevision.hash !== result.hash) {
3216
+ result.version = version.increment(currentRevision.version);
3217
+ result.timeUpdated = new Date().getTime();
3218
+ }
3219
+ }
3220
+ }
3221
+
3222
+ return result;
3223
+ };
3224
+
3225
+ /**
3226
+ * Builds filter txt file from directory contents
3227
+ *
3228
+ * @param {string} filterDir - The path to the directory containing filters.
3229
+ * @param {Array<number>} whitelist - An array whitelist filters IDs.
3230
+ * @param {Array<number>} blacklist - An array blacklist filters IDs.
3231
+ * @returns {Promise<void>} A promise that resolves when all filters and its subdirectories have been processed.
3232
+ */
3233
+ const buildFilter = async function (filterDir, whitelist, blacklist) {
3234
+ const templateContent = readFile$1(path.join(filterDir, TEMPLATE_FILE));
3235
+ if (!templateContent) {
3236
+ throw new Error('Invalid template');
3237
+ }
3238
+
3239
+ const metadata = JSON.parse(readFile$1(path.join(filterDir, METADATA_FILE)));
3240
+
3241
+ const { filterId } = metadata;
3242
+
3243
+ if (whitelist && whitelist.length > 0 && whitelist.indexOf(filterId) < 0) {
3244
+ logger.info(`Filter ${filterId} skipped due to '--include' option`);
3245
+ return;
3246
+ }
3247
+
3248
+ if (blacklist && blacklist.length > 0 && blacklist.indexOf(filterId) >= 0) {
3249
+ logger.info(`Filter ${filterId} skipped due to '--skip' option`);
3250
+ return;
3251
+ }
3252
+
3253
+ // check whether the filter is disabled after other checks to avoid unnecessary logging
3254
+ if (metadata.disabled) {
3255
+ logger.warn(`Filter ${filterId} skipped as disabled`);
3256
+ skipFilter(metadata);
3257
+ return;
3258
+ }
3259
+
3260
+ const trustLevel = metadata.trustLevel ? metadata.trustLevel : DEFAULT_TRUST_LEVEL;
3261
+ // eslint-disable-next-line no-undef
3262
+ const trustLevelSettings = path.resolve(__dirname$2, TRUST_LEVEL_DIR, `exclusions-${trustLevel}.txt`);
3263
+
3264
+ const { name: filterName } = metadata;
3265
+ logger.info(`Compiling ${filterName}`);
3266
+ const result = await compile$1(filterDir, filterName, templateContent, trustLevelSettings);
3267
+
3268
+ if (!checkAffinityDirectives(result.lines)) {
3269
+ throw new Error(`Error validating !#safari_cb_affinity directive in filter ${filterId}`);
3270
+ }
3271
+
3272
+ const compiled = result.lines;
3273
+ const { excluded, invalid } = result;
3274
+
3275
+ addFilter(metadata, result, invalid);
3276
+ logger.info(`Compiled length:${compiled.length}`);
3277
+ logger.info(`Excluded length:${excluded.length}`);
3278
+
3279
+ const compiledData = compiled.join('\r\n');
3280
+
3281
+ logger.info(`Writing filter file, lines:${compiled.length}`);
3282
+ writeFile(path.join(filterDir, FILTER_FILE), compiledData);
3283
+ logger.info(`Writing excluded file, lines:${excluded.length}`);
3284
+ writeFile(path.join(filterDir, EXCLUDED_LINES_FILE), excluded.join('\r\n'));
3285
+ logger.info('Writing revision file..');
3286
+
3287
+ // eslint-disable-next-line no-buffer-constructor
3288
+ const hash = Buffer.from(md5(compiledData, { asString: true })).toString('base64').trim();
3289
+ const revisionFile = path.join(filterDir, REVISION_FILE);
3290
+ const revision = makeRevision(revisionFile, hash);
3291
+ writeFile(revisionFile, JSON.stringify(revision, null, '\t'));
3292
+ };
3293
+
3294
+ /**
3295
+ * Asynchronously parses a directory and processes filters based on the provided whitelist and blacklist.
3296
+ *
3297
+ * @param {string} filtersDir - The path to the directory containing filters.
3298
+ * @param {Array<number>} whitelist - An array whitelist filters IDs.
3299
+ * @param {Array<number>} blacklist - An array blacklist filters IDs.
3300
+ * @returns {Promise<void>} A promise that resolves when all filters and its subdirectories have been processed.
3301
+ */
3302
+ const parseDirectory = async function (filtersDir, whitelist, blacklist) {
3303
+ const items = fs.readdirSync(filtersDir)
3304
+ .sort((a, b) => getFilterIdFromDirName(a) - getFilterIdFromDirName(b));
3305
+
3306
+ // eslint-disable-next-line no-restricted-syntax
3307
+ for (const directory of items) {
3308
+ const filterDir = path.join(filtersDir, directory);
3309
+ if (fs.lstatSync(filterDir).isDirectory()) {
3310
+ const template = path.join(filterDir, TEMPLATE_FILE);
3311
+ if (fs.existsSync(template)) {
3312
+ logger.info(`Building filter ${directory}...`);
3313
+ // eslint-disable-next-line no-await-in-loop
3314
+ await buildFilter(filterDir, whitelist, blacklist);
3315
+ logger.info(`Filter ${directory} ok`);
3316
+ } else {
3317
+ // eslint-disable-next-line no-await-in-loop
3318
+ await parseDirectory(filterDir, whitelist, blacklist);
3319
+ }
3320
+ }
3321
+ }
3322
+ };
3323
+
3324
+ /**
3325
+ * Asynchronously builds and processes filter files, generates platform data, and creates a report.
3326
+ *
3327
+ * @async
3328
+ * @function build
3329
+ * @param {string} filtersDir - The directory containing filter files to be processed.
3330
+ * @param {string} logFile - The path to the log file where logs will be written.
3331
+ * @param {string} reportFile - The path to the report file to be created.
3332
+ * @param {string} platformsPath - The path where platform data will be generated.
3333
+ * @param {Object} platformsConfig - The configuration object for platforms.
3334
+ * @param {Array<number>} whitelist - A list of filter file names to include in processing.
3335
+ * @param {Array<number>} blacklist - A list of filter file names to exclude from processing.
3336
+ * @returns {Promise<void>} A promise that resolves when the build process is complete.
3337
+ */
3338
+ const build = async (
3339
+ filtersDir,
3340
+ logFile,
3341
+ reportFile,
3342
+ platformsPath,
3343
+ platformsConfig,
3344
+ whitelist,
3345
+ blacklist,
3346
+ ) => {
3347
+ logger.initialize(logFile);
3348
+ init(FILTER_FILE, METADATA_FILE, REVISION_FILE, platformsConfig, ADGUARD_FILTERS_SERVER_URL);
3349
+
3350
+ await parseDirectory(filtersDir, whitelist, blacklist);
3351
+
3352
+ logger.info('Generating platforms');
3353
+ await generate(filtersDir, platformsPath, whitelist, blacklist);
3354
+ logger.info('Generating platforms done');
3355
+ create(reportFile);
3356
+ };
3357
+
3358
+ /* eslint-disable global-require */
3359
+
3360
+
3361
+ const SCHEMA_EXTENSION = '.schema.json';
3362
+ const OLD_MAC_V1_PLATFORM = 'mac';
3363
+ const OLD_MAC_V2_PLATFORM = 'mac_v2';
3364
+
3365
+ /**
3366
+ * Loads all available schemas from dir
3367
+ *
3368
+ * @param {string} dir - The directory path containing the schema files.
3369
+ * @returns {Object} An object with keys - schema file names and values - parsed JSON schema objects.
3370
+ */
3371
+ const loadSchemas = (dir) => {
3372
+ const schemas = {};
3373
+
3374
+ const items = fs.readdirSync(dir);
3375
+ // eslint-disable-next-line no-restricted-syntax
3376
+ for (const f of items) {
3377
+ if (f.endsWith(SCHEMA_EXTENSION)) {
3378
+ const validationFileName = f.substr(0, f.indexOf(SCHEMA_EXTENSION));
3379
+
3380
+ logger.info(`Loading schema for ${validationFileName}`);
3381
+ schemas[validationFileName] = JSON.parse(fs.readFileSync(path.join(dir, f)));
3382
+ }
3383
+ }
3384
+
3385
+ return schemas;
3386
+ };
3387
+
3388
+ /**
3389
+ * Recursively validates dir content with provided schemas
3390
+ *
3391
+ * @param dir
3392
+ * @param validator
3393
+ * @param schemas
3394
+ * @param oldSchemas
3395
+ * @param filtersRequiredAmount
3396
+ * @returns {boolean}
3397
+ */
3398
+ const validateDir = (dir, validator, schemas, oldSchemas, filtersRequiredAmount) => {
3399
+ let items;
3400
+ try {
3401
+ items = fs.readdirSync(dir);
3402
+ } catch (e) {
3403
+ logger.info(e.message);
3404
+ return false;
3405
+ }
3406
+ // eslint-disable-next-line no-restricted-syntax
3407
+ for (const f of items) {
3408
+ const item = path.join(dir, f);
3409
+ if (fs.lstatSync(item).isDirectory()) {
3410
+ if (!validateDir(item, validator, schemas, oldSchemas)) {
3411
+ return false;
3412
+ }
3413
+ } else {
3414
+ const fileName = path.basename(item, '.json');
3415
+ let schema = schemas[fileName];
3416
+
3417
+ // Validate `mac` (mac v1) dir with old schemas
3418
+ if (path.basename(path.dirname(item)) === OLD_MAC_V1_PLATFORM) {
3419
+ logger.info('Look up old schemas for mac directory');
3420
+ schema = oldSchemas[OLD_MAC_V1_PLATFORM][fileName];
3421
+ }
3422
+
3423
+ // Validate `mac_v2` dir with old schemas
3424
+ if (path.basename(path.dirname(item)) === OLD_MAC_V2_PLATFORM) {
3425
+ logger.info('Look up old schemas for mac_v2 directory');
3426
+ schema = oldSchemas[OLD_MAC_V2_PLATFORM][fileName];
3427
+ }
3428
+
3429
+ if (schema) {
3430
+ logger.info(`Validating ${item}`);
3431
+
3432
+ const json = JSON.parse(fs.readFileSync(item));
3433
+
3434
+ // Validate filters amount
3435
+ if (fileName === 'filters') {
3436
+ if (json.filters.length < filtersRequiredAmount) {
3437
+ logger.error(`Invalid filters amount in ${item}`);
3438
+ return false;
3439
+ }
3440
+ }
3441
+
3442
+ const validate = validator.compile(schema);
3443
+ const valid = validate(json);
3444
+
3445
+ // json can be updated with default values
3446
+ fs.writeFileSync(item, JSON.stringify(json, null, '\t'));
3447
+
3448
+ // duplicate to .js file as well
3449
+ const jsFileName = `${fileName}.js`;
3450
+ fs.writeFileSync(
3451
+ path.join(path.dirname(item), jsFileName),
3452
+ JSON.stringify(json, null, '\t'),
3453
+ );
3454
+
3455
+ if (!valid) {
3456
+ logger.error(`Invalid json in ${item}, errors:`);
3457
+ logger.error(validate.errors);
3458
+ return false;
3459
+ }
3460
+ }
3461
+ }
3462
+ }
3463
+
3464
+ return true;
3465
+ };
3466
+
3467
+ /**
3468
+ * Validates json schemas for all the filters.json and filters_i18n.json found in platforms path
3469
+ *
3470
+ * @param platformsPath - Path to platforms folder
3471
+ * @param jsonSchemasConfigDir - Path to json schemas config folder
3472
+ * @param filtersRequiredAmount - Minimum required amount of filters
3473
+ */
3474
+ const validate$1 = (platformsPath, jsonSchemasConfigDir, filtersRequiredAmount) => {
3475
+ logger.info('Validating json schemas for platforms');
3476
+
3477
+ const schemas = loadSchemas(jsonSchemasConfigDir);
3478
+
3479
+ const oldSchemasMacV1 = loadSchemas(path.join(jsonSchemasConfigDir, OLD_MAC_V1_PLATFORM));
3480
+ const oldSchemasMacV2 = loadSchemas(path.join(jsonSchemasConfigDir, OLD_MAC_V2_PLATFORM));
3481
+ const oldSchemas = {
3482
+ [OLD_MAC_V1_PLATFORM]: oldSchemasMacV1,
3483
+ [OLD_MAC_V2_PLATFORM]: oldSchemasMacV2,
3484
+ };
3485
+
3486
+ const ajv = new Ajv({
3487
+ allErrors: true,
3488
+ useDefaults: true,
3489
+ });
3490
+
3491
+ const result = validateDir(platformsPath, ajv, schemas, oldSchemas, filtersRequiredAmount);
3492
+
3493
+ logger.info('Validating json schemas for platforms - done');
3494
+ logger.info(`Validation result: ${result}`);
3495
+
3496
+ return result;
3497
+ };
3498
+
3499
+ const schemaValidator = { validate: validate$1 };
3500
+
3501
+ /* eslint-disable global-require */
3502
+
3503
+ const __dirname$1 = path.dirname(new URL(import.meta.url).pathname);
3504
+
3505
+ /**
3506
+ * Each filter, group, tag should have two keys.
3507
+ */
3508
+ const REQUIRED_ENDINGS = [
3509
+ 'name',
3510
+ 'description',
3511
+ ];
3512
+
3513
+ const LOCALES_FILE_EXTENSION = '.json';
3514
+ const BASE_LOCALE = 'en';
3515
+
3516
+ const REQUIRED_FILES = [
3517
+ 'filters',
3518
+ 'groups',
3519
+ 'tags',
3520
+ ].map((el) => `${el}${LOCALES_FILE_EXTENSION}`);
3521
+
3522
+ // each message key should consist of three parts
3523
+ // e.g. 'filter.3.name' or 'tag.29.description'
3524
+ const MESSAGE_KEY_NAME_PARTS_COUNT = 3;
3525
+
3526
+ const WARNING_REASONS = {
3527
+ MISSED_FILES: 'missed files',
3528
+ NO_MESSAGES: 'empty file or no messages in file',
3529
+ INVALID_DATA_OBJ: 'invalid or absent message key/value',
3530
+ };
3531
+
3532
+ const WARNING_TYPES = {
3533
+ CRITICAL: 'critical',
3534
+ LOW: 'low',
3535
+ };
3536
+
3537
+ /**
3538
+ * Sync reads file content
3539
+ * @param filePath - path to locales file
3540
+ */
3541
+ const readFile = (filePath) => fs.readFileSync(path.resolve(__dirname$1, filePath), 'utf8');
3542
+
3543
+ /**
3544
+ * Sync reads directory content
3545
+ * @param dirPath - path to directory
3546
+ */
3547
+ const readDir = (dirPath) => fs.readdirSync(path.resolve(__dirname$1, dirPath), 'utf8');
3548
+
3549
+ /**
3550
+ * Validates messages keys
3551
+ * @param {Array} keys locale messages keys
3552
+ * @param {string} id filters / groups / tags
3553
+ */
3554
+ const areValidMessagesKeys = (keys, id) => {
3555
+ if (keys.length !== REQUIRED_ENDINGS.length) {
3556
+ return false;
3557
+ }
3558
+ const areValidKeys = !keys
3559
+ .find((key) => {
3560
+ const keyNameParts = key.split('.');
3561
+ const propPrefix = id.slice(0, -1);
3562
+ const filterId = Number(keyNameParts[1]);
3563
+ return keyNameParts.length !== MESSAGE_KEY_NAME_PARTS_COUNT
3564
+ || keyNameParts[0] !== propPrefix
3565
+ || !(Number.isInteger(filterId))
3566
+ || !(filterId > 0)
3567
+ || !(REQUIRED_ENDINGS.includes(keyNameParts[2]));
3568
+ });
3569
+ return areValidKeys;
3570
+ };
3571
+
3572
+ /**
3573
+ * Validates locale messages values
3574
+ * @param {string[]} values
3575
+ */
3576
+ const areValidMessagesValues = (values) => values.every((v) => v !== '');
3577
+
3578
+ /**
3579
+ * Prepares invalid locales data object for results
3580
+ * @param {Object} obj iterable locales messages object
3581
+ * @returns {Array}
3582
+ */
3583
+ const prepareWarningDetails = (obj) => Object.entries(obj).map(([key, value]) => `"${key}": "${value}"`);
3584
+
3585
+ /**
3586
+ * Returns map of base locale keys
3587
+ * @param dirPath
3588
+ */
3589
+ const getBaseLocaleKeys = (dirPath) => {
3590
+ const baseLocaleKeys = {};
3591
+
3592
+ const baseLocalePath = path.join(dirPath, BASE_LOCALE);
3593
+ const baseLocaleFiles = readDir(baseLocalePath);
3594
+
3595
+ baseLocaleFiles.forEach((fileName) => {
3596
+ const baseLocaleData = JSON.parse(readFile(path.join(baseLocalePath, fileName)));
3597
+ baseLocaleKeys[fileName] = baseLocaleData.flatMap((entry) => Object.keys(entry));
3598
+ });
3599
+ return baseLocaleKeys;
3600
+ };
3601
+
3602
+ /**
3603
+ * Compares messagesData keys to base locale keys
3604
+ * @param baseLocaleKeys
3605
+ * @param messagesData
3606
+ * @param localeWarnings
3607
+ */
3608
+ const compareKeys = (baseLocaleKeys, messagesData, localeWarnings) => {
3609
+ const messagesDataKeys = messagesData.flatMap((entry) => Object.keys(entry));
3610
+
3611
+ baseLocaleKeys.forEach((entry) => {
3612
+ if (!messagesDataKeys.includes(entry)) {
3613
+ localeWarnings.push([
3614
+ WARNING_TYPES.CRITICAL,
3615
+ WARNING_REASONS.INVALID_DATA_OBJ,
3616
+ [entry],
3617
+ ]);
3618
+ }
3619
+ });
3620
+ };
3621
+
3622
+ /**
3623
+ * Prepares raw warnings for results
3624
+ * @param {Array[]} warnings collected raw warnings
3625
+ * @returns {Warning[]}
3626
+ */
3627
+ const prepareWarnings = (warnings) => warnings.map(([type, reason, details]) => ({ type, reason, details }));
3628
+
3629
+ /**
3630
+ * @typedef {Object} Warning
3631
+ * @property {string} type
3632
+ * @property {string} reason
3633
+ * @property {string[]} details
3634
+ */
3635
+
3636
+ /**
3637
+ * @typedef {Object} Result
3638
+ * @property {string} locale
3639
+ * @property {Warning[]} warnings
3640
+ */
3641
+
3642
+ /**
3643
+ * Logs collected results of locales validation
3644
+ * @param {Result[]} results
3645
+ * @returns {string}
3646
+ */
3647
+ const createLog = (results) => {
3648
+ const log = [];
3649
+ log.push('There are issues with:');
3650
+ results.forEach((res) => {
3651
+ log.push(`- ${res.locale}:`);
3652
+ res.warnings.forEach((warning) => {
3653
+ log.push(` - ${warning.type} priority - ${warning.reason}:`);
3654
+ warning.details.forEach((detail) => {
3655
+ log.push(` ${detail}`);
3656
+ });
3657
+ });
3658
+ });
3659
+ return log.join('\n');
3660
+ };
3661
+
3662
+ /**
3663
+ * @typedef {Object} ValidationResult
3664
+ * @property {boolean} ok
3665
+ * @property {Result[]} data
3666
+ * @property {string} log
3667
+ */
3668
+
3669
+ /**
3670
+ * Validates locales messages
3671
+ * @param {string} dirPath relative path to locales directory
3672
+ * @returns {ValidationResult}
3673
+ */
3674
+ const validate = (dirPath, requiredLocales) => {
3675
+ logger.info('Validating locales...');
3676
+ const results = [];
3677
+ let locales;
3678
+ try {
3679
+ locales = readDir(dirPath);
3680
+ } catch (e) {
3681
+ throw new Error(`There is no locales dir '${dirPath}'`);
3682
+ }
3683
+
3684
+ if (locales.length === 0) {
3685
+ throw new Error(`Locales dir '${dirPath}' is empty`);
3686
+ }
3687
+
3688
+ const baseLocaleKeysMap = getBaseLocaleKeys(dirPath);
3689
+
3690
+ locales.forEach((locale) => {
3691
+ const localeWarnings = [];
3692
+ const filesList = readDir(path.join(dirPath, locale));
3693
+ // checks all needed files presence
3694
+ const missedFiles = REQUIRED_FILES
3695
+ .filter((el) => !filesList.includes(el));
3696
+ if (missedFiles.length !== 0) {
3697
+ localeWarnings.push([
3698
+ // if there are missedFiles, we consider it's critical
3699
+ WARNING_TYPES.CRITICAL,
3700
+ WARNING_REASONS.MISSED_FILES,
3701
+ missedFiles,
3702
+ ]);
3703
+ }
3704
+
3705
+ const presentFiles = REQUIRED_FILES
3706
+ .filter((el) => !missedFiles.includes(el));
3707
+
3708
+ // iterate over existent files
3709
+ presentFiles.forEach((fileName) => {
3710
+ const messagesPath = path.join(dirPath, locale, fileName);
3711
+ let messagesData;
3712
+ try {
3713
+ messagesData = JSON.parse(readFile(messagesPath));
3714
+ } catch (e) {
3715
+ localeWarnings.push([
3716
+ // if there is invalid data format, we consider it's critical
3717
+ WARNING_TYPES.CRITICAL,
3718
+ WARNING_REASONS.NO_MESSAGES,
3719
+ [fileName],
3720
+ ]);
3721
+ return;
3722
+ }
3723
+
3724
+ if (messagesData.length === 0) {
3725
+ // for some locales there is no translations
3726
+ // so it should bt critical only for required (our) locales
3727
+ const warningType = requiredLocales.includes(locale)
3728
+ ? WARNING_TYPES.CRITICAL
3729
+ : WARNING_TYPES.LOW;
3730
+ localeWarnings.push([
3731
+ warningType,
3732
+ WARNING_REASONS.NO_MESSAGES,
3733
+ [fileName],
3734
+ ]);
3735
+ }
3736
+
3737
+ if (requiredLocales.includes(locale)) {
3738
+ // check if all keys from base locale are presented in messagesData
3739
+ compareKeys(baseLocaleKeysMap[fileName], messagesData, localeWarnings);
3740
+ }
3741
+
3742
+ messagesData.forEach((obj) => {
3743
+ const messagesKeys = Object.keys(obj);
3744
+ const messagesValues = Object.values(obj);
3745
+ const extensionLength = LOCALES_FILE_EXTENSION.length;
3746
+ const id = fileName.slice(0, -extensionLength);
3747
+ if (!areValidMessagesKeys(messagesKeys, id)
3748
+ || !areValidMessagesValues(messagesValues)) {
3749
+ localeWarnings.push([
3750
+ // invalid messages data object is always critical
3751
+ WARNING_TYPES.CRITICAL,
3752
+ WARNING_REASONS.INVALID_DATA_OBJ,
3753
+ prepareWarningDetails(obj),
3754
+ ]);
3755
+ }
3756
+ });
3757
+ });
3758
+
3759
+ if (localeWarnings.length !== 0) {
3760
+ const warnings = prepareWarnings(localeWarnings);
3761
+ results.push({ locale, warnings });
3762
+ }
3763
+ });
3764
+
3765
+ if (results.length === 0) {
3766
+ logger.info('Validation result: OK');
3767
+ return { ok: true };
3768
+ }
3769
+
3770
+ const isOK = !results
3771
+ .some((res) => {
3772
+ const isCriticalWarning = res.warnings
3773
+ .some((warning) => warning.type === WARNING_TYPES.CRITICAL);
3774
+ return isCriticalWarning;
3775
+ });
3776
+ const resultsLog = createLog(results);
3777
+ if (isOK) {
3778
+ logger.warn(resultsLog);
3779
+ } else {
3780
+ logger.error(resultsLog);
3781
+ }
3782
+
3783
+ return { ok: isOK, data: results, log: resultsLog };
3784
+ };
3785
+
3786
+ const localesValidator = { validate };
3787
+
3788
+ /**
3789
+ * @file Platforms configuration.
3790
+ *
3791
+ * It shall be overridden by custom configuration:
3792
+ * @see {@link https://github.com/AdguardTeam/FiltersRegistry/blob/master/scripts/build/custom_platforms.js}
3793
+ *
3794
+ * IMPORTANT: During making any changes in this file,
3795
+ * the custom_platforms.js should also be updated through PR on GitHub.
3796
+ */
3797
+
3798
+ /**
3799
+ * Pattern to check if rule contains `$domain` modifier with regular expression
3800
+ *
3801
+ * In Safari, `if-domain` and `unless-domain` do not support regexps, only `*`
3802
+ * https://github.com/AdguardTeam/FiltersRegistry/pull/806
3803
+ *
3804
+ * @example
3805
+ * ```[$domain=/^inattv\d+\.pro$/]#%#//scriptlet('set-constant', 'config.adv', 'emptyObj')```
3806
+ */
3807
+ const DOMAIN_WITH_REGEXPS_PATTERNS = [
3808
+ '\\$domain=\/',
3809
+ ',domain=\/',
3810
+ ];
3811
+
3812
+ /**
3813
+ * Pattern to check if rule contains `$all` modifier
3814
+ *
3815
+ * @example
3816
+ * ```/?t=popunder&$all```
3817
+ */
3818
+ const ALL_MODIFIER_PATTERNS = [
3819
+ '\\$all',
3820
+ ];
3821
+
3822
+ /**
3823
+ * Pattern to check if rule contains `$mp4` modifier
3824
+ *
3825
+ * @example
3826
+ * ```Deprecated, use $redirect=noopmp4-1s instead```
3827
+ */
3828
+ const MP4_MODIFIER_PATTERNS = [
3829
+ '\\$(.*,)?mp4',
3830
+ ];
3831
+
3832
+ /**
3833
+ * Pattern to check if rule contains `$network` modifier
3834
+ *
3835
+ * @example
3836
+ * ```57.128.71.215$network```
3837
+ */
3838
+ const NETWORK_MODIFIER_PATTERNS = [
3839
+ '\\$network',
3840
+ ];
3841
+
3842
+ /**
3843
+ * Pattern to check if rule contains `$webrtc` modifier
3844
+ *
3845
+ * @example
3846
+ * ```Removed and no longer supported```
3847
+ */
3848
+ const WEBRTC_MODIFIER_PATTERNS = [
3849
+ '\\$webrtc',
3850
+ ];
3851
+
3852
+ /**
3853
+ * Pattern to check if rule contains `$csp` modifier
3854
+ *
3855
+ * @example
3856
+ * ```||deloplen.com^$csp=script-src 'none'```
3857
+ */
3858
+ const CSP_MODIFIER_PATTERNS = [
3859
+ '\\$csp',
3860
+ ];
3861
+
3862
+ /**
3863
+ * Pattern to check if rule contains `$$` modifier
3864
+ *
3865
+ * Do not exclude scriptlets which contain '$$' when excluding '$$' and '$@$' rules
3866
+ * https://github.com/AdguardTeam/FiltersRegistry/issues/731
3867
+ *
3868
+ * @example
3869
+ * ```mail.com$$script[tag-content="uabp"][min-length="20000"][max-length="300000"]```
3870
+ */
3871
+ const HTML_FILTERING_MODIFIER_PATTERNS = [
3872
+ '^((?!#%#).)*\\$\\$|\\$\\@\\$',
3873
+ ];
3874
+
3875
+ /**
3876
+ * Pattern to check if rule contains `$protobuf` modifier
3877
+ * and does not contain non-basic modifiers like `$domain` or `$path`.
3878
+ *
3879
+ * Pattern parts:
3880
+ * - `\\$` — modifiers divider
3881
+ * - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
3882
+ * `[$path=...]##.textad,protobuf` and rules with `$removeparam` modifier like `$removeparam=protobuf`
3883
+ * - `.*protobuf` — protobuf modifier itself
3884
+ * - `(,|=|$)` — end of line or modifiers divider, as `$protobuf` can be followed by other modifiers (`,`),
3885
+ * it may have a value (`=`), or it may be the last modifier in the rule (`$`).
3886
+ *
3887
+ * Currently it is not supported, but it can be added in the future
3888
+ * https://github.com/AdguardTeam/CoreLibs/issues/1778
3889
+ */
3890
+ const PROTOBUF_MODIFIER_PATTERNS = [
3891
+ '\\$(?!#|(path|domain)=.*]|.*removeparam=).*protobuf(,|=|$)',
3892
+ ];
3893
+
3894
+ /**
3895
+ * Pattern to check if rule contains `$app` modifier
3896
+ * and does not contain non-basic modifiers like `$domain` or `$path`.
3897
+ *
3898
+ * Pattern parts:
3899
+ * - `\\$` — modifiers divider
3900
+ * - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
3901
+ * `[$path=...]##.textad,[app="ads"]` and rules with `$removeparam` modifier like `$removeparam=app=ads`
3902
+ * - `.*app=` — app= modifier itself
3903
+ *
3904
+ * @example
3905
+ * ```@@||imasdk.googleapis.com^$app=tv.htv.app```
3906
+ */
3907
+ const APP_MODIFIER_PATTERNS = [
3908
+ '\\$(?!#|(path|domain)=.*]|.*removeparam=).*app=',
3909
+ ];
3910
+
3911
+ /**
3912
+ * Pattern to check if rule contains `$extension` modifier
3913
+ * and does not contain non-basic modifiers like `$domain` or `$path`.
3914
+ *
3915
+ * Pattern parts:
3916
+ * - `\\$` — modifiers divider
3917
+ * - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
3918
+ * `[$path=...]##.textad,extension` and rules with `$removeparam` modifier like `$removeparam=extension`
3919
+ * - `.*extension` — extension modifier itself
3920
+ * - `(,|=|$)` — end of line or modifiers divider, as `$extension` can be followed by other modifiers (`,`),
3921
+ * it may have a value (`=`), or it may be the last modifier in the rule (`$`).
3922
+ *
3923
+ * @example
3924
+ * ```@@||radar.cloudflare.com^$elemhide,extension,content```
3925
+ */
3926
+ const EXTENSION_MODIFIER_PATTERNS = [
3927
+ '\\$(?!#|(path|domain)=.*]|.*removeparam=).*extension(,|=|$)',
3928
+ ];
3929
+
3930
+ /**
3931
+ * Pattern to check if rule contains only `$content` modifier.
3932
+ *
3933
+ * @example
3934
+ * ```@@||telegram.hr^$content```
3935
+ */
3936
+ const ONLY_CONTENT_MODIFIER_PATTERNS = [
3937
+ '\\$content$',
3938
+ ];
3939
+
3940
+ /**
3941
+ * Pattern to check if rule contains `$content` modifier
3942
+ * and does not contain non-basic modifiers like `$domain` or `$path`.
3943
+ *
3944
+ * Pattern parts:
3945
+ * - `\\$` — modifiers divider
3946
+ * - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
3947
+ * `[$path=...]##.textad,content` and rules with `$removeparam` modifier like `$removeparam=content`
3948
+ * - `.*content` — content modifier itself
3949
+ * - `(,|$)` — end of line or modifiers divider, as `$content` can be followed by other modifiers (`,`).
3950
+ *
3951
+ * @example
3952
+ * ```@@||dnsleaktest.com^$content,elemhide,jsinject```
3953
+ */
3954
+ const CONTENT_MODIFIER_PATTERNS = [
3955
+ '\\$(?!#|(path|domain)=.*]|.*removeparam=).*content(,|$)',
3956
+ ];
3957
+
3958
+ /**
3959
+ * Pattern to check if rule contains `$jsinject` modifier
3960
+ * and does not contain non-basic modifiers like `$domain` or `$path`.
3961
+ *
3962
+ * Pattern parts:
3963
+ * - `\\$` — modifiers divider
3964
+ * - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
3965
+ * `[$path=...]##.textad,jsinject` and rules with `$removeparam` modifier like `$removeparam=jsinject`
3966
+ * - `.*jsinject` — jsinject modifier itself
3967
+ * - `(,|$)` — end of line or modifiers divider, as `$jsinject` can be followed by other modifiers (`,`).
3968
+ *
3969
+ * @example
3970
+ * ```@@://www.atlassian.com^$elemhide,jsinject,extension```
3971
+ */
3972
+ const JSINJECT_MODIFIER_PATTERNS = [
3973
+ '\\$(?!#|(path|domain)=.*]|.*removeparam=).*jsinject(,|$)',
3974
+ ];
3975
+
3976
+ /**
3977
+ * Pattern to check if rule contains `$urlblock` modifier
3978
+ * and does not contain non-basic modifiers like `$domain` or `$path`.
3979
+ *
3980
+ * Pattern parts:
3981
+ * - `\\$` — modifiers divider
3982
+ * - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
3983
+ * `[$path=...]##.textad,urlblock` and rules with `$removeparam` modifier like `$removeparam=urlblock`
3984
+ * - `.*urlblock` — urlblock modifier itself
3985
+ * - `(,|$)` — end of line or modifiers divider, as `$urlblock` can be followed by other modifiers (`,`).
3986
+ *
3987
+ * @example
3988
+ * ```@@||google.com/settings/ads/onweb$urlblock```
3989
+ */
3990
+ const URLBLOCK_MODIFIER_PATTERNS = [
3991
+ '\\$(?!#|(path|domain)=.*]|.*removeparam=).*urlblock(,|$)',
3992
+ ];
3993
+
3994
+ /**
3995
+ * Pattern to check if rule contains `$referrerpolicy` modifier
3996
+ * and does not contain non-basic modifiers like `$domain` or `$path`.
3997
+ *
3998
+ * Pattern parts:
3999
+ * - `\\$` — modifiers divider
4000
+ * - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
4001
+ * `[$path=...]##.textad,referrerpolicy` and rules with `$removeparam` modifier like `$removeparam=referrerpolicy`
4002
+ * - `.*referrerpolicy` — referrerpolicy modifier itself
4003
+ * - `(,|=|$)` — end of line or modifiers divider, as `$referrerpolicy` can be followed by other modifiers (`,`),
4004
+ * it may have a value (`=`), or it may be the last modifier in the rule (`$`).
4005
+ *
4006
+ * @example
4007
+ * ```||yallo.tv^$referrerpolicy=origin```
4008
+ */
4009
+ const REFERRERPOLICY_MODIFIER_PATTERNS = [
4010
+ '\\$(?!#|(path|domain)=.*]|.*removeparam=).*referrerpolicy(,|=|$)',
4011
+ ];
4012
+
4013
+ /**
4014
+ * Pattern to check if rule contains `$replace` modifier
4015
+ * and does not contain non-basic modifiers like `$domain` or `$path`.
4016
+ *
4017
+ * Pattern parts:
4018
+ * - `\\$` — modifiers divider
4019
+ * - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
4020
+ * `[$path=...]##.textad,[replace="ads"]` and rules with `$removeparam` modifier like `$removeparam=replace=ads`
4021
+ * - `.*replace` — replace modifier itself
4022
+ * - `(,|=|$)` — end of line or modifiers divider, as `$replace` can be followed by other modifiers (`,`),
4023
+ * it may have a value (`=`), or it may be the last modifier in the rule (`$`).
4024
+ *
4025
+ * @example
4026
+ * ```||pubads.g.doubleclick.net/gampad/live/ads?correlator=$replace=/(<VAST[\s\S]*?>)[\s\S]*<\/VAST>/\$1<\/VAST>/```
4027
+ */
4028
+ const REPLACE_MODIFIER_PATTERNS = [
4029
+ '\\$(?!#|(path|domain)=.*]|.*removeparam=).*replace(,|=|$)',
4030
+ ];
4031
+
4032
+ /* eslint-disable max-len */
4033
+
4034
+ /**
4035
+ * Pattern to check if rule contains `$hls` modifier
4036
+ * and does not contain non-basic modifiers like `$domain` or `$path`.
4037
+ *
4038
+ * Pattern parts:
4039
+ * - `\\$` — modifiers divider
4040
+ * - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
4041
+ * `[$path=...]##.textad,[hls="ads"]` and rules with `$removeparam` modifier like `$removeparam=hls=ads`
4042
+ * - `.*hls` — hls modifier itself
4043
+ * - `(,|=|$)` — end of line or modifiers divider, as `$hls` can be followed by other modifiers (`,`),
4044
+ * it may have a value (`=`), or it may be the last modifier in the rule (`$`).
4045
+ *
4046
+ * @example
4047
+ * ```||pubads.g.doubleclick.net/ondemand/hls/*.m3u8$hls=/redirector\.googlevideo\.com\/videoplayback[\s\S]*?dclk_video_ads/,domain=10play.com.au```
4048
+ */
4049
+ const HLS_MODIFIER_PATTERNS = [
4050
+ '\\$(?!#|(path|domain)=.*]|.*removeparam=).*hls(,|=|$)',
4051
+ ];
4052
+
4053
+ /**
4054
+ * Pattern to check if rule contains `$jsonprune` modifier
4055
+ * and does not contain non-basic modifiers like `$domain` or `$path`.
4056
+ *
4057
+ * Pattern parts:
4058
+ * - `\\$` — modifiers divider
4059
+ * - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
4060
+ * `[$path=...]##.textad,[jsonprune="ads"]` and rules with `$removeparam` modifier like `$removeparam=jsonprune`
4061
+ * - `.*jsonprune` — jsonprune modifier itself
4062
+ * - `(,|=|$)` — end of line or modifiers divider, as `$jsonprune` can be followed by other modifiers (`,`),
4063
+ * it may have a value (`=`), or it may be the last modifier in the rule (`$`).
4064
+ *
4065
+ * @example
4066
+ * ```.com/watch?v=$xmlhttprequest,jsonprune=\$..[adPlacements\, adSlots\, playerAds],domain=youtubekids.com|youtube-nocookie.com|youtube.com```
4067
+ */
4068
+ const JSONPRUNE_MODIFIER_PATTERNS = [
4069
+ '\\$(?!#|(path|domain)=.*]|.*removeparam=).*jsonprune(,|=|$)',
4070
+ ];
4071
+
4072
+ /* eslint-enable max-len */
4073
+
4074
+ /**
4075
+ * Pattern to check if rule contains `$removeparam` modifier
4076
+ * and does not contain non-basic modifiers like `$domain` or `$path`.
4077
+ *
4078
+ * Pattern parts:
4079
+ * - `\\$` — modifiers divider
4080
+ * - `(?!#|(path|domain)=.*]).*` — negative lookahead to exclude CSS rules (#$#) and rules like
4081
+ * `[$path=...]##.textad,[removeparam="ads"]`
4082
+ * - `.*removeparam` — removeparam modifier itself
4083
+ * - `(,|=|$)` — end of line or modifiers divider, as `$removeparam` can be followed by other modifiers (`,`),
4084
+ * it may have a value (`=`), or it may be the last modifier in the rule (`$`).
4085
+ *
4086
+ * @example
4087
+ * ```$removeparam=fb_ref```
4088
+ */
4089
+ const REMOVEPARAM_MODIFIER_PATTERNS = [
4090
+ '\\$(?!#|(path|domain)=.*]).*removeparam(,|=|$)',
4091
+ ];
4092
+
4093
+ /**
4094
+ * Pattern to check if rule contains `$removeheader` modifier
4095
+ * and does not contain non-basic modifiers like `$domain` or `$path`.
4096
+ *
4097
+ * Pattern parts:
4098
+ * - `\\$` — modifiers divider
4099
+ * - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
4100
+ * `[$path=...]##.textad,[removeheader="ads"]` and rules with `$removeparam` modifier like `$removeparam=removeheader`
4101
+ * - `.*removeheader` — removeheader modifier itself
4102
+ * - `(,|=|$)` — end of line or modifiers divider, as `$removeheader` can be followed by other modifiers (`,`),
4103
+ * it may have a value (`=`), or it may be the last modifier in the rule (`$`).
4104
+ *
4105
+ * @example
4106
+ * ```||dubznetwork.com^$removeheader=refresh```
4107
+ */
4108
+ const REMOVEHEADER_MODIFIER_PATTERNS = [
4109
+ '\\$(?!#|(path|domain)=.*]|.*removeparam=).*removeheader(,|=|$)',
4110
+ ];
4111
+
4112
+ /**
4113
+ * Pattern to check if rule contains `$stealth` modifier
4114
+ * and does not contain non-basic modifiers like `$domain` or `$path`.
4115
+ *
4116
+ * Pattern parts:
4117
+ * - `\\$` — modifiers divider
4118
+ * - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
4119
+ * `[$path=...]##.textad,[stealth="ads"]` and rules with `$removeparam` modifier like `$removeparam=stealth`
4120
+ * - `.*stealth` — stealth modifier itself
4121
+ * - `(,|=|$)` — end of line or modifiers divider, as `$stealth` can be followed by other modifiers (`,`),
4122
+ * it may have a value (`=`), or it may be the last modifier in the rule (`$`).
4123
+ *
4124
+ * @example
4125
+ * ```@@.php?play_vid=$subdocument,stealth=referrer,domain=xyflv.cc```
4126
+ */
4127
+ const STEALTH_MODIFIER_PATTERNS = [
4128
+ '\\$(?!#|(path|domain)=.*]|.*removeparam=).*stealth(,|=|$)',
4129
+ ];
4130
+
4131
+ /**
4132
+ * Pattern to check if rule contains `$cookie` modifier
4133
+ * and does not contain non-basic modifiers like `$domain` or `$path`.
4134
+ *
4135
+ * Pattern parts:
4136
+ * - `\\$` — modifiers divider
4137
+ * - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
4138
+ * `[$path=...]##.textad,cookie` and rules with `$removeparam` modifier like `$removeparam=cookie`
4139
+ * - `.*cookie` — cookie modifier itself
4140
+ * - `(,|=|$)` — end of line or modifiers divider, as `$cookie` can be followed by other modifiers (`,`),
4141
+ * it may have a value (`=`), or it may be the last modifier in the rule (`$`).
4142
+ *
4143
+ * @example
4144
+ * ```$cookie=_ga```
4145
+ */
4146
+ const COOKIE_MODIFIER_PATTERNS = [
4147
+ '\\$(?!#|(path|domain)=.*]|.*removeparam=).*cookie(,|=|$)',
4148
+ ];
4149
+
4150
+ /**
4151
+ * Pattern to check if rule contains `$redirect` modifier
4152
+ * and does not contain non-basic modifiers like `$domain` or `$path`.
4153
+ *
4154
+ * Pattern parts:
4155
+ * - `\\$` — modifiers divider
4156
+ * - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
4157
+ * `[$path=...]##.textad,redirect` and rules with `$removeparam` modifier like `$removeparam=redirect`
4158
+ * - `.*redirect` — redirect modifier itself
4159
+ * - `(,|=|$)` — end of line or modifiers divider, as `$redirect` can be followed by other modifiers (`,`),
4160
+ * it may have a value (`=`), or it may be the last modifier in the rule (`$`).
4161
+ *
4162
+ * @example
4163
+ * ```||google-analytics.com/analytics.js$script,redirect=google-analytics,domain=~olx.*|~banki.ru|~bigc.co.th```
4164
+ */
4165
+ const REDIRECT_MODIFIER_PATTERNS = [
4166
+ '\\$(?!#|(path|domain)=.*]|.*removeparam=).*redirect(,|=|$)',
4167
+ ];
4168
+
4169
+ /**
4170
+ * Pattern to check if rule contains `$redirect-rule` modifier
4171
+ * and does not contain non-basic modifiers like `$domain` or `$path`.
4172
+ *
4173
+ * Pattern parts:
4174
+ * - `\\$` — modifiers divider
4175
+ * - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
4176
+ * `[$path=...]##.textad,redirect-rule` and rules with `$removeparam` modifier like `$removeparam=redirect-rule`
4177
+ * - `.*redirect-rule` — redirect-rule modifier itself
4178
+ * - `(,|=|$)` — end of line or modifiers divider, as `$redirect-rule` can be followed by other modifiers (`,`),
4179
+ * it may have a value (`=`), or it may be the last modifier in the rule (`$`).
4180
+ *
4181
+ * @example
4182
+ * ```$script,third-party,redirect-rule=noopjs,domain=paraphraser.io```
4183
+ */
4184
+ const REDIRECT_RULE_MODIFIER_PATTERNS = [
4185
+ '\\$(?!#|(path|domain)=.*]|.*removeparam=).*redirect-rule(,|=|$)',
4186
+ ];
4187
+
4188
+ /**
4189
+ * Pattern to check if rule contains `$empty` modifier
4190
+ * and does not contain non-basic modifiers like `$domain` or `$path`.
4191
+ *
4192
+ * Pattern parts:
4193
+ * - `\\$` — modifiers divider
4194
+ * - `(?!#|(path|domain)=.*]|.*removeparam=).*` — negative lookahead to exclude CSS rules (#$#) and rules like
4195
+ * `[$path=...]##.textad,empty` and rules with `$removeparam` modifier like `$removeparam=empty`
4196
+ * - `.*empty` — empty modifier itself
4197
+ * - `(,|$)` — end of line or modifiers divider, as `$empty` can be followed by other modifiers (`,`).
4198
+ *
4199
+ * @example
4200
+ * ```Deprecated, use $redirect=nooptext instead```
4201
+ */
4202
+ const EMPTY_MODIFIER_PATTERNS = [
4203
+ '\\$(?!#|(path|domain)=.*]|.*removeparam=).*empty(,|$)',
4204
+ ];
4205
+
4206
+ /**
4207
+ * Pattern to detect scriptlets and JavaScript rules
4208
+ *
4209
+ * @example
4210
+ * ```w3resource.com#%#//scriptlet('prevent-setTimeout', 'ins.adsbygoogle')```
4211
+ * @example
4212
+ * ```meczyki.pl#%#!function(){window.YLHH={bidder:{startAuction:function(){}}};}();```
4213
+ */
4214
+ const JAVASCRIPT_RULES_PATTERNS = [
4215
+ '#%#',
4216
+ '#@%#',
4217
+ ];
4218
+
4219
+ /**
4220
+ * Pattern to detect CSS rules
4221
+ *
4222
+ * @example
4223
+ * ```windowslite.net#$#body { overflow: auto !important; }```
4224
+ */
4225
+ const CSS_RULES_PATTERNS = [
4226
+ '#\\$#',
4227
+ '#@\\$#',
4228
+ ];
4229
+
4230
+ /**
4231
+ * Pattern to detect CSS rules with `@media` queries
4232
+ *
4233
+ * @example
4234
+ * ```windowslite.net#$#body { overflow: auto !important; }```
4235
+ */
4236
+ const CSS_MEDIA_RULES_PATTERNS = [
4237
+ '#\\$#@media ',
4238
+ ];
4239
+
4240
+ /**
4241
+ * Patterns to match unblocking basic rules with `$important` modifier
4242
+ * which is not supported by uBlock Origin.
4243
+ *
4244
+ * @see {@link https://github.com/AdguardTeam/FiltersCompiler/issues/200}
4245
+ */
4246
+ const UNBLOCKING_IMPORTANT_RULES_PATTERNS = [
4247
+ '@@.*?(\\$|,)important',
4248
+ ];
4249
+
4250
+ /**
4251
+ * Pattern to detect Extended CSS rules
4252
+ *
4253
+ * @example
4254
+ * ```xup.in#?##xupab```
4255
+ * @example
4256
+ * ```lunar.az#?#.sagpanel div[class^="yenisb"]:contains(Reklam)```
4257
+ * @example
4258
+ * ```haal.fashion#?#div:has(> div > div > div.dfp-ad-unit)```
4259
+ */
4260
+ const EXTENDED_CSS_RULES_PATTERNS = [
4261
+ '\\[-ext-',
4262
+ ':has\\(',
4263
+ ':has-text\\(',
4264
+ ':contains\\(',
4265
+ ':matches-css\\(',
4266
+ ':matches-attr\\(',
4267
+ ':matches-property\\(',
4268
+ ':xpath\\(',
4269
+ ':nth-ancestor\\(',
4270
+ ':upward\\(',
4271
+ ':remove\\(',
4272
+ ':matches-css-before\\(',
4273
+ ':matches-css-after\\(',
4274
+ ':-abp-has\\(',
4275
+ ':-abp-contains\\(',
4276
+ '#\\?#',
4277
+ '#\\$\\?#',
4278
+ '#@\\?#',
4279
+ '#@\\$\\?#',
4280
+ ];
4281
+
4282
+ /**
4283
+ * Used for `EXTENSION_CHROMIUM`, `EXTENSION_CHROMIUM_MV3`, `EXTENSION_EDGE`, and `EXTENSION_OPERA` platforms.
4284
+ */
4285
+ const CHROMIUM_BASED_EXTENSION_PATTERNS = [
4286
+ ...HTML_FILTERING_MODIFIER_PATTERNS,
4287
+ ...REPLACE_MODIFIER_PATTERNS,
4288
+ ...APP_MODIFIER_PATTERNS,
4289
+ ...NETWORK_MODIFIER_PATTERNS,
4290
+ ...PROTOBUF_MODIFIER_PATTERNS,
4291
+ ...EXTENSION_MODIFIER_PATTERNS,
4292
+ ...HLS_MODIFIER_PATTERNS,
4293
+ ...JSONPRUNE_MODIFIER_PATTERNS,
4294
+ ...REFERRERPOLICY_MODIFIER_PATTERNS,
4295
+ ...CONTENT_MODIFIER_PATTERNS,
4296
+ ];
4297
+
4298
+ /**
4299
+ * Used for `EXTENSION_SAFARI` and `IOS` platforms.
4300
+ */
4301
+ const SAFARI_BASED_EXTENSION_PATTERNS = [
4302
+ ...DOMAIN_WITH_REGEXPS_PATTERNS,
4303
+ ...HTML_FILTERING_MODIFIER_PATTERNS,
4304
+ ...EXTENSION_MODIFIER_PATTERNS,
4305
+ ...REMOVEPARAM_MODIFIER_PATTERNS,
4306
+ ...REMOVEHEADER_MODIFIER_PATTERNS,
4307
+ ...MP4_MODIFIER_PATTERNS,
4308
+ ...REPLACE_MODIFIER_PATTERNS,
4309
+ ...STEALTH_MODIFIER_PATTERNS,
4310
+ ...COOKIE_MODIFIER_PATTERNS,
4311
+ ...APP_MODIFIER_PATTERNS,
4312
+ ...PROTOBUF_MODIFIER_PATTERNS,
4313
+ ...REDIRECT_MODIFIER_PATTERNS,
4314
+ ...REDIRECT_RULE_MODIFIER_PATTERNS,
4315
+ ...EMPTY_MODIFIER_PATTERNS,
4316
+ ...WEBRTC_MODIFIER_PATTERNS,
4317
+ ...CSP_MODIFIER_PATTERNS,
4318
+ ...ONLY_CONTENT_MODIFIER_PATTERNS,
4319
+ ...NETWORK_MODIFIER_PATTERNS,
4320
+ ...REFERRERPOLICY_MODIFIER_PATTERNS,
4321
+ ...HLS_MODIFIER_PATTERNS,
4322
+ ...JSONPRUNE_MODIFIER_PATTERNS,
4323
+ ];
4324
+
4325
+ /* eslint-disable max-len */
4326
+ /**
4327
+ * Pattern to detect Extended CSS `:matches-property()` rules
4328
+ *
4329
+ * @example
4330
+ * ```unsplash.com#?#.ripi6 > div:matches-property(/__reactFiber/.return.return.memoizedProps.ad)```
4331
+ * @example
4332
+ * ```
4333
+ * androidauthority.com#?#main div[class]:has(> div[class]:matches-property(/__reactFiber/.return.memoizedProps.type=med_rect_atf))
4334
+ * ```
4335
+ */
4336
+ const CSS_MATCHES_PROPERTY_RULES_PATTERNS = [
4337
+ ':matches-property\\(',
4338
+ ];
4339
+
4340
+ /**
4341
+ * Pattern to detect generic CSS rules
4342
+ *
4343
+ * @example
4344
+ * ```#$#div[class="adsbygoogle"][id="ad-detector"] { display: block !important; }```
4345
+ * @example
4346
+ * ```#$#.pub_728x90.text-ad.textAd.text_ad.text_ads.text-ads.text-ad-links { display: block !important; }```
4347
+ */
4348
+ const CSS_GENERIC_RULES_PATTERNS = [
4349
+ '^#\\$#',
4350
+ ];
4351
+
4352
+ const platformsConfig = {
4353
+ 'WINDOWS': {
4354
+ 'platform': 'windows',
4355
+ 'path': 'windows',
4356
+ 'expires': '12 hours',
4357
+ 'configuration': {
4358
+ 'ignoreRuleHints': false,
4359
+ 'replacements': null,
4360
+ },
4361
+ 'defines': {
4362
+ 'adguard': true,
4363
+ 'adguard_app_windows': true,
4364
+ },
4365
+ },
4366
+ 'MAC': {
4367
+ 'platform': 'mac',
4368
+ 'path': 'mac',
4369
+ 'configuration': {
4370
+ 'ignoreRuleHints': false,
4371
+ 'replacements': null,
4372
+ },
4373
+ 'defines': {
4374
+ 'adguard': true,
4375
+ 'adguard_app_mac': true,
4376
+ },
4377
+ },
4378
+ 'MAC_V2': {
4379
+ 'platform': 'mac',
4380
+ 'path': 'mac_v2',
4381
+ 'expires': '12 hours',
4382
+ 'configuration': {
4383
+ 'ignoreRuleHints': false,
4384
+ 'replacements': null,
4385
+ },
4386
+ 'defines': {
4387
+ 'adguard': true,
4388
+ 'adguard_app_mac': true,
4389
+ },
4390
+ },
4391
+ 'MAC_V3': {
4392
+ 'platform': 'mac',
4393
+ 'path': 'mac_v3',
4394
+ 'expires': '12 hours',
4395
+ 'configuration': {
4396
+ 'ignoreRuleHints': false,
4397
+ 'replacements': null,
4398
+ },
4399
+ 'defines': {
4400
+ 'adguard': true,
4401
+ 'adguard_app_mac': true,
4402
+ },
4403
+ },
4404
+ 'ANDROID': {
4405
+ 'platform': 'android',
4406
+ 'path': 'android',
4407
+ 'configuration': {
4408
+ 'ignoreRuleHints': false,
4409
+ 'replacements': null,
4410
+ },
4411
+ 'defines': {
4412
+ 'adguard': true,
4413
+ 'adguard_app_android': true,
4414
+ },
4415
+ },
4416
+ 'CLI': {
4417
+ 'platform': 'cli',
4418
+ 'path': 'cli',
4419
+ 'expires': '12 hours',
4420
+ 'configuration': {
4421
+ 'ignoreRuleHints': false,
4422
+ 'replacements': null,
4423
+ },
4424
+ 'defines': {
4425
+ 'adguard': true,
4426
+ 'adguard_app_cli': true,
4427
+ },
4428
+ },
4429
+ 'EXTENSION_CHROMIUM': {
4430
+ 'platform': 'ext_chromium',
4431
+ 'path': 'extension/chromium',
4432
+ 'expires': '10 days',
4433
+ 'configuration': {
4434
+ 'removeRulePatterns': CHROMIUM_BASED_EXTENSION_PATTERNS,
4435
+ 'replacements': null,
4436
+ 'ignoreRuleHints': false,
4437
+ },
4438
+ 'defines': {
4439
+ 'adguard': true,
4440
+ 'adguard_ext_chromium': true,
4441
+ },
4442
+ },
4443
+ 'EXTENSION_CHROMIUM_MV3': {
4444
+ 'platform': 'ext_chromium_mv3',
4445
+ 'path': 'extension/chromium-mv3',
4446
+ 'expires': '10 days',
4447
+ 'configuration': {
4448
+ 'removeRulePatterns': [
4449
+ ...CHROMIUM_BASED_EXTENSION_PATTERNS,
4450
+ ...REDIRECT_MODIFIER_PATTERNS,
4451
+ ],
4452
+ 'replacements': null,
4453
+ 'ignoreRuleHints': false,
4454
+ },
4455
+ 'defines': {
4456
+ 'adguard': true,
4457
+ 'adguard_ext_chromium_mv3': true,
4458
+ },
4459
+ },
4460
+ 'EXTENSION_EDGE': {
4461
+ 'platform': 'ext_edge',
4462
+ 'path': 'extension/edge',
4463
+ 'expires': '10 days',
4464
+ 'configuration': {
4465
+ 'removeRulePatterns': CHROMIUM_BASED_EXTENSION_PATTERNS,
4466
+ 'replacements': null,
4467
+ 'ignoreRuleHints': false,
4468
+ },
4469
+ 'defines': {
4470
+ 'adguard': true,
4471
+ 'adguard_ext_edge': true,
4472
+ 'adguard_ext_chromium': true,
4473
+ },
4474
+ },
4475
+ 'EXTENSION_OPERA': {
4476
+ 'platform': 'ext_opera',
4477
+ 'path': 'extension/opera',
4478
+ 'expires': '10 days',
4479
+ 'configuration': {
4480
+ 'removeRulePatterns': CHROMIUM_BASED_EXTENSION_PATTERNS,
4481
+ 'replacements': null,
4482
+ 'ignoreRuleHints': false,
4483
+ },
4484
+ 'defines': {
4485
+ 'adguard': true,
4486
+ 'adguard_ext_opera': true,
4487
+ 'adguard_ext_chromium': true,
4488
+ },
4489
+ },
4490
+ 'EXTENSION_FIREFOX': {
4491
+ 'platform': 'ext_ff',
4492
+ 'path': 'extension/firefox',
4493
+ 'expires': '10 days',
4494
+ 'configuration': {
4495
+ 'removeRulePatterns': [
4496
+ ...HTML_FILTERING_MODIFIER_PATTERNS,
4497
+ ...APP_MODIFIER_PATTERNS,
4498
+ ...NETWORK_MODIFIER_PATTERNS,
4499
+ ...PROTOBUF_MODIFIER_PATTERNS,
4500
+ ...EXTENSION_MODIFIER_PATTERNS,
4501
+ ...HLS_MODIFIER_PATTERNS,
4502
+ ...JSONPRUNE_MODIFIER_PATTERNS,
4503
+ ...REFERRERPOLICY_MODIFIER_PATTERNS,
4504
+
4505
+ ],
4506
+ 'replacements': null,
4507
+ 'ignoreRuleHints': false,
4508
+ },
4509
+ 'defines': {
4510
+ 'adguard': true,
4511
+ 'adguard_ext_firefox': true,
4512
+ },
4513
+ },
4514
+ 'EXTENSION_SAFARI': {
4515
+ 'platform': 'ext_safari',
4516
+ 'path': 'extension/safari',
4517
+ 'configuration': {
4518
+ 'removeRulePatterns': SAFARI_BASED_EXTENSION_PATTERNS,
4519
+ 'replacements': null,
4520
+ 'ignoreRuleHints': false,
4521
+ },
4522
+ 'defines': {
4523
+ 'adguard': true,
4524
+ 'adguard_ext_safari': true,
4525
+ },
4526
+ },
4527
+ 'IOS': {
4528
+ 'platform': 'ios',
4529
+ 'path': 'ios',
4530
+ 'configuration': {
4531
+ 'removeRulePatterns': SAFARI_BASED_EXTENSION_PATTERNS,
4532
+ 'replacements': null,
4533
+ 'ignoreRuleHints': false,
4534
+ },
4535
+ 'defines': {
4536
+ 'adguard': true,
4537
+ 'adguard_app_ios': true,
4538
+ },
4539
+ },
4540
+ 'EXTENSION_ANDROID_CONTENT_BLOCKER': {
4541
+ 'platform': 'ext_android_cb',
4542
+ 'path': 'extension/android-content-blocker',
4543
+ 'configuration': {
4544
+ 'removeRulePatterns': [
4545
+ ...DOMAIN_WITH_REGEXPS_PATTERNS,
4546
+ ...HTML_FILTERING_MODIFIER_PATTERNS,
4547
+ ...EXTENSION_MODIFIER_PATTERNS,
4548
+ ...REMOVEPARAM_MODIFIER_PATTERNS,
4549
+ ...REMOVEHEADER_MODIFIER_PATTERNS,
4550
+ ...JAVASCRIPT_RULES_PATTERNS,
4551
+ ...CSS_RULES_PATTERNS,
4552
+ ...MP4_MODIFIER_PATTERNS,
4553
+ ...REPLACE_MODIFIER_PATTERNS,
4554
+ ...STEALTH_MODIFIER_PATTERNS,
4555
+ ...COOKIE_MODIFIER_PATTERNS,
4556
+ ...EMPTY_MODIFIER_PATTERNS,
4557
+ ...APP_MODIFIER_PATTERNS,
4558
+ ...PROTOBUF_MODIFIER_PATTERNS,
4559
+ ...CSP_MODIFIER_PATTERNS,
4560
+ ...EXTENDED_CSS_RULES_PATTERNS,
4561
+ ...REDIRECT_MODIFIER_PATTERNS,
4562
+ ...REDIRECT_RULE_MODIFIER_PATTERNS,
4563
+ ...ONLY_CONTENT_MODIFIER_PATTERNS,
4564
+ ...ALL_MODIFIER_PATTERNS,
4565
+ ...NETWORK_MODIFIER_PATTERNS,
4566
+ ...REFERRERPOLICY_MODIFIER_PATTERNS,
4567
+ ...HLS_MODIFIER_PATTERNS,
4568
+ ...JSONPRUNE_MODIFIER_PATTERNS,
4569
+ ...JSINJECT_MODIFIER_PATTERNS,
4570
+ ...URLBLOCK_MODIFIER_PATTERNS,
4571
+ ],
4572
+ 'ignoreRuleHints': false,
4573
+ },
4574
+ 'defines': {
4575
+ 'adguard': true,
4576
+ 'adguard_ext_android_cb': true,
4577
+ },
4578
+ },
4579
+ 'EXTENSION_UBLOCK': {
4580
+ 'platform': 'ext_ublock',
4581
+ 'path': 'extension/ublock',
4582
+ 'configuration': {
4583
+ 'removeRulePatterns': [
4584
+ ...HTML_FILTERING_MODIFIER_PATTERNS,
4585
+ ...MP4_MODIFIER_PATTERNS,
4586
+ ...REPLACE_MODIFIER_PATTERNS,
4587
+ ...STEALTH_MODIFIER_PATTERNS,
4588
+ ...COOKIE_MODIFIER_PATTERNS,
4589
+ ...APP_MODIFIER_PATTERNS,
4590
+ ...NETWORK_MODIFIER_PATTERNS,
4591
+ ...PROTOBUF_MODIFIER_PATTERNS,
4592
+ ...EXTENSION_MODIFIER_PATTERNS,
4593
+ ...JSINJECT_MODIFIER_PATTERNS,
4594
+ ...URLBLOCK_MODIFIER_PATTERNS,
4595
+ ...CONTENT_MODIFIER_PATTERNS,
4596
+ ...WEBRTC_MODIFIER_PATTERNS,
4597
+ ...CSS_MEDIA_RULES_PATTERNS,
4598
+ ...HLS_MODIFIER_PATTERNS,
4599
+ ...REFERRERPOLICY_MODIFIER_PATTERNS,
4600
+ ...JSONPRUNE_MODIFIER_PATTERNS,
4601
+ ...UNBLOCKING_IMPORTANT_RULES_PATTERNS,
4602
+ ...REMOVEHEADER_MODIFIER_PATTERNS,
4603
+ ...CSS_MATCHES_PROPERTY_RULES_PATTERNS, // TODO: remove when this issue is fixed - https://github.com/AdguardTeam/FiltersCompiler/issues/252
4604
+ ...CSS_GENERIC_RULES_PATTERNS,
4605
+ ],
4606
+ 'ignoreRuleHints': false,
4607
+ 'adbHeader': '![Adblock Plus 2.0]',
4608
+ },
4609
+ 'defines': {
4610
+ 'ext_ublock': true,
4611
+ },
4612
+ },
4613
+ };
4614
+
4615
+ // Sets RuleConverter to use logger of current library
4616
+ setLogger(logger);
4617
+
4618
+ // Sets configuration compatibility
4619
+ setConfiguration({ compatibility: CompatibilityTypes.Corelibs });
4620
+
4621
+ const __dirname = path.dirname(new URL(import.meta.url).pathname);
4622
+
4623
+ const jsonSchemasConfigDir = path.join(__dirname, './schemas/');
4624
+
4625
+ process.on('unhandledRejection', (error) => {
4626
+ throw error;
4627
+ });
4628
+
4629
+ const compile = (path, logPath, reportFile, platformsPath, whitelist, blacklist, customPlatformsConfig) => {
4630
+ if (customPlatformsConfig) {
4631
+ logger.info('Using custom platforms configuration');
4632
+ // eslint-disable-next-line no-restricted-syntax, guard-for-in
4633
+ for (const platform in customPlatformsConfig) {
4634
+ logger.info(`Redefining platform ${platform}`);
4635
+ platformsConfig[platform] = customPlatformsConfig[platform];
4636
+ }
4637
+ }
4638
+
4639
+ return build(
4640
+ path,
4641
+ logPath,
4642
+ reportFile,
4643
+ platformsPath,
4644
+ platformsConfig,
4645
+ whitelist,
4646
+ blacklist,
4647
+ );
4648
+ };
4649
+
4650
+ const validateJSONSchema = (platformsPath, requiredFiltersAmount) => {
4651
+ return schemaValidator.validate(platformsPath, jsonSchemasConfigDir, requiredFiltersAmount);
4652
+ };
4653
+
4654
+ const validateLocales = (localesDirPath, requiredLocales) => {
4655
+ return localesValidator.validate(localesDirPath, requiredLocales);
4656
+ };
4657
+
4658
+ export { compile, validateJSONSchema, validateLocales };