@adguard/dnr-converter 1.0.0-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/README.md +476 -0
  2. package/dist/cli.cjs +58912 -0
  3. package/dist/constants.js +23 -0
  4. package/dist/declarative-rule/declarative-rule.js +15 -0
  5. package/dist/declarative-rule/header-info.js +12 -0
  6. package/dist/declarative-rule/modify-header-info.js +31 -0
  7. package/dist/declarative-rule/query-key-value.js +12 -0
  8. package/dist/declarative-rule/query-transform.js +12 -0
  9. package/dist/declarative-rule/redirect.js +14 -0
  10. package/dist/declarative-rule/rule-action.js +44 -0
  11. package/dist/declarative-rule/rule-condition.js +77 -0
  12. package/dist/declarative-rule/url-transform.js +30 -0
  13. package/dist/errors/conversion-errors/empty-domains-error.js +24 -0
  14. package/dist/errors/conversion-errors/empty-resources-error.js +24 -0
  15. package/dist/errors/conversion-errors/index.js +17 -0
  16. package/dist/errors/conversion-errors/invalid-declarative-rule-error.js +22 -0
  17. package/dist/errors/conversion-errors/unsupported-modifier-error.js +20 -0
  18. package/dist/errors/conversion-errors/unsupported-regexp-error.js +27 -0
  19. package/dist/errors/converter-options-errors/empty-or-negative-number-of-rules-error.js +18 -0
  20. package/dist/errors/converter-options-errors/index.js +15 -0
  21. package/dist/errors/converter-options-errors/negative-number-of-rules-error.js +18 -0
  22. package/dist/errors/converter-options-errors/resources-path-error.js +18 -0
  23. package/dist/errors/limitation-errors/index.js +17 -0
  24. package/dist/errors/limitation-errors/max-scanned-rules-error.js +20 -0
  25. package/dist/errors/limitation-errors/too-many-error.js +28 -0
  26. package/dist/errors/limitation-errors/too-many-regexp-rules-error.js +25 -0
  27. package/dist/errors/limitation-errors/too-many-rules-error.js +25 -0
  28. package/dist/errors/limitation-errors/too-many-unsafe-rules-error.js +25 -0
  29. package/dist/errors/unavailable-sources-errors/unavailable-filter-source-error.js +21 -0
  30. package/dist/errors/unavailable-sources-errors/unavailable-ruleset-source-error.js +21 -0
  31. package/dist/filter/filter.js +92 -0
  32. package/dist/filter-converter/filter-converter.js +385 -0
  33. package/dist/grouped-rules-converters/url-transform-converter.js +340 -0
  34. package/dist/index.js +30 -0
  35. package/dist/package.json.js +3 -0
  36. package/dist/re2-regexp/re2-validator.js +37 -0
  37. package/dist/re2-regexp/regex-validator-extension.js +30 -0
  38. package/dist/re2.wasm +0 -0
  39. package/dist/rule/option-names.js +65 -0
  40. package/dist/rule/rule-badfilter.js +81 -0
  41. package/dist/rule/rule-priority.js +88 -0
  42. package/dist/rule/rule-validator.js +455 -0
  43. package/dist/rule/rule.js +853 -0
  44. package/dist/rule/string-utils.js +49 -0
  45. package/dist/rule-converters/csp-converter.js +105 -0
  46. package/dist/rule-converters/regular-rule-converter.js +789 -0
  47. package/dist/rule-converters/remove-header-converter.js +97 -0
  48. package/dist/rule-converters/rules-converter.js +365 -0
  49. package/dist/rule-converters/rules-grouper.js +75 -0
  50. package/dist/rules-scanner/rules-scanner.js +136 -0
  51. package/dist/ruleset/metadata-rule.js +35 -0
  52. package/dist/ruleset/metadata-ruleset.js +162 -0
  53. package/dist/ruleset/rules-hash-map.js +71 -0
  54. package/dist/ruleset/ruleset-with-source-map.js +406 -0
  55. package/dist/ruleset/ruleset.js +54 -0
  56. package/dist/ruleset/source-map.js +283 -0
  57. package/dist/types/cli/convert-filters.d.ts +32 -0
  58. package/dist/types/cli/index.d.ts +7 -0
  59. package/dist/types/cli/utils.d.ts +19 -0
  60. package/dist/types/src/declarative-rule/declarative-rule.d.ts +32 -0
  61. package/dist/types/src/declarative-rule/header-info.d.ts +31 -0
  62. package/dist/types/src/declarative-rule/modify-header-info.d.ts +46 -0
  63. package/dist/types/src/declarative-rule/query-key-value.d.ts +24 -0
  64. package/dist/types/src/declarative-rule/query-transform.d.ts +21 -0
  65. package/dist/types/src/declarative-rule/redirect.d.ts +32 -0
  66. package/dist/types/src/declarative-rule/rule-action.d.ts +68 -0
  67. package/dist/types/src/declarative-rule/rule-condition.d.ts +262 -0
  68. package/dist/types/src/declarative-rule/url-transform.d.ts +69 -0
  69. package/dist/types/src/errors/conversion-errors/empty-domains-error.d.ts +21 -0
  70. package/dist/types/src/errors/conversion-errors/empty-resources-error.d.ts +21 -0
  71. package/dist/types/src/errors/conversion-errors/index.d.ts +27 -0
  72. package/dist/types/src/errors/conversion-errors/invalid-declarative-rule-error.d.ts +30 -0
  73. package/dist/types/src/errors/conversion-errors/unsupported-modifier-error.d.ts +20 -0
  74. package/dist/types/src/errors/conversion-errors/unsupported-regexp-error.d.ts +23 -0
  75. package/dist/types/src/errors/converter-options-errors/empty-or-negative-number-of-rules-error.d.ts +13 -0
  76. package/dist/types/src/errors/converter-options-errors/index.d.ts +23 -0
  77. package/dist/types/src/errors/converter-options-errors/negative-number-of-rules-error.d.ts +13 -0
  78. package/dist/types/src/errors/converter-options-errors/resources-path-error.d.ts +13 -0
  79. package/dist/types/src/errors/limitation-errors/index.d.ts +27 -0
  80. package/dist/types/src/errors/limitation-errors/max-scanned-rules-error.d.ts +18 -0
  81. package/dist/types/src/errors/limitation-errors/too-many-error.d.ts +33 -0
  82. package/dist/types/src/errors/limitation-errors/too-many-regexp-rules-error.d.ts +21 -0
  83. package/dist/types/src/errors/limitation-errors/too-many-rules-error.d.ts +21 -0
  84. package/dist/types/src/errors/limitation-errors/too-many-unsafe-rules-error.d.ts +21 -0
  85. package/dist/types/src/errors/unavailable-sources-errors/unavailable-filter-source-error.d.ts +19 -0
  86. package/dist/types/src/errors/unavailable-sources-errors/unavailable-ruleset-source-error.d.ts +19 -0
  87. package/dist/types/src/filter/filter.d.ts +74 -0
  88. package/dist/types/src/filter/types.d.ts +45 -0
  89. package/dist/types/src/filter-converter/conversion-result.d.ts +35 -0
  90. package/dist/types/src/filter-converter/filter-converter-options.d.ts +81 -0
  91. package/dist/types/src/filter-converter/filter-converter.d.ts +197 -0
  92. package/dist/types/src/index.d.ts +37 -0
  93. package/dist/types/src/rule/rule-types.d.ts +33 -0
  94. package/dist/types/src/rule/rule-validator.d.ts +185 -0
  95. package/dist/types/src/rule/rule.d.ts +269 -0
  96. package/dist/types/src/ruleset/metadata-ruleset.d.ts +128 -0
  97. package/dist/types/src/ruleset/rules-hash-map.d.ts +70 -0
  98. package/dist/types/src/ruleset/ruleset-with-source-map.d.ts +318 -0
  99. package/dist/types/src/ruleset/ruleset.d.ts +86 -0
  100. package/dist/types/src/ruleset/source-map.d.ts +45 -0
  101. package/dist/types/src/ruleset/types.d.ts +68 -0
  102. package/dist/types/src/utils/is-safe-rule.d.ts +20 -0
  103. package/dist/types/src/utils/ruleset-utils.d.ts +45 -0
  104. package/dist/types/src/version.d.ts +6 -0
  105. package/dist/url-transform-converter.js +338 -0
  106. package/dist/utils/error.js +22 -0
  107. package/dist/utils/is-safe-rule.js +25 -0
  108. package/dist/utils/lazy-loader.js +116 -0
  109. package/dist/utils/lazy.js +50 -0
  110. package/dist/utils/ruleset-utils.js +76 -0
  111. package/dist/utils/string.js +180 -0
  112. package/dist/utils/valibot.js +74 -0
  113. package/dist/version.js +7 -0
  114. package/package.json +122 -0
@@ -0,0 +1,97 @@
1
+ import { RegularRuleConverter } from './regular-rule-converter.js';
2
+
3
+ /**
4
+ * Describes how to convert `$removeheader` rules.
5
+ *
6
+ * Incompatible-modifier handling is performed by `RuleDeclarativeValidator`
7
+ * (`checkRemoveHeaderCompatibleModifiersFn`): a `$removeheader` rule combined
8
+ * with an incompatible modifier is rejected with an `UnsupportedModifierError`
9
+ * and skipped — no declarative rule is emitted — following the package's
10
+ * "report limitation + skip" convention.
11
+ *
12
+ * Incompatible modifiers are detected in two ways, because dnr-converter
13
+ * stores modifiers differently from tsurlfilter:
14
+ * - loop-reachable modifiers present in `enabledModifiers` and absent from
15
+ * `REMOVEHEADER_COMPATIBLE_MODIFIERS` (e.g. `$method`);
16
+ * - the `$to` modifier, which is field-only but incompatible per tsurlfilter
17
+ * (checked explicitly on `permittedToDomains`/`restrictedToDomains`).
18
+ *
19
+ * `$domain` and `$denyallow` are compatible (they set no flag in tsurlfilter)
20
+ * and convert normally, so they are not rejected.
21
+ *
22
+ * @see {@link RegularRuleConverter} parent class.
23
+ */ class RemoveHeaderConverter extends RegularRuleConverter {
24
+ /**
25
+ * Creates rule template for grouping similar `$removeheader` rules.
26
+ *
27
+ * @param rule {@link DeclarativeRule} to create template from.
28
+ *
29
+ * @returns Stringified rule template.
30
+ */ static createRuleTemplate(rule) {
31
+ /**
32
+ * Deep copy to drop references to source rule.
33
+ *
34
+ * Note: `Partial` type is used because we need to delete some fields,
35
+ * but we cannot mark them as optional in the parent type.
36
+ */ const template = JSON.parse(JSON.stringify(rule));
37
+ /**
38
+ * Remove ID field from the template as it is unique
39
+ * per rule and should not be used for grouping.
40
+ * Also remove request and response headers as they
41
+ * may differ between rules but still should be grouped together.
42
+ */ delete template.id;
43
+ delete template.action?.requestHeaders;
44
+ delete template.action?.responseHeaders;
45
+ return JSON.stringify(template);
46
+ }
47
+ /**
48
+ * Combines two similar `$removeheader` {@link DeclarativeRule}
49
+ * rules into one by merging their response and request header values.
50
+ *
51
+ * @param sourceRule The source rule to merge into.
52
+ * @param ruleToMerge The rule to merge into the source rule.
53
+ *
54
+ * @returns The combined {@link DeclarativeRule}.
55
+ */ static combineRulePair(sourceRule, ruleToMerge) {
56
+ // Deep copy to drop references to source rule
57
+ const resultRule = JSON.parse(JSON.stringify(sourceRule));
58
+ /**
59
+ * Merge response and request headers from {@link ruleToMerge} into {@link resultRule}:
60
+ * - if headers exist in {@link resultRule} - append headers from {@link ruleToMerge};
61
+ * - if headers do not exist in {@link resultRule} - set headers from {@link ruleToMerge}.
62
+ */ let { responseHeaders: responseHeadersToMerge, requestHeaders: requestHeadersToMerge } = ruleToMerge.action;
63
+ const { responseHeaders: resultResponseHeaders, requestHeaders: resultRequestHeaders } = resultRule.action;
64
+ if (responseHeadersToMerge) {
65
+ // Deep copy to avoid reference issues
66
+ responseHeadersToMerge = JSON.parse(JSON.stringify(responseHeadersToMerge));
67
+ if (resultResponseHeaders) {
68
+ resultResponseHeaders.push(...responseHeadersToMerge);
69
+ } else {
70
+ resultRule.action.responseHeaders = responseHeadersToMerge;
71
+ }
72
+ }
73
+ if (requestHeadersToMerge) {
74
+ // Deep copy to avoid reference issues
75
+ requestHeadersToMerge = JSON.parse(JSON.stringify(requestHeadersToMerge));
76
+ if (resultRequestHeaders) {
77
+ resultRequestHeaders.push(...requestHeadersToMerge);
78
+ } else {
79
+ resultRule.action.requestHeaders = requestHeadersToMerge;
80
+ }
81
+ }
82
+ return resultRule;
83
+ }
84
+ /**
85
+ * Groups converted `$removeheader` rules by merging rules with identical
86
+ * conditions.
87
+ *
88
+ * @param converted Converted rules before grouping.
89
+ *
90
+ * @returns Converted rules after grouping.
91
+ */ // eslint-disable-next-line class-methods-use-this -- template method override
92
+ groupConverted(converted) {
93
+ return RegularRuleConverter.groupConvertedRules(converted, RemoveHeaderConverter.createRuleTemplate, RemoveHeaderConverter.combineRulePair);
94
+ }
95
+ }
96
+
97
+ export { RemoveHeaderConverter };
@@ -0,0 +1,365 @@
1
+ import { InvalidDeclarativeRuleError } from '../errors/conversion-errors/invalid-declarative-rule-error.js';
2
+ import { TooManyRegexpRulesError } from '../errors/limitation-errors/too-many-regexp-rules-error.js';
3
+ import { TooManyRulesError } from '../errors/limitation-errors/too-many-rules-error.js';
4
+ import { TooManyUnsafeRulesError } from '../errors/limitation-errors/too-many-unsafe-rules-error.js';
5
+ import { UrlTransformRulesConverter } from '../grouped-rules-converters/url-transform-converter.js';
6
+ import { isSafeRule } from '../utils/is-safe-rule.js';
7
+ import { Lazy } from '../utils/lazy.js';
8
+ import { CspConverter } from './csp-converter.js';
9
+ import { RegularRuleConverter } from './regular-rule-converter.js';
10
+ import { RemoveHeaderConverter } from './remove-header-converter.js';
11
+ import { RulesGroup, RulesGrouper } from './rules-grouper.js';
12
+
13
+ /**
14
+ * Class that converts list of {@link Rule} into list of {@link DeclarativeRule}.
15
+ */ class RulesConverter {
16
+ /**
17
+ * Converts list of {@link ScannedFilter} into {@link ConvertedRules} by:
18
+ * 1. Applying `$badfilter` rules
19
+ * 2. For each group of rules (inside one filter) runs specified converter.
20
+ *
21
+ * TODO: The `$removeparam`, `$removeheader`, `$csp` converters
22
+ * can also combine rules across multiple filters.
23
+ *
24
+ * @see {@link RulesConverter.CONVERTERS}.
25
+ *
26
+ * @param scannedFilters List of {@link ScannedFilter} to convert.
27
+ * @param options Options for conversion.
28
+ *
29
+ * @returns Result object of {@link ConvertedRules}.
30
+ */ static async convert(scannedFilters, options) {
31
+ const filters = RulesConverter.applyBadFilter(scannedFilters);
32
+ let converted = {
33
+ sourceMapValues: [],
34
+ declarativeRules: [],
35
+ errors: []
36
+ };
37
+ /**
38
+ * Set to store unique IDs of declarative rules, it will be modified
39
+ * during the conversion process after each converted rule.
40
+ *
41
+ * Note: since we apply post-converting processing via grouping similar
42
+ * rules for some modifiers, we may have some "released" IDs, but we
43
+ * suppose that 2^31-1 is enough for all rules even with such not used
44
+ * IDs, so to keep the code simple we don't delete them from the set
45
+ * after conversion.
46
+ */ const uniqueIds = new Set();
47
+ for (const [filterId, groupedRules] of filters){
48
+ const { sourceMapValues, declarativeRules, errors } = await RulesConverter.convertRules(filterId, groupedRules, uniqueIds, options);
49
+ converted.sourceMapValues = converted.sourceMapValues.concat(sourceMapValues);
50
+ converted.declarativeRules = converted.declarativeRules.concat(declarativeRules);
51
+ converted.errors = converted.errors.concat(errors);
52
+ }
53
+ converted = RulesConverter.checkLimitations(converted, options?.maxNumberOfRules, options?.maxNumberOfUnsafeRules, options?.maxNumberOfRegexpRules);
54
+ if (!RulesConverter.checkRulesHaveUniqueIds(converted.declarativeRules)) {
55
+ throw new Error('Declarative rules have non-unique identifiers.');
56
+ }
57
+ if (!RulesConverter.checkRulesHaveCorrectIds(converted.declarativeRules)) {
58
+ throw new Error('Declarative rules have incorrect identifiers.');
59
+ }
60
+ return converted;
61
+ }
62
+ /**
63
+ * Converts filter's {@link GroupedRules} into {@link ConvertedRules}.
64
+ *
65
+ * @param filterId The filter ID.
66
+ * @param groupsRules {@link GroupedRules} to convert.
67
+ * @param usedIds Set with already used IDs to exclude duplications in IDs.
68
+ * @param options Options for conversion.
69
+ *
70
+ * @returns Result object of {@link ConvertedRules}.
71
+ */ static async convertRules(filterId, groupsRules, usedIds, options) {
72
+ const converted = {
73
+ sourceMapValues: [],
74
+ declarativeRules: [],
75
+ errors: []
76
+ };
77
+ // Map because RulesGroup values are numbers
78
+ const groups = Object.keys(groupsRules).map(Number);
79
+ const groupResults = await Promise.all(groups.map(async (key)=>{
80
+ if (key === RulesGroup.BadFilter) {
81
+ return null;
82
+ }
83
+ const rules = groupsRules[key];
84
+ if (!rules || rules.length === 0) {
85
+ return null;
86
+ }
87
+ const converter = new RulesConverter.CONVERTERS[key](options?.resourcesPath);
88
+ return converter.convert(filterId, rules, usedIds);
89
+ }));
90
+ for (const result of groupResults){
91
+ if (result === null) {
92
+ continue;
93
+ }
94
+ converted.sourceMapValues = converted.sourceMapValues.concat(result.sourceMapValues);
95
+ converted.declarativeRules = converted.declarativeRules.concat(result.declarativeRules);
96
+ converted.errors = converted.errors.concat(result.errors);
97
+ }
98
+ return converted;
99
+ }
100
+ /**
101
+ * Checks that IDs of a list of {@link DeclarativeRule} fit into the range of 1 to 2^31-1.
102
+ *
103
+ * This check is needed because we have post-converting grouping rules,
104
+ * where some code could easily change any part of an already converted DNR
105
+ * rule, and we would receive a critical error.
106
+ * That's why we added post-processing checks.
107
+ *
108
+ * @see {@link https://groups.google.com/a/chromium.org/g/chromium-extensions/c/yVb56u5Vf0s?}
109
+ *
110
+ * @param rules List of {@link DeclarativeRule}.
111
+ *
112
+ * @returns `true` if every rule ID fit in allowed range, otherwise `false`.
113
+ */ static checkRulesHaveCorrectIds(rules) {
114
+ return rules.every(({ id })=>id >= RulesConverter.MIN_DECLARATIVE_RULE_ID && id <= RulesConverter.MAX_DECLARATIVE_RULE_ID);
115
+ }
116
+ /**
117
+ * Checks that list of {@link DeclarativeRule} have unique IDs.
118
+ *
119
+ * This check is needed because we have post-converting grouping rules,
120
+ * where some code could easily change any part of an already converted DNR
121
+ * rule, and we would receive a critical error.
122
+ * That's why we added post-processing checks.
123
+ *
124
+ * @param rules List of {@link DeclarativeRule}.
125
+ *
126
+ * @returns `true` if every rule have unique ID, otherwise `false`.
127
+ */ static checkRulesHaveUniqueIds(rules) {
128
+ const uniqueIds = new Set();
129
+ for (const { id } of rules){
130
+ if (uniqueIds.has(id)) {
131
+ return false;
132
+ }
133
+ uniqueIds.add(id);
134
+ }
135
+ return true;
136
+ }
137
+ /**
138
+ * Checks whether the {@link DeclarativeRule} is regex.
139
+ *
140
+ * @see {@link https://developer.chrome.com/docs/extensions/reference/api/declarativeNetRequest#property-RuleCondition-regexFilter}
141
+ *
142
+ * @param rule {@link DeclarativeRule} to check.
143
+ *
144
+ * @returns `true` if the `rule` is regex, otherwise `false`.
145
+ */ static isRegexRule(rule) {
146
+ return rule.condition.regexFilter !== undefined;
147
+ }
148
+ /**
149
+ * Groups items by their declarative rule id, preserving the input order
150
+ * within each group and the key first-appearance order across groups.
151
+ *
152
+ * @param items Flat list of items to group.
153
+ * @param idAccessor Function that extracts the declarative rule id from an
154
+ * item.
155
+ *
156
+ * @returns Map from declarative rule id to the list of items for that id.
157
+ */ static groupByDeclarativeRuleId(items, idAccessor) {
158
+ const index = new Map();
159
+ items.forEach((item)=>{
160
+ const id = idAccessor(item);
161
+ const list = index.get(id);
162
+ const newValue = list ? list.concat(item) : [
163
+ item
164
+ ];
165
+ index.set(id, newValue);
166
+ });
167
+ return index;
168
+ }
169
+ /**
170
+ * Removes sources and errors associated with a truncated rule.
171
+ *
172
+ * @param ruleId The ID of the truncated rule.
173
+ * @param sourcesIndex The index of sources.
174
+ * @param errorsIndex The index of errors.
175
+ * @param excludedRulesIds The list of excluded rule IDs.
176
+ */ static removeTruncatedRuleSourcesAndErrors(ruleId, sourcesIndex, errorsIndex, excludedRulesIds) {
177
+ // Removing a source for a truncated rule
178
+ const sources = sourcesIndex.get(ruleId) || [];
179
+ const sourcesRulesIds = sources.map(({ sourceRuleIndex })=>sourceRuleIndex);
180
+ sourcesIndex.set(ruleId, []);
181
+ // Removing an error for a truncated rule
182
+ errorsIndex.set(ruleId, []);
183
+ // Note: be sure, that sourceRulesIds are not too much to overflow stack.
184
+ excludedRulesIds.push(...sourcesRulesIds);
185
+ }
186
+ /**
187
+ * Check that {@link ConvertedRules} matches the specified constraints and
188
+ * cuts rules if needed as from list also from source map.
189
+ *
190
+ * @param converted {@link ConvertedRules} to check.
191
+ * @param maxNumberOfRules Maximum number of converted rules.
192
+ * @param maxNumberOfUnsafeRules Maximum number of converted unsafe rules.
193
+ * @param maxNumberOfRegexpRules Maximum number of converted regexp rules.
194
+ *
195
+ * @returns Transformed converted rules with modified (if abbreviated)
196
+ * counters, declarative rules list, source map and errors.
197
+ */ static checkLimitations(converted, maxNumberOfRules, maxNumberOfUnsafeRules, maxNumberOfRegexpRules) {
198
+ const limitations = [];
199
+ // We apply restrictions only to transformed rules, so we need to filter
200
+ // rule conversion errors if we remove the transformed rule associated
201
+ // with those errors
202
+ let { declarativeRules, sourceMapValues } = converted;
203
+ const convertedRulesErrors = [];
204
+ const otherErrors = [];
205
+ for(let i = 0; i < converted.errors.length; i += 1){
206
+ const e = converted.errors[i];
207
+ // Checks only errors of converted declarative rules
208
+ if (e instanceof InvalidDeclarativeRuleError) {
209
+ convertedRulesErrors.push(e);
210
+ } else {
211
+ otherErrors.push(e);
212
+ }
213
+ }
214
+ // Indexes are built lazily on the first truncation that needs them.
215
+ // In the common case (no limit exceeded) they are never constructed.
216
+ const sourcesIndexLoader = new Lazy(()=>RulesConverter.groupByDeclarativeRuleId(sourceMapValues, (s)=>s.declarativeRuleId));
217
+ const errorsIndexLoader = new Lazy(()=>RulesConverter.groupByDeclarativeRuleId(convertedRulesErrors, (e)=>e.declarativeRule.id));
218
+ // Checks and, if necessary, trims the maximum number of rules
219
+ if (maxNumberOfRules && declarativeRules.length > 0) {
220
+ const filteredRules = [];
221
+ const excludedRulesIds = [];
222
+ let unsafeRulesCounter = 0;
223
+ for(let i = 0; i < declarativeRules.length; i += 1){
224
+ const rule = declarativeRules[i];
225
+ if (maxNumberOfUnsafeRules && !isSafeRule(rule)) {
226
+ unsafeRulesCounter += 1;
227
+ if (unsafeRulesCounter > maxNumberOfUnsafeRules) {
228
+ RulesConverter.removeTruncatedRuleSourcesAndErrors(rule.id, sourcesIndexLoader.get(), errorsIndexLoader.get(), excludedRulesIds);
229
+ continue;
230
+ }
231
+ }
232
+ if (i < maxNumberOfRules) {
233
+ filteredRules.push(rule);
234
+ continue;
235
+ }
236
+ RulesConverter.removeTruncatedRuleSourcesAndErrors(rule.id, sourcesIndexLoader.get(), errorsIndexLoader.get(), excludedRulesIds);
237
+ }
238
+ if (maxNumberOfUnsafeRules && unsafeRulesCounter > maxNumberOfUnsafeRules) {
239
+ const err = new TooManyUnsafeRulesError(// eslint-disable-next-line max-len
240
+ `After conversion, too many unsafe rules remain: ${unsafeRulesCounter} exceeds the limit provided - ${maxNumberOfUnsafeRules}`, excludedRulesIds, maxNumberOfUnsafeRules, unsafeRulesCounter - maxNumberOfUnsafeRules);
241
+ limitations.push(err);
242
+ }
243
+ if (declarativeRules.length > maxNumberOfRules) {
244
+ const err = new TooManyRulesError(// eslint-disable-next-line max-len
245
+ `After conversion, too many declarative rules remain: ${declarativeRules.length} exceeds the limit provided - ${maxNumberOfRules}`, excludedRulesIds, maxNumberOfRules, declarativeRules.length - maxNumberOfRules);
246
+ limitations.push(err);
247
+ }
248
+ declarativeRules = filteredRules;
249
+ }
250
+ // Checks and, if necessary, trims the maximum number of regexp rules
251
+ if (maxNumberOfRegexpRules) {
252
+ const filteredRules = [];
253
+ const excludedRulesIds = [];
254
+ let regexpRulesCounter = 0;
255
+ for(let i = 0; i < declarativeRules.length; i += 1){
256
+ const rule = declarativeRules[i];
257
+ if (RulesConverter.isRegexRule(rule)) {
258
+ regexpRulesCounter += 1;
259
+ if (regexpRulesCounter > maxNumberOfRegexpRules) {
260
+ RulesConverter.removeTruncatedRuleSourcesAndErrors(rule.id, sourcesIndexLoader.get(), errorsIndexLoader.get(), excludedRulesIds);
261
+ continue;
262
+ }
263
+ }
264
+ filteredRules.push(rule);
265
+ }
266
+ if (regexpRulesCounter > maxNumberOfRegexpRules) {
267
+ const err = new TooManyRegexpRulesError(// eslint-disable-next-line max-len
268
+ `After conversion, too many regexp rules remain: ${regexpRulesCounter} exceeds the limit provided - ${maxNumberOfRegexpRules}`, excludedRulesIds, maxNumberOfRegexpRules, regexpRulesCounter - maxNumberOfRegexpRules);
269
+ limitations.push(err);
270
+ }
271
+ declarativeRules = filteredRules;
272
+ }
273
+ // Filter truncated rules' sources/errors in original order to keep the
274
+ // serialized source map byte-for-byte identical regardless of
275
+ // truncation. Truncated rules are the index groups emptied to `[]`
276
+ // (every declarative rule has ≥1 source, so an empty group only results
277
+ // from truncation).
278
+ if (sourcesIndexLoader.isLoaded()) {
279
+ const sourcesIndex = sourcesIndexLoader.get();
280
+ sourceMapValues = sourceMapValues.filter((source)=>{
281
+ const kept = sourcesIndex.get(source.declarativeRuleId);
282
+ return kept !== undefined && kept.length > 0;
283
+ });
284
+ }
285
+ const convertedErrors = errorsIndexLoader.isLoaded() ? convertedRulesErrors.filter((e)=>{
286
+ const kept = errorsIndexLoader.get().get(e.declarativeRule.id);
287
+ return kept !== undefined && kept.length > 0;
288
+ }) : convertedRulesErrors;
289
+ const finalErrors = convertedErrors.concat(otherErrors);
290
+ return {
291
+ sourceMapValues,
292
+ declarativeRules,
293
+ errors: finalErrors,
294
+ limitations
295
+ };
296
+ }
297
+ /**
298
+ * Filters rules that have been affected by `$badfilter` rules and groups them by modifiers.
299
+ *
300
+ * @param scannedFilters List of {@link ScannedFilter}.
301
+ *
302
+ * @returns Result tuple of {@link FiltersIdsWithGroupedRules}.
303
+ */ static applyBadFilter(scannedFilters) {
304
+ let allBadFilterRules = [];
305
+ // Group rules
306
+ const filterIdsWithGroupedRules = scannedFilters.map(({ id, rules })=>{
307
+ const rulesToProcess = RulesGrouper.groupRules(rules);
308
+ allBadFilterRules = allBadFilterRules.concat(rulesToProcess[RulesGroup.BadFilter]);
309
+ const tuple = [
310
+ id,
311
+ rulesToProcess
312
+ ];
313
+ return tuple;
314
+ });
315
+ // Define filter function
316
+ const filterByBadFilterFn = (ruleToTest)=>{
317
+ for (const rule of allBadFilterRules){
318
+ if (rule.negatesBadfilter(ruleToTest)) {
319
+ return false;
320
+ }
321
+ }
322
+ return true;
323
+ };
324
+ // For each group of filters' rules apply filter function
325
+ return filterIdsWithGroupedRules.map(([filterId, groupedRules])=>{
326
+ const filtered = groupedRules;
327
+ // Map because RulesGroup values are numbers
328
+ const groups = Object.keys(filtered).map(Number);
329
+ groups.forEach((key)=>{
330
+ filtered[key] = filtered[key].filter(filterByBadFilterFn);
331
+ });
332
+ // Clean up bad filters rules — they are not converted
333
+ // to declarative rules, only used for negation above
334
+ filtered[RulesGroup.BadFilter] = [];
335
+ return [
336
+ filterId,
337
+ filtered
338
+ ];
339
+ });
340
+ }
341
+ }
342
+ /**
343
+ * Minimum allowed ID for a declarative rule.
344
+ *
345
+ * Note 1: The declarative ID of a rule must be a natural number.
346
+ * Note 2: `1` is reserved for the metadata rule.
347
+ */ RulesConverter.MIN_DECLARATIVE_RULE_ID = 2;
348
+ /**
349
+ * Maximum allowed ID for a declarative rule.
350
+ *
351
+ * Note: The declarative identifier of a rule must be less than signed 32-bit
352
+ * integer. The maximum value of a signed 32-bit integer is 2^31 - 1.
353
+ *
354
+ * @see {@link https://groups.google.com/a/chromium.org/g/chromium-extensions/c/yVb56u5Vf0s?}
355
+ */ RulesConverter.MAX_DECLARATIVE_RULE_ID = 2 ** 31 - 1;
356
+ /**
357
+ * Map of converters for each rules group.
358
+ */ RulesConverter.CONVERTERS = {
359
+ [RulesGroup.Regular]: RegularRuleConverter,
360
+ [RulesGroup.Csp]: CspConverter,
361
+ [RulesGroup.RemoveHeader]: RemoveHeaderConverter,
362
+ [RulesGroup.UrlTransform]: UrlTransformRulesConverter
363
+ };
364
+
365
+ export { RulesConverter };
@@ -0,0 +1,75 @@
1
+ import { OPTION_NAMES } from '../rule/option-names.js';
2
+
3
+ /**
4
+ * Rule groups for declarative rules.
5
+ */ var RulesGroup = /*#__PURE__*/ function(RulesGroup) {
6
+ /**
7
+ * Regular rules (also handles `$removeparam` — no longer merged,
8
+ * each rule produces its own DNR rule with a param-aware urlFilter
9
+ * for multi-hop redirect chaining).
10
+ */ RulesGroup[RulesGroup["Regular"] = 0] = "Regular";
11
+ /**
12
+ * `$removeheader` rules.
13
+ */ RulesGroup[RulesGroup["RemoveHeader"] = 1] = "RemoveHeader";
14
+ /**
15
+ * `$csp` rules.
16
+ */ RulesGroup[RulesGroup["Csp"] = 2] = "Csp";
17
+ /**
18
+ * `$badfilter` rules.
19
+ *
20
+ * These rules are not converted to declarative rules, but are used
21
+ * to negate other rules during the conversion process.
22
+ */ RulesGroup[RulesGroup["BadFilter"] = 3] = "BadFilter";
23
+ /**
24
+ * `$urltransform` rules.
25
+ */ RulesGroup[RulesGroup["UrlTransform"] = 4] = "UrlTransform";
26
+ return RulesGroup;
27
+ }({});
28
+ /**
29
+ * Utility class to group list of {@link Rule} into {@link GroupedRules}.
30
+ */ class RulesGrouper {
31
+ /**
32
+ * Returns group for provided `rule`.
33
+ *
34
+ * @param rule {@link Rule} to get group for.
35
+ *
36
+ * @returns Rule group ({@link RulesGroup}).
37
+ */ static getRuleGroup(rule) {
38
+ if (rule.isModifierEnabled(OPTION_NAMES.REMOVEHEADER)) {
39
+ return 1;
40
+ }
41
+ if (rule.isModifierEnabled(OPTION_NAMES.CSP)) {
42
+ return 2;
43
+ }
44
+ if (rule.isModifierEnabled(OPTION_NAMES.BADFILTER)) {
45
+ return 3;
46
+ }
47
+ if (rule.isModifierEnabled(OPTION_NAMES.URLTRANSFORM)) {
48
+ return 4;
49
+ }
50
+ return 0;
51
+ }
52
+ /**
53
+ * Groups the list of {@link Rule} into {@link GroupedRules}.
54
+ *
55
+ * @param rules List of {@link Rule} to group.
56
+ *
57
+ * @returns Grouped result of {@link GroupedRules}.
58
+ */ static groupRules(rules) {
59
+ const groupedRules = {
60
+ [1]: [],
61
+ [3]: [],
62
+ [0]: [],
63
+ [2]: [],
64
+ [4]: []
65
+ };
66
+ for(let i = 0; i < rules.length; i += 1){
67
+ const rule = rules[i];
68
+ const group = RulesGrouper.getRuleGroup(rule);
69
+ groupedRules[group].push(rule);
70
+ }
71
+ return groupedRules;
72
+ }
73
+ }
74
+
75
+ export { RulesGroup, RulesGrouper };
@@ -0,0 +1,136 @@
1
+ import { RuleCategory } from '@adguard/agtree';
2
+ import { FilterListParser } from '@adguard/agtree/parser';
3
+ import { MaxScannedRulesError } from '../errors/limitation-errors/max-scanned-rules-error.js';
4
+ import { OPTION_NAMES } from '../rule/option-names.js';
5
+ import { Rule } from '../rule/rule.js';
6
+
7
+ /**
8
+ * Class that responsible for scanning filter rules and converting them into {@link Rule}.
9
+ */ class RulesScanner {
10
+ /**
11
+ * Scans the list of filters for network rules.
12
+ *
13
+ * @param filters List of {@link IFilter}.
14
+ * @param filterFn If this function is specified, it will be applied to each
15
+ * rule after it has been parsed and transformed. This function is needed
16
+ * for example to apply `$badfilter`: to exclude negated rules from the array
17
+ * of rules that will be returned.
18
+ * @param maxScannedRules Maximum number of network rules to
19
+ * scan, all other rules will be ignored. It will be applied to each filter
20
+ * separately, not for cumulative scope of rules from all filters, because
21
+ * it looks simpler and more predictable solution to prevent too long scan.
22
+ *
23
+ * @returns Result object of {@link ScannedFiltersWithErrors}.
24
+ */ static async scanFilters(filters, filterFn, maxScannedRules) {
25
+ const result = {
26
+ errors: [],
27
+ filters: []
28
+ };
29
+ for(let i = 0; i < filters.length; i += 1){
30
+ const filter = filters[i];
31
+ // eslint-disable-next-line no-await-in-loop
32
+ const { errors, rules } = await RulesScanner.scanRules(filter, filterFn, maxScannedRules);
33
+ const badFilterRules = rules.filter(RulesScanner.isBadFilterRule);
34
+ result.errors = result.errors.concat(errors);
35
+ result.filters.push({
36
+ id: filter.getId(),
37
+ rules,
38
+ badFilterRules
39
+ });
40
+ }
41
+ return result;
42
+ }
43
+ /**
44
+ * Extracts only the network rules (ignore cosmetic and host rules)
45
+ * and tries to convert each line into {@link Rule}.
46
+ *
47
+ * @param filter From which filter the rules should be scanned.
48
+ * @param filterFn If this function is specified, it will be applied to each
49
+ * rule after it has been parsed and transformed. This function is needed
50
+ * for example to apply `$badfilter`: to exclude negated rules from the array
51
+ * of rules that will be returned.
52
+ * @param maxScannedRules Maximum number of network rules to
53
+ * scan, all other rules will be ignored and an error {@link MaxScannedRulesError}
54
+ * will be added to the list of result errors.
55
+ *
56
+ * @returns Result object of {@link ScannedRulesWithErrors}.
57
+ */ static async scanRules(filter, filterFn, maxScannedRules) {
58
+ const id = filter.getId();
59
+ const content = await filter.getContent();
60
+ // Parse filter content into AST
61
+ const ast = FilterListParser.parse(content, RulesScanner.PARSER_OPTIONS);
62
+ // Build result object
63
+ let scannedRulesCount = 0;
64
+ const result = {
65
+ errors: [],
66
+ rules: []
67
+ };
68
+ for(let i = 0; i < ast.children.length; i += 1){
69
+ /**
70
+ * We use `!` because location info and raw rule is always included in our parser options.
71
+ *
72
+ * @see {@link RulesScanner.PARSER_OPTIONS}
73
+ */ const node = ast.children[i];
74
+ const index = node.start;
75
+ const raw = node.raws.text;
76
+ if (node.category === RuleCategory.Invalid) {
77
+ const { name, message } = node.error;
78
+ const msg = `[${name}] ${message}: filter id - ${id}, line index - ${index}, line - ${raw}`;
79
+ result.errors.push(new Error(msg));
80
+ continue;
81
+ }
82
+ try {
83
+ const rules = Rule.parseFromNode(id, index, node);
84
+ const filteredRules = filterFn ? rules.filter(filterFn) : rules;
85
+ result.rules.push(...filteredRules);
86
+ scannedRulesCount += filteredRules.length;
87
+ if (maxScannedRules !== undefined && scannedRulesCount >= maxScannedRules) {
88
+ const lastRuleLineIndex = rules[rules.length - 1].index;
89
+ // This error needed for future improvements, for example
90
+ // to show in the UI which rules were skipped.
91
+ const msg = `Maximum number of scanned network rules reached at line index ${lastRuleLineIndex}.`;
92
+ result.errors.push(new MaxScannedRulesError(msg, lastRuleLineIndex));
93
+ break;
94
+ }
95
+ } catch (e) {
96
+ if (e instanceof Error) {
97
+ result.errors.push(e);
98
+ } else {
99
+ // eslint-disable-next-line max-len
100
+ const msg = `Unknown error during creating network rule from raw string: filter id - ${id}, line index - ${index}, line - ${raw}`;
101
+ result.errors.push(new Error(msg));
102
+ }
103
+ continue;
104
+ }
105
+ }
106
+ return result;
107
+ }
108
+ /**
109
+ * Checks whether the given rule is a bad filter rule.
110
+ *
111
+ * @param rule {@link Rule} to check.
112
+ *
113
+ * @returns `true` if the rule is a bad filter rule, `false` otherwise.
114
+ */ static isBadFilterRule(rule) {
115
+ return rule.isModifierEnabled(OPTION_NAMES.BADFILTER);
116
+ }
117
+ }
118
+ /**
119
+ * Parser options for filter scanning.
120
+ */ RulesScanner.PARSER_OPTIONS = {
121
+ // We don't want parser to throw errors, so we can collect them all in the result object
122
+ tolerant: true,
123
+ // Location info is needed for source mapping
124
+ isLocIncluded: true,
125
+ // All syntaxes (abp, ubo) should be parsed
126
+ parseAbpSpecificRules: true,
127
+ parseUboSpecificRules: true,
128
+ // Raw text is needed for error reporting
129
+ includeRaws: true,
130
+ // We don't need to process comments
131
+ ignoreComments: true,
132
+ // We only need to process network rules
133
+ parseHostRules: false
134
+ };
135
+
136
+ export { RulesScanner };