@adguard/dnr-converter 2.0.0-beta.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/README.md +62 -3
  2. package/dist/cli.cjs +51902 -47539
  3. package/dist/constants.js +4 -1
  4. package/dist/errors/conversion-errors/index.js +3 -2
  5. package/dist/errors/conversion-errors/unexpected-conversion-error.js +23 -0
  6. package/dist/errors/scan-errors.js +39 -0
  7. package/dist/errors/unavailable-sources-errors/unavailable-filter-source-error.js +10 -1
  8. package/dist/filter-converter/filter-converter.js +151 -6
  9. package/dist/index.js +5 -1
  10. package/dist/package.json.js +1 -1
  11. package/dist/rule/rule-badfilter.js +144 -55
  12. package/dist/rule/rule-validator.js +56 -6
  13. package/dist/rule/rule.js +95 -9
  14. package/dist/rule-converters/action-kind.js +121 -0
  15. package/dist/rule-converters/csp-condition-coverage.js +268 -0
  16. package/dist/rule-converters/csp-converter.js +132 -11
  17. package/dist/rule-converters/csp-rules-resolver.js +417 -0
  18. package/dist/rule-converters/regular-rule-converter.js +479 -129
  19. package/dist/rule-converters/rules-converter.js +75 -7
  20. package/dist/rule-converters/rules-grouper.js +5 -3
  21. package/dist/rules-scanner/rules-scanner.js +124 -56
  22. package/dist/ruleset/ruleset-with-source-map.js +5 -1
  23. package/dist/types/src/errors/conversion-errors/index.d.ts +7 -2
  24. package/dist/types/src/errors/conversion-errors/unexpected-conversion-error.d.ts +21 -0
  25. package/dist/types/src/errors/scan-errors.d.ts +38 -0
  26. package/dist/types/src/errors/unavailable-sources-errors/unavailable-filter-source-error.d.ts +9 -1
  27. package/dist/types/src/filter-converter/filter-converter-options.d.ts +18 -1
  28. package/dist/types/src/filter-converter/filter-converter.d.ts +83 -6
  29. package/dist/types/src/index.d.ts +7 -3
  30. package/dist/types/src/rule/rule-validator.d.ts +21 -0
  31. package/dist/types/src/rule/rule.d.ts +52 -2
  32. package/dist/types/src/rule-converters/csp-converter.d.ts +12 -0
  33. package/dist/types/src/ruleset/ruleset-with-source-map.d.ts +14 -0
  34. package/dist/utils/string.js +27 -1
  35. package/package.json +5 -5
  36. package/dist/rule/string-utils.js +0 -49
@@ -1,4 +1,5 @@
1
1
  import { RuleGenerator } from '@adguard/agtree/generator';
2
+ import { ADBLOCK_URL_SEPARATOR, ADBLOCK_URL_START, RegExpUtils, ADBLOCK_URL_SEPARATOR_REGEX, ADBLOCK_URL_START_REGEX } from '@adguard/agtree/utils';
2
3
  import { getRedirectFilename } from '@adguard/scriptlets/redirects';
3
4
  import { MASK_ANY_CHARACTER, MASK_NEGATE_CHARACTER, MASK_REGEX_RULE, CSP_HEADER_NAME, PERMISSIONS_POLICY_HEADER_NAME, DOT } from '../constants.js';
4
5
  import { POPULAR_TLDS } from '../constants/popular-tlds.js';
@@ -8,14 +9,16 @@ import { re2Validator } from '../re2-regexp/re2-validator.js';
8
9
  import { OPTION_NAMES } from '../rule/option-names.js';
9
10
  import { RuleDeclarativeValidator } from '../rule/rule-validator.js';
10
11
  import { getErrorMessage } from '../utils/error.js';
11
- import { isRegexPattern, toASCII, prepareASCII, removeSlashes } from '../utils/string.js';
12
+ import { encodeNonAscii, prepareASCII, isRegexPattern, toASCII, removeSlashes } from '../utils/string.js';
13
+ import { selectActionKinds, ActionKind } from './action-kind.js';
12
14
  import { UnsupportedModifierError } from '../errors/conversion-errors/unsupported-modifier-error.js';
13
15
  import { HeaderOperation } from '../declarative-rule/modify-header-info.js';
14
16
  import { RuleActionType } from '../declarative-rule/rule-action.js';
15
17
  import { DomainType, ResourceType } from '../declarative-rule/rule-condition.js';
18
+ import { UnsupportedRegexpError } from '../errors/conversion-errors/unsupported-regexp-error.js';
16
19
  import { EmptyResourcesError } from '../errors/conversion-errors/empty-resources-error.js';
17
20
  import { EmptyDomainsError } from '../errors/conversion-errors/empty-domains-error.js';
18
- import { UnsupportedRegexpError } from '../errors/conversion-errors/unsupported-regexp-error.js';
21
+ import { UnexpectedConversionError } from '../errors/conversion-errors/unexpected-conversion-error.js';
19
22
 
20
23
  /**
21
24
  * @typedef {import('../errors/conversion-errors').UnsupportedModifierError} UnsupportedModifierError
@@ -38,6 +41,64 @@ import { UnsupportedRegexpError } from '../errors/conversion-errors/unsupported-
38
41
  */ static isWildcardTldDomain(domain) {
39
42
  return domain.length > RegularRuleConverter.WILDCARD_TLD.length && domain.endsWith(RegularRuleConverter.WILDCARD_TLD);
40
43
  }
44
+ /**
45
+ * Converts a URL filter to an ASCII-only string.
46
+ *
47
+ * The hostname is converted to punycode, while the rest of the pattern is
48
+ * percent-encoded: this is how DNR matches it against a request URL. An
49
+ * explicit port belongs to the rest of the pattern, not to the hostname.
50
+ *
51
+ * The hostname is the leading part of the pattern after an optional domain
52
+ * anchor, scheme and `*`/`*.` wildcard. A leading part without a dot is
53
+ * percent-encoded as a path fragment, unless the pattern is anchored to a
54
+ * domain or a scheme, where the hostname is guaranteed.
55
+ *
56
+ * @param pattern URL filter pattern.
57
+ * @param isCaseSensitive Whether the pattern is matched case-sensitively.
58
+ *
59
+ * @returns URL filter pattern with non-ASCII characters converted.
60
+ */ static prepareUrlFilter(pattern, isCaseSensitive) {
61
+ const domainAnchor = pattern.startsWith('||') ? '||' : '';
62
+ let rest = pattern.substring(domainAnchor.length);
63
+ const schemeMatch = /^\|?[a-z][a-z0-9+.-]*:\/\//i.exec(rest);
64
+ const scheme = schemeMatch ? schemeMatch[0] : '';
65
+ rest = rest.substring(scheme.length);
66
+ const wildcardMatch = /^\*\.?/.exec(rest);
67
+ const wildcard = wildcardMatch ? wildcardMatch[0] : '';
68
+ rest = rest.substring(wildcard.length);
69
+ // A `*` inside an anchored hostname is a wildcard, not a separator, so
70
+ // a pattern like `||foo*.пример.рф^` keeps the whole hostname. An
71
+ // unanchored pattern has no hostname before the `*`: the leading part
72
+ // is a path fragment there.
73
+ const hasAnchor = domainAnchor !== '' || scheme !== '';
74
+ const hostnameEndIndex = rest.search(hasAnchor ? /[\^/|?:]/ : /[*^/|?:]/);
75
+ const hostname = hostnameEndIndex === -1 ? rest : rest.substring(0, hostnameEndIndex);
76
+ const suffix = hostnameEndIndex === -1 ? '' : rest.substring(hostnameEndIndex);
77
+ const isHostname = hasAnchor || hostname.includes('.');
78
+ const preparedHostname = isHostname ? RegularRuleConverter.prepareHostname(hostname, isCaseSensitive) : encodeNonAscii(hostname);
79
+ return `${domainAnchor}${scheme}${wildcard}${preparedHostname}${encodeNonAscii(suffix)}`;
80
+ }
81
+ /**
82
+ * Converts a hostname, possibly containing `*` wildcards, to ASCII.
83
+ *
84
+ * The parts between the wildcards are converted separately: punycode
85
+ * cannot encode a `*`, so converting the whole string would fold it into a
86
+ * label and the rule would never match.
87
+ *
88
+ * The hostname is lowercased, because a host in a URL is always
89
+ * lowercased and punycode keeps the case of non-ASCII letters, so
90
+ * `ПРИМЕР.рф` would otherwise be converted to a label which never matches.
91
+ * The case is kept for a `$match-case` rule, which asks for a
92
+ * case-sensitive comparison.
93
+ *
94
+ * @param hostname Hostname with optional `*` wildcards.
95
+ * @param isCaseSensitive Whether the pattern is matched case-sensitively.
96
+ *
97
+ * @returns ASCII hostname.
98
+ */ static prepareHostname(hostname, isCaseSensitive) {
99
+ const prepared = isCaseSensitive ? hostname : hostname.toLowerCase();
100
+ return prepared.split('*').map((part)=>part === '' ? part : prepareASCII(part)).join('*');
101
+ }
41
102
  /**
42
103
  * Expands wildcard TLD domain into concrete popular TLD domains.
43
104
  *
@@ -48,6 +109,16 @@ import { UnsupportedRegexpError } from '../errors/conversion-errors/unsupported-
48
109
  const domainPrefix = domain.slice(0, -RegularRuleConverter.WILDCARD_TLD.length);
49
110
  return POPULAR_TLDS.map((tld)=>`${domainPrefix}.${tld}`);
50
111
  }
112
+ /**
113
+ * Checks whether a domain from a network rule modifier can be represented
114
+ * in a DNR condition.
115
+ *
116
+ * @param domain Domain to check.
117
+ *
118
+ * @returns `true` if DNR supports the domain.
119
+ */ static isRepresentableDomain(domain) {
120
+ return !isRegexPattern(domain) && (!domain.includes(MASK_ANY_CHARACTER) || RegularRuleConverter.isWildcardTldDomain(domain));
121
+ }
51
122
  /**
52
123
  * Prepares domains for DNR condition arrays.
53
124
  *
@@ -55,27 +126,79 @@ import { UnsupportedRegexpError } from '../errors/conversion-errors/unsupported-
55
126
  * dropped; wildcard TLD domains (e.g. `example.*`) are expanded to the
56
127
  * popular TLD list.
57
128
  *
129
+ * Domains are lowercased before the punycode conversion: punycode keeps the
130
+ * case of non-ASCII letters, so `ПРИМЕР.рф` and `пример.рф` would otherwise
131
+ * produce different `xn--` labels. A host in a URL is always lowercased, so
132
+ * the case written in a modifier carries no meaning, while the condition
133
+ * arrays are also compared with each other by the CSP resolver.
134
+ *
58
135
  * @param domains Domains from network rule modifier.
59
136
  *
60
- * @returns ASCII-only unique domains supported by DNR.
137
+ * @returns Lowercased ASCII-only unique domains supported by DNR.
61
138
  */ static prepareDeclarativeDomains(domains) {
62
139
  const expandedDomains = domains?.flatMap((domain)=>{
63
- if (isRegexPattern(domain)) {
140
+ if (!RegularRuleConverter.isRepresentableDomain(domain)) {
64
141
  return [];
65
142
  }
66
143
  if (RegularRuleConverter.isWildcardTldDomain(domain)) {
67
144
  return RegularRuleConverter.expandWildcardTldDomain(domain);
68
145
  }
69
- if (domain.includes(MASK_ANY_CHARACTER)) {
70
- return [];
71
- }
72
146
  return [
73
147
  domain
74
148
  ];
75
149
  }) ?? [];
76
150
  return [
77
- ...new Set(toASCII(expandedDomains))
151
+ ...new Set(toASCII(expandedDomains.map((domain)=>domain.toLowerCase())))
152
+ ];
153
+ }
154
+ /**
155
+ * Checks whether all domain conditions can be represented in DNR.
156
+ *
157
+ * A port cannot appear in a DNR domain list, so a `$domain` entry written
158
+ * as `example.com:8080` is not a domain which drops a part of the
159
+ * condition: it has no valid declarative form at all.
160
+ *
161
+ * @param rule Source rule.
162
+ *
163
+ * @returns `true` if conversion does not drop any domain condition.
164
+ */ static areDomainConditionsRepresentable(rule) {
165
+ const domainLists = [
166
+ rule.permittedDomains,
167
+ rule.restrictedDomains,
168
+ rule.permittedToDomains,
169
+ rule.restrictedToDomains,
170
+ rule.denyAllowDomains
78
171
  ];
172
+ return domainLists.every((domains)=>domains?.every((domain)=>!isRegexPattern(domain) && !domain.includes(RegularRuleConverter.PORT_SEPARATOR) && (!domain.includes(MASK_ANY_CHARACTER) || RegularRuleConverter.isWildcardTldDomain(domain))) ?? true);
173
+ }
174
+ /**
175
+ * Checks whether a condition keeps every restriction of the rule.
176
+ *
177
+ * {@link RegularRuleConverter.getCondition} stores one inclusion list per
178
+ * field, so a rule which combines an inclusion with an exclusion in the
179
+ * same field loses the inclusion: `$csp,script,~image` keeps only `~image`,
180
+ * and `$to=~a.com,denyallow=b.com` keeps only `$denyallow`. A CSP exception
181
+ * would then cancel more than written, so it must not be applied.
182
+ *
183
+ * @param rule Source rule.
184
+ *
185
+ * @returns `true` if the condition keeps all restrictions of the rule.
186
+ */ static areConditionRestrictionsRepresentable(rule) {
187
+ const hasPermittedResourceTypes = rule.permittedResourceTypes.length !== 0;
188
+ const hasRestrictedResourceTypes = rule.restrictedResourceTypes.length !== 0;
189
+ const hasRestrictedToDomains = (rule.restrictedToDomains?.length ?? 0) !== 0;
190
+ const hasDenyAllowDomains = (rule.denyAllowDomains?.length ?? 0) !== 0;
191
+ return !(hasPermittedResourceTypes && hasRestrictedResourceTypes) && !(hasRestrictedToDomains && hasDenyAllowDomains);
192
+ }
193
+ /**
194
+ * Checks whether every domain from a network rule modifier can be
195
+ * represented in a DNR condition.
196
+ *
197
+ * @param domains Domains from network rule modifier.
198
+ *
199
+ * @returns `true` if every domain is supported by DNR.
200
+ */ static areDomainsRepresentable(domains) {
201
+ return domains?.every((domain)=>RegularRuleConverter.isRepresentableDomain(domain)) ?? true;
79
202
  }
80
203
  /**
81
204
  * Checks if {@link Rule} can be converted to {@link RuleActionType.AllowAllRequests}.
@@ -185,6 +308,9 @@ import { UnsupportedRegexpError } from '../errors/conversion-errors/unsupported-
185
308
  * - value cannot be URI-decoded.
186
309
  * - param name contains urlFilter special characters (`|`, `*`, `^`).
187
310
  *
311
+ * Non-ASCII characters in the param name are percent-encoded, because
312
+ * the token is matched against a url where they are encoded as well.
313
+ *
188
314
  * @param rule Rule with $removeparam modifier enabled.
189
315
  *
190
316
  * @returns A urlFilter token string, or null if augmentation should be skipped.
@@ -209,7 +335,8 @@ import { UnsupportedRegexpError } from '../errors/conversion-errors/unsupported-
209
335
  if (RegularRuleConverter.URL_FILTER_SPECIAL_CHARS.test(decoded)) {
210
336
  return null;
211
337
  }
212
- return `^${decoded}=`;
338
+ // The token is matched against the url, where non-ASCII characters are percent-encoded.
339
+ return `^${encodeNonAscii(decoded)}=`;
213
340
  }
214
341
  /**
215
342
  * Returns rule modify headers action.
@@ -281,7 +408,7 @@ import { UnsupportedRegexpError } from '../errors/conversion-errors/unsupported-
281
408
  if (!rule.isModifierEnabled(OPTION_NAMES.CSP)) {
282
409
  return null;
283
410
  }
284
- const cspHeaderValue = rule.advancedModifierValue;
411
+ const cspHeaderValue = rule.cspDirective;
285
412
  if (!cspHeaderValue) {
286
413
  return null;
287
414
  }
@@ -321,93 +448,127 @@ import { UnsupportedRegexpError } from '../errors/conversion-errors/unsupported-
321
448
  * @throws Error {@link ResourcesPathError} when specified an empty path to the web accessible resources.
322
449
  * @throws Error {@link UnsupportedModifierError} when a `$removeparam` value cannot be URI-decoded.
323
450
  */ getAction(rule) {
324
- if (rule.allowlist) {
325
- if (rule.isFilteringDisabled() && RegularRuleConverter.isCompatibleWithAllowAllRequests(rule)) {
326
- return {
327
- type: RuleActionType.AllowAllRequests
328
- };
329
- }
330
- return {
331
- type: RuleActionType.Allow
332
- };
333
- }
334
- if (rule.isModifierEnabled(OPTION_NAMES.REDIRECT)) {
335
- const redirectAction = this.getRedirectAction(rule);
336
- if (redirectAction) {
337
- return {
338
- type: RuleActionType.Redirect,
339
- redirect: redirectAction
340
- };
341
- }
342
- }
343
- if (rule.isModifierEnabled(OPTION_NAMES.REMOVEPARAM)) {
344
- const removeParamRedirectAction = RegularRuleConverter.getRemoveParamRedirectAction(rule);
345
- if (removeParamRedirectAction) {
346
- return {
347
- type: RuleActionType.Redirect,
348
- redirect: removeParamRedirectAction
349
- };
350
- }
351
- }
352
- if (rule.isModifierEnabled(OPTION_NAMES.REMOVEHEADER)) {
353
- const modifyHeadersAction = RegularRuleConverter.getModifyHeadersAction(rule);
354
- if (modifyHeadersAction?.requestHeaders) {
355
- return {
356
- type: RuleActionType.ModifyHeaders,
357
- requestHeaders: modifyHeadersAction.requestHeaders
358
- };
359
- }
360
- if (modifyHeadersAction?.responseHeaders) {
361
- return {
362
- type: RuleActionType.ModifyHeaders,
363
- responseHeaders: modifyHeadersAction.responseHeaders
364
- };
365
- }
366
- }
367
- if (rule.isModifierEnabled(OPTION_NAMES.CSP)) {
368
- const headersAction = RegularRuleConverter.getAddingCspHeadersAction(rule);
369
- if (headersAction) {
370
- return {
371
- type: RuleActionType.ModifyHeaders,
372
- responseHeaders: [
373
- headersAction
374
- ]
375
- };
376
- }
377
- }
378
- if (rule.isModifierEnabled(OPTION_NAMES.PERMISSIONS)) {
379
- const headersAction = RegularRuleConverter.getAddingPermissionsHeadersAction(rule);
380
- if (headersAction) {
381
- return {
382
- type: RuleActionType.ModifyHeaders,
383
- responseHeaders: [
384
- headersAction
385
- ]
386
- };
387
- }
388
- }
389
- if (rule.isModifierEnabled(OPTION_NAMES.COOKIE)) {
390
- const removeCookieHeaders = RegularRuleConverter.getRemovingCookieHeadersAction(rule);
391
- if (removeCookieHeaders) {
392
- const { responseHeaders, requestHeaders } = removeCookieHeaders;
393
- return {
394
- type: RuleActionType.ModifyHeaders,
395
- responseHeaders,
396
- requestHeaders
397
- };
451
+ const kinds = selectActionKinds(rule);
452
+ for(let i = 0; i < kinds.length; i += 1){
453
+ const action = this.buildAction(rule, kinds[i]);
454
+ if (action) {
455
+ return action;
398
456
  }
399
457
  }
400
458
  return {
401
459
  type: RuleActionType.Block
402
460
  };
403
461
  }
462
+ /**
463
+ * Builds the action of the given kind for a rule.
464
+ *
465
+ * @param rule {@link Rule} to build an action for.
466
+ * @param kind Action kind to build.
467
+ *
468
+ * @returns Action or `null` when the modifier cannot produce it, so the
469
+ * next kind in {@link selectActionKinds} is used.
470
+ */ buildAction(rule, kind) {
471
+ switch(kind){
472
+ case ActionKind.Allow:
473
+ {
474
+ if (rule.isFilteringDisabled() && RegularRuleConverter.isCompatibleWithAllowAllRequests(rule)) {
475
+ return {
476
+ type: RuleActionType.AllowAllRequests
477
+ };
478
+ }
479
+ return {
480
+ type: RuleActionType.Allow
481
+ };
482
+ }
483
+ case ActionKind.Redirect:
484
+ {
485
+ const redirectAction = this.getRedirectAction(rule);
486
+ return redirectAction ? {
487
+ type: RuleActionType.Redirect,
488
+ redirect: redirectAction
489
+ } : null;
490
+ }
491
+ case ActionKind.RemoveParamRedirect:
492
+ {
493
+ const removeParamRedirectAction = RegularRuleConverter.getRemoveParamRedirectAction(rule);
494
+ return removeParamRedirectAction ? {
495
+ type: RuleActionType.Redirect,
496
+ redirect: removeParamRedirectAction
497
+ } : null;
498
+ }
499
+ case ActionKind.RemoveHeader:
500
+ {
501
+ const modifyHeadersAction = RegularRuleConverter.getModifyHeadersAction(rule);
502
+ if (modifyHeadersAction?.requestHeaders) {
503
+ return {
504
+ type: RuleActionType.ModifyHeaders,
505
+ requestHeaders: modifyHeadersAction.requestHeaders
506
+ };
507
+ }
508
+ if (modifyHeadersAction?.responseHeaders) {
509
+ return {
510
+ type: RuleActionType.ModifyHeaders,
511
+ responseHeaders: modifyHeadersAction.responseHeaders
512
+ };
513
+ }
514
+ return null;
515
+ }
516
+ case ActionKind.Csp:
517
+ {
518
+ const headersAction = RegularRuleConverter.getAddingCspHeadersAction(rule);
519
+ return headersAction ? {
520
+ type: RuleActionType.ModifyHeaders,
521
+ responseHeaders: [
522
+ headersAction
523
+ ]
524
+ } : null;
525
+ }
526
+ case ActionKind.Permissions:
527
+ {
528
+ const headersAction = RegularRuleConverter.getAddingPermissionsHeadersAction(rule);
529
+ return headersAction ? {
530
+ type: RuleActionType.ModifyHeaders,
531
+ responseHeaders: [
532
+ headersAction
533
+ ]
534
+ } : null;
535
+ }
536
+ case ActionKind.Cookie:
537
+ {
538
+ const removeCookieHeaders = RegularRuleConverter.getRemovingCookieHeadersAction(rule);
539
+ if (!removeCookieHeaders) {
540
+ return null;
541
+ }
542
+ const { responseHeaders, requestHeaders } = removeCookieHeaders;
543
+ return {
544
+ type: RuleActionType.ModifyHeaders,
545
+ responseHeaders,
546
+ requestHeaders
547
+ };
548
+ }
549
+ case ActionKind.Block:
550
+ {
551
+ return {
552
+ type: RuleActionType.Block
553
+ };
554
+ }
555
+ default:
556
+ {
557
+ // A new action kind must be handled explicitly instead of
558
+ // silently becoming a blocking rule.
559
+ const unknownKind = kind;
560
+ throw new Error(`Unhandled action kind: ${unknownKind}`);
561
+ }
562
+ }
563
+ }
404
564
  /**
405
565
  * Retrieves the condition for the provided {@link Rule}.
406
566
  *
407
567
  * @param rule {@link Rule} to get condition for.
568
+ * @param useRemoveParamRegex Whether to use a regexp for eligible named parameter removal.
408
569
  *
409
570
  * @returns A rule condition that describes to which request the declarative rule should be applied.
410
- */ static getCondition(rule) {
571
+ */ static getCondition(rule, useRemoveParamRegex = true) {
411
572
  const condition = {};
412
573
  // set `urlFilter` or `regexFilter` depending on the pattern type
413
574
  const { pattern } = rule;
@@ -417,19 +578,40 @@ import { UnsupportedRegexpError } from '../errors/conversion-errors/unsupported-
417
578
  } else {
418
579
  // A pattern beginning with ||* is not allowed. Use * instead.
419
580
  const patternWithoutVerticals = pattern.startsWith('||*') ? pattern.substring(2) : pattern;
420
- condition.urlFilter = prepareASCII(patternWithoutVerticals);
581
+ condition.urlFilter = RegularRuleConverter.prepareUrlFilter(patternWithoutVerticals, rule.isModifierEnabled(OPTION_NAMES.MATCH_CASE));
421
582
  }
422
583
  }
423
- // For $removeparam rules with a specific named parameter, append
424
- // a param-aware token to urlFilter so that the rule only matches
425
- // when the target parameter is present in the URL query string.
584
+ // For $removeparam rules with a specific named parameter, add
585
+ // a parameter-aware URL condition that stops matching after removal.
426
586
  // This enables Chrome DNR to chain multiple redirect hops,
427
587
  // stripping one parameter per hop until all are removed.
428
588
  if (rule.isModifierEnabled(OPTION_NAMES.REMOVEPARAM)) {
429
589
  const paramToken = RegularRuleConverter.getRemoveParamToken(rule);
430
590
  if (paramToken !== null) {
431
591
  if (condition.urlFilter) {
432
- condition.urlFilter += `*${paramToken}`;
592
+ const prefix = condition.urlFilter.slice(0, -ADBLOCK_URL_SEPARATOR.length);
593
+ const hasDomainAnchor = prefix.startsWith(ADBLOCK_URL_START);
594
+ const authority = hasDomainAnchor ? prefix.slice(ADBLOCK_URL_START.length).split('/')[0] : '';
595
+ if (useRemoveParamRegex && condition.urlFilter.endsWith(ADBLOCK_URL_SEPARATOR) && prefix.startsWith('|') && !/[*?#^]/.test(prefix) && (!hasDomainAnchor || authority.length > 0 && !authority.includes('@'))) {
596
+ // Share the trailing separator with the query delimiter when the parameter is first.
597
+ // Anchoring keeps this delimiter at the actual query boundary.
598
+ // Unanchored patterns can match inside the query and retain native URL-filter matching.
599
+ const patternRegexp = RegExpUtils.patternToRegexp(condition.urlFilter);
600
+ let prefixRegexp = patternRegexp.slice(0, -ADBLOCK_URL_SEPARATOR_REGEX.length);
601
+ if (hasDomainAnchor) {
602
+ // Domain anchors start at the parsed host, after any credentials, on any URL scheme.
603
+ prefixRegexp = prefixRegexp.replace(ADBLOCK_URL_START_REGEX, '^[a-z][a-z0-9+.-]*://(?:[^[:^ascii:]/?#@]*@)?(?:[^[:^ascii:]/?#@]*\\.)?');
604
+ }
605
+ const paramRegexp = RegExpUtils.patternToRegexp(paramToken);
606
+ const paramNameRegexp = paramRegexp.slice(ADBLOCK_URL_SEPARATOR_REGEX.length);
607
+ // DNR matches ASCII-serialized URLs. Bound classes to ASCII to keep RE2 compilation small.
608
+ const pathSeparator = '[^[:^ascii:] a-zA-Z0-9.%_?#-]';
609
+ const pathTail = hasDomainAnchor && !prefix.includes('/') ? '(?:[^[:^ascii:] a-zA-Z0-9.%_/?#@-][^[:^ascii:]/?#@]*)?(?:/[^[:^ascii:]?#]*)?' : `(?:${pathSeparator}[^[:^ascii:]?#]*)?`;
610
+ condition.regexFilter = `${prefixRegexp}${pathTail}` + `\\?(?:[^[:^ascii:]#]*&)?${paramNameRegexp}`;
611
+ delete condition.urlFilter;
612
+ } else {
613
+ condition.urlFilter += `*${paramToken}`;
614
+ }
433
615
  } else if (!condition.regexFilter) {
434
616
  condition.urlFilter = paramToken;
435
617
  }
@@ -595,12 +777,137 @@ import { UnsupportedRegexpError } from '../errors/conversion-errors/unsupported-
595
777
  // Set calculated priority
596
778
  declarativeRule.priority = rule.priority;
597
779
  // Validate created declarative rule and throw error if not valid
598
- const conversionErr = await RegularRuleConverter.checkRuleApplication(rule, declarativeRule);
780
+ let conversionErr = await RegularRuleConverter.checkRuleApplication(rule, declarativeRule);
781
+ if (conversionErr instanceof UnsupportedRegexpError && !rule.isRegexRule() && rule.isModifierEnabled(OPTION_NAMES.REMOVEPARAM)) {
782
+ // Preserve native URL-filter matching when the generated parameter-aware regexp is unsupported.
783
+ declarativeRule.condition = RegularRuleConverter.getCondition(rule, false);
784
+ conversionErr = await RegularRuleConverter.checkRuleApplication(rule, declarativeRule);
785
+ }
599
786
  if (conversionErr) {
600
787
  throw conversionErr;
601
788
  }
602
789
  return declarativeRule;
603
790
  }
791
+ /**
792
+ * Checks whether {@link convertRules} builds the additional `main_frame`
793
+ * rule for a rule, see {@link buildTargetDomainRule}.
794
+ *
795
+ * @param rule Source {@link Rule}.
796
+ *
797
+ * @returns `true` if the additional rule is built.
798
+ */ static hasTargetDomainRule(rule) {
799
+ return RegularRuleConverter.getTargetDomainCondition(rule, {
800
+ id: 0,
801
+ action: {
802
+ type: RuleActionType.Block
803
+ },
804
+ condition: RegularRuleConverter.getCondition(rule)
805
+ }) !== null;
806
+ }
807
+ /**
808
+ * Builds the additional `main_frame` rule when the `$domain` modifier
809
+ * must also match the target hostname of a document request.
810
+ *
811
+ * Chrome matches `initiatorDomains` against the request initiator only,
812
+ * and a document request started by the browser (new tab page, address
813
+ * bar, bookmark, reload) has no initiator. AdGuard matches `$domain`
814
+ * against the target hostname for document requests when the pattern is
815
+ * not domain-specific, or when the `$domain` list contains only excluded
816
+ * domains, so the additional rule keeps the same behavior.
817
+ *
818
+ * @param rule Source {@link Rule}.
819
+ * @param declarativeRule Converted {@link DeclarativeRule}.
820
+ * @param usedIds Set with already used IDs to exclude duplications in IDs.
821
+ *
822
+ * @returns Additional {@link DeclarativeRule}, or `null` when the
823
+ * additional rule is not needed.
824
+ */ static buildTargetDomainRule(rule, declarativeRule, usedIds) {
825
+ const targetDomainCondition = RegularRuleConverter.getTargetDomainCondition(rule, declarativeRule);
826
+ if (!targetDomainCondition) {
827
+ return null;
828
+ }
829
+ return {
830
+ id: RegularRuleConverter.generateId(rule, usedIds),
831
+ action: JSON.parse(JSON.stringify(declarativeRule.action)),
832
+ condition: targetDomainCondition,
833
+ priority: declarativeRule.priority
834
+ };
835
+ }
836
+ /**
837
+ * Builds the condition of the additional `main_frame` rule that matches
838
+ * the `$domain` modifier against the request URL hostname.
839
+ *
840
+ * Note: for a `$domain` list with only excluded domains, the original
841
+ * rule still matches a browser-initiated document request to an excluded
842
+ * domain, because DNR has no condition for a missing initiator. AdGuard
843
+ * does not match that request.
844
+ *
845
+ * @param rule Source {@link Rule}.
846
+ * @param declarativeRule Converted {@link DeclarativeRule}.
847
+ *
848
+ * @returns Condition for the additional rule, or `null` when the
849
+ * additional rule is not needed.
850
+ */ static getTargetDomainCondition(rule, declarativeRule) {
851
+ /**
852
+ * DNR cannot express "first party or no initiator". Chrome treats a
853
+ * request without an initiator as third-party, while MV2 treats it as
854
+ * first-party, so the additional rule would widen a `$third-party`
855
+ * or a `$~third-party` rule. Skip it in this case.
856
+ */ if (declarativeRule.condition.domainType) {
857
+ return null;
858
+ }
859
+ const permittedDomains = declarativeRule.condition.initiatorDomains ?? [];
860
+ const excludedDomains = declarativeRule.condition.excludedInitiatorDomains ?? [];
861
+ if (permittedDomains.length === 0 && excludedDomains.length === 0) {
862
+ return null;
863
+ }
864
+ /**
865
+ * Every domain condition must be fully representable in the
866
+ * additional rule. Otherwise it would drop a constraint that the
867
+ * source rule has and match requests the source rule does not.
868
+ */ if (!RegularRuleConverter.areDomainsRepresentable(rule.permittedDomains) || !RegularRuleConverter.areDomainsRepresentable(rule.restrictedDomains) || !RegularRuleConverter.areDomainsRepresentable(rule.permittedToDomains) || !RegularRuleConverter.areDomainsRepresentable(rule.restrictedToDomains) || !RegularRuleConverter.areDomainsRepresentable(rule.denyAllowDomains)) {
869
+ return null;
870
+ }
871
+ /**
872
+ * A permitted `$to` also sets `requestDomains`, and the additional
873
+ * rule would replace them with the `$domain` domains. `$denyallow`
874
+ * and a negated `$to` only add `excludedRequestDomains`, which the
875
+ * additional rule merges.
876
+ */ if ((rule.permittedToDomains?.length ?? 0) > 0 && permittedDomains.length > 0) {
877
+ return null;
878
+ }
879
+ if (!declarativeRule.condition.resourceTypes?.includes(ResourceType.MainFrame)) {
880
+ return null;
881
+ }
882
+ /**
883
+ * AdGuard matches the target hostname for a document request when
884
+ * the pattern is not domain-specific or when the `$domain` list
885
+ * contains only excluded domains.
886
+ */ const hasOnlyExcludedDomains = permittedDomains.length === 0 && excludedDomains.length > 0;
887
+ if (!hasOnlyExcludedDomains && (rule.isRegexRule() || RegularRuleConverter.DOMAIN_SPECIFIC_PATTERN_PREFIXES.some((prefix)=>rule.pattern.startsWith(prefix)))) {
888
+ return null;
889
+ }
890
+ const condition = {
891
+ ...declarativeRule.condition,
892
+ resourceTypes: [
893
+ ResourceType.MainFrame
894
+ ]
895
+ };
896
+ delete condition.initiatorDomains;
897
+ delete condition.excludedInitiatorDomains;
898
+ if (permittedDomains.length > 0) {
899
+ condition.requestDomains = [
900
+ ...permittedDomains
901
+ ];
902
+ }
903
+ if (excludedDomains.length > 0) {
904
+ condition.excludedRequestDomains = Array.from(new Set([
905
+ ...condition.excludedRequestDomains ?? [],
906
+ ...excludedDomains
907
+ ]));
908
+ }
909
+ return condition;
910
+ }
604
911
  /**
605
912
  * Verifies whether the converted {@link DeclarativeRule}:
606
913
  * - has non-empty resource types,
@@ -651,22 +958,20 @@ import { UnsupportedRegexpError } from '../errors/conversion-errors/unsupported-
651
958
  }
652
959
  /**
653
960
  * Checks the captured conversion error, if it is one of the expected
654
- * conversion errors - returns it, otherwise adds information about
655
- * the original rule, packages it into a new error and returns it.
961
+ * conversion errors - returns it, otherwise packages it into a new error
962
+ * which keeps the original rule and the captured error as a cause.
656
963
  *
657
- * @param index Index of {@link Rule}.
964
+ * @param rule {@link Rule} which could not be converted.
658
965
  * @param id Identifier of the desired declarative rule.
659
966
  * @param error Captured error.
660
967
  *
661
968
  * @returns Initial error or new packaged error.
662
- */ static catchConversionError(index, id, error) {
969
+ */ static catchConversionError(rule, id, error) {
663
970
  if (isConversionError(error)) {
664
971
  return error;
665
972
  }
666
- const message = `Non-categorized error during a conversion rule (index - ${index}, id - ${id})`;
667
- return error instanceof Error ? new Error(message, {
668
- cause: error
669
- }) : new Error(message);
973
+ const message = `Non-categorized error during a conversion rule (index - ${rule.index}, id - ${id})`;
974
+ return new UnexpectedConversionError(message, rule, error);
670
975
  }
671
976
  /**
672
977
  * Converts the provided list of {@link Rule} into {@link DeclarativeRule},
@@ -697,16 +1002,25 @@ import { UnsupportedRegexpError } from '../errors/conversion-errors/unsupported-
697
1002
  // Rule doesn't require conversion (e.g. $elemhide-only rules)
698
1003
  return;
699
1004
  }
700
- // Convert rule and collect source map value
701
- const converted = await this.convertRule(id, rule);
702
- res.sourceMapValues.push({
703
- declarativeRuleId: converted.id,
704
- sourceRuleIndex: index,
705
- filterId: filterListId
1005
+ // Convert rule and collect source map values
1006
+ const declarativeRule = await this.convertRule(id, rule);
1007
+ const convertedRules = [
1008
+ declarativeRule
1009
+ ];
1010
+ const targetDomainRule = RegularRuleConverter.buildTargetDomainRule(rule, declarativeRule, usedIds);
1011
+ if (targetDomainRule) {
1012
+ convertedRules.push(targetDomainRule);
1013
+ }
1014
+ convertedRules.forEach((converted)=>{
1015
+ res.sourceMapValues.push({
1016
+ declarativeRuleId: converted.id,
1017
+ sourceRuleIndex: index,
1018
+ filterId: filterListId
1019
+ });
1020
+ res.declarativeRules.push(converted);
706
1021
  });
707
- res.declarativeRules.push(converted);
708
1022
  } catch (e) {
709
- const err = RegularRuleConverter.catchConversionError(index, id, e);
1023
+ const err = RegularRuleConverter.catchConversionError(rule, id, e);
710
1024
  res.errors.push(err);
711
1025
  }
712
1026
  }));
@@ -724,15 +1038,17 @@ import { UnsupportedRegexpError } from '../errors/conversion-errors/unsupported-
724
1038
  * a declarative rule. This template is used to compare different
725
1039
  * declarative rules.
726
1040
  * @param combineRulePair A function that combines two similar
727
- * declarative rules into one by merging their specific properties.
1041
+ * declarative rules into one by merging their specific properties, or
1042
+ * returns `null` when the two rules cannot be combined and must stay
1043
+ * separate.
728
1044
  *
729
1045
  * @returns Object with grouped similar declarative rules.
730
1046
  */ static groupConvertedRules(converted, createRuleTemplate, combineRulePair) {
731
- const rulesTemplates = new Map();
732
- const saveRuleTemplate = (rule)=>{
733
- const template = createRuleTemplate(rule);
734
- rulesTemplates.set(template, rule);
735
- };
1047
+ /**
1048
+ * Rules of one template, in order. A template describes the matching
1049
+ * condition, so the rules of one bucket may still be incompatible with
1050
+ * each other, e.g. two `$csp` rules with conflicting directives.
1051
+ */ const rulesTemplates = new Map();
736
1052
  const result = {
737
1053
  declarativeRules: [],
738
1054
  sourceMapValues: [],
@@ -742,7 +1058,7 @@ import { UnsupportedRegexpError } from '../errors/conversion-errors/unsupported-
742
1058
  declarativeRules.forEach((declarativeRule)=>{
743
1059
  // Try to find a sibling declarative rule
744
1060
  const template = createRuleTemplate(declarativeRule);
745
- const siblingDeclarativeRule = rulesTemplates.get(template);
1061
+ const siblings = rulesTemplates.get(template) ?? [];
746
1062
  // Try to find the source mapping for the current declarative rule
747
1063
  const source = sourceMapValues.find((s)=>s.declarativeRuleId === declarativeRule.id);
748
1064
  if (source === undefined) {
@@ -750,25 +1066,39 @@ import { UnsupportedRegexpError } from '../errors/conversion-errors/unsupported-
750
1066
  return;
751
1067
  }
752
1068
  /**
753
- * If a similar rule is found, combine the two declarative rules into one
754
- * and save the combined rule's template for future comparisons.
755
- * Also, update the source mapping to reflect the new combined rule ID.
1069
+ * If a sibling can be combined with the current rule, replace it
1070
+ * with the combined rule and update the source mapping to reflect
1071
+ * the combined rule ID.
756
1072
  *
757
- * If no similar rule is found, simply save the current
758
- * rule's template and add the source mapping as is.
759
- */ if (siblingDeclarativeRule) {
760
- const combinedRule = combineRulePair(siblingDeclarativeRule, declarativeRule);
761
- saveRuleTemplate(combinedRule);
762
- result.sourceMapValues.push({
763
- ...source,
764
- declarativeRuleId: combinedRule.id
765
- });
766
- } else {
767
- saveRuleTemplate(declarativeRule);
1073
+ * If no sibling accepts the rule, it is added to the bucket as is:
1074
+ * a rule which cannot be merged must not be merged with a rule it
1075
+ * contradicts.
1076
+ */ let siblingIndex = -1;
1077
+ let combinedRule = null;
1078
+ for(let index = 0; index < siblings.length; index += 1){
1079
+ const candidate = combineRulePair(siblings[index], declarativeRule);
1080
+ if (candidate !== null) {
1081
+ siblingIndex = index;
1082
+ combinedRule = candidate;
1083
+ break;
1084
+ }
1085
+ }
1086
+ if (siblingIndex === -1 || combinedRule === null) {
1087
+ rulesTemplates.set(template, [
1088
+ ...siblings,
1089
+ declarativeRule
1090
+ ]);
768
1091
  result.sourceMapValues.push(source);
1092
+ return;
769
1093
  }
1094
+ const mergedRule = combinedRule;
1095
+ rulesTemplates.set(template, siblings.map((rule, index)=>index === siblingIndex ? mergedRule : rule));
1096
+ result.sourceMapValues.push({
1097
+ ...source,
1098
+ declarativeRuleId: mergedRule.id
1099
+ });
770
1100
  });
771
- result.declarativeRules = Array.from(rulesTemplates.values());
1101
+ result.declarativeRules = Array.from(rulesTemplates.values()).flat();
772
1102
  return result;
773
1103
  }
774
1104
  /**
@@ -846,5 +1176,25 @@ import { UnsupportedRegexpError } from '../errors/conversion-errors/unsupported-
846
1176
  /**
847
1177
  * Wildcard TLD suffix (`.*`) used in domain modifiers, e.g. `example.*`.
848
1178
  */ RegularRuleConverter.WILDCARD_TLD = `${DOT}${MASK_ANY_CHARACTER}`;
1179
+ /**
1180
+ * Separator between a host and a port in a URL.
1181
+ */ RegularRuleConverter.PORT_SEPARATOR = ':';
1182
+ /**
1183
+ * Pattern prefixes that target a specific domain.
1184
+ *
1185
+ * AdGuard matches the `$domain` modifier against the target hostname of
1186
+ * a document request when the pattern is not domain-specific, or when the
1187
+ * `$domain` list contains only excluded domains. This mirrors
1188
+ * `NetworkRule.matchDomainModifier()` and
1189
+ * `Pattern.isPatternDomainSpecific()` in `@adguard/tsurlfilter`.
1190
+ *
1191
+ * @see {@link https://github.com/AdguardTeam/tsurlfilter/blob/master/packages/tsurlfilter/src/rules/network-rule.ts | NetworkRule.matchDomainModifier()}
1192
+ * @see {@link https://github.com/AdguardTeam/tsurlfilter/blob/master/packages/tsurlfilter/src/rules/pattern.ts | Pattern.isPatternDomainSpecific()}
1193
+ */ RegularRuleConverter.DOMAIN_SPECIFIC_PATTERN_PREFIXES = [
1194
+ ADBLOCK_URL_START,
1195
+ 'http://',
1196
+ 'https:/',
1197
+ '://'
1198
+ ];
849
1199
 
850
1200
  export { RegularRuleConverter };