@adguard/hostlist-compiler 2.0.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,36 +1,52 @@
1
+ const _ = require('lodash');
2
+ const consola = require('consola');
3
+ const tldts = require('tldts');
4
+ const utils = require('../utils');
5
+ const ruleUtils = require('../rule');
6
+
7
+ // Prefix and separators for domain patterns
8
+ const DOMAIN_PREFIX = '||';
9
+ const DOMAIN_SEPARATOR = '^';
10
+ const WILDCARD = '*';
11
+ const WILDCARD_DOMAIN_PART = '*.';
12
+
13
+ /**
14
+ * Checks if a pattern is a 3-octet subnet with trailing dot or wildcard AND a || prefix.
15
+ * These are the only subnet patterns that work in AdGuard Home for ValidateAllowIp.
16
+ * Examples: ||192.168.1. ||192.168.1.*
17
+ *
18
+ * @param {string} s - The pattern string to check
19
+ * @returns {boolean}
20
+ */
21
+ function is3OctetSubnetWithSuffix(s) {
22
+ const c = utils.classifyIpPattern(s);
23
+ // Valid 3-octet subnet: ||prefix, trailing dot or wildcard, no caret
24
+ return c !== null && c.octetCount === 3 && c.isSubnetWildcard && !c.hasCaret && c.prefix === DOMAIN_PREFIX;
25
+ }
26
+
1
27
  /**
2
- * Checks if a pattern is an IP-like subnet with | or || prefix: 2 or 3 octets.
3
- * Examples: ||1.1^, ||1.1.2^, |1.1^, ||192.168^
4
- * These patterns block IP subnets (left-anchored) and should be rejected in Validate,
5
- * but allowed in ValidateAllowIp.
28
+ * Checks if a pattern is an IP-like subnet with | or || prefix: 1, 2, or 3 octets.
29
+ * These patterns should be rejected in Validate:
30
+ * - With ^ separator (||1^, ||1.1^, ||1.1.2^) — do NOT work
31
+ * - With trailing dot (||1.1., ||1.1.2.) — IP wildcard patterns
32
+ * - With trailing wildcard (||1.1.*, ||1.1.2.*) — IP wildcard patterns
33
+ *
34
+ * In ValidateAllowIp, 3-octet wildcards (||192.168.1., ||192.168.1.*) are allowed,
35
+ * but 1-2 octet patterns are still rejected as too wide.
36
+ *
6
37
  * @param {string} s - The pattern string to check
7
38
  * @returns {boolean} True if the pattern is an IP subnet pattern, false otherwise
8
39
  */
9
40
  function isIpSubnetPattern(s) {
10
- // Must have | or || prefix to be a subnet pattern (left-anchored)
11
- if (!s.startsWith('|')) {
41
+ const c = utils.classifyIpPattern(s);
42
+ if (!c || c.prefix === '') {
12
43
  return false;
13
44
  }
14
- // Remove prefixes (||, |)
15
- let pattern = s;
16
- if (pattern.startsWith('||')) {
17
- pattern = pattern.slice(2);
18
- } else if (pattern.startsWith('|')) {
19
- pattern = pattern.slice(1);
20
- }
21
- // Remove ^ and | at the end
22
- pattern = pattern.replace(/\^\|?$/, '');
23
- // Remove trailing dot if present
24
- if (pattern.endsWith('.')) {
25
- pattern = pattern.slice(0, -1);
26
- }
27
- // Must have 2 or 3 octets (1.1 or 1.1.2)
28
- const parts = pattern.split('.');
29
- if (parts.length !== 2 && parts.length !== 3) {
30
- return false;
45
+ if (c.octetCount < 4) {
46
+ return true;
31
47
  }
32
- // All parts must be numbers 0-255
33
- return parts.every((p) => /^\d{1,3}$/.test(p) && Number(p) >= 0 && Number(p) <= 255);
48
+ // 4-octet with trailing dot or wildcard is a malformed IP pattern (not a complete address)
49
+ return c.hasTrailingDot || c.hasTrailingWildcard;
34
50
  }
35
51
 
36
52
  /**
@@ -38,38 +54,61 @@ function isIpSubnetPattern(s) {
38
54
  * Examples: 1.1^, 1.1.1^, 1.1.1.1^, 192.168^
39
55
  * These patterns match string endings and block unpredictably (e.g., 1.1^ blocks 1.1.1.1, 1.1.111.1, example1.1).
40
56
  * They should ALWAYS be rejected (in both Validate and ValidateAllowIp).
57
+ * Note: In ValidateAllowIp, these patterns are normalized to ||ip^ before validation,
58
+ * so this function mainly catches patterns that slip through normalization.
41
59
  * @param {string} s - The pattern string to check
42
60
  * @returns {boolean} True if the pattern is an IP suffix pattern, false otherwise
43
61
  */
44
62
  function isIpSuffixPattern(s) {
45
- // Must NOT have | or || prefix (otherwise it's a subnet pattern, not a suffix)
46
- if (s.startsWith('|')) {
63
+ const c = utils.classifyIpPattern(s);
64
+ return c !== null && c.prefix === '' && c.hasCaret && c.octetCount >= 2;
65
+ }
66
+
67
+ /**
68
+ * Checks if a pattern is an unsafe/ambiguous IP pattern that should be rejected.
69
+ * These are patterns WITHOUT ^ that look like partial IPs:
70
+ * - 1-2 octet patterns (1.2., 1.2.*, 192.168) — too wide
71
+ * - 3 octet patterns without trailing dot/wildcard (192.168.1) — ambiguous
72
+ *
73
+ * These should be rejected in ALL validators (Validate, ValidateAllowIp, ValidateAllowPublicSuffix).
74
+ *
75
+ * @param {string} s - The pattern string to check
76
+ * @returns {boolean} True if the pattern is unsafe, false otherwise
77
+ */
78
+ function isUnsafeIpPattern(s) {
79
+ // Skip if has ^ (handled by isIpSuffixPattern)
80
+ if (s.includes('^')) {
47
81
  return false;
48
82
  }
49
- // Remove ^ and | at the end
50
- let pattern = s.replace(/\^\|?$/, '');
51
- // Remove trailing dot if present
52
- if (pattern.endsWith('.')) {
53
- pattern = pattern.slice(0, -1);
54
- }
55
- // Must have 2, 3, or 4 octets
56
- const parts = pattern.split('.');
57
- if (parts.length < 2 || parts.length > 4) {
83
+
84
+ const c = utils.classifyIpPattern(s);
85
+ if (!c) {
58
86
  return false;
59
87
  }
60
- // All parts must be numbers 0-255
61
- return parts.every((p) => /^\d{1,3}$/.test(p) && Number(p) >= 0 && Number(p) <= 255);
88
+
89
+ // 1-2 octets are always too wide
90
+ if (c.isTooWide) {
91
+ return true;
92
+ }
93
+
94
+ // 3 octets without trailing dot/wildcard are ambiguous
95
+ if (c.isAmbiguous3Octet) {
96
+ return true;
97
+ }
98
+
99
+ // 3 or 4-octet with trailing dot or wildcard, no prefix (e.g. 10.10.34., 10.10.34.*, 1.2.3.4., 1.2.3.4.*)
100
+ // Patterns with `||` prefix are handled by isIpSubnetPattern; in ValidateAllowIp,
101
+ // prefix-less 3-octet ones are pre-normalized to `||ip.` by ip-normalize.js before reaching here.
102
+ if ((c.octetCount === 3 || c.octetCount === 4)
103
+ && (c.hasTrailingDot || c.hasTrailingWildcard)
104
+ && c.prefix === '') {
105
+ return true;
106
+ }
107
+
108
+ return false;
62
109
  }
63
- const _ = require('lodash');
64
- const consola = require('consola');
65
- const tldts = require('tldts');
66
- const utils = require('../utils');
67
- const ruleUtils = require('../rule');
68
110
 
69
- const DOMAIN_PREFIX = '||';
70
- const DOMAIN_SEPARATOR = '^';
71
- const WILDCARD = '*';
72
- const WILDCARD_DOMAIN_PART = '*.';
111
+ const HOSTNAME_ALNUM_PATTERN = /[a-zA-Z0-9]/;
73
112
  // Matches exact domain-style adblock patterns: ||example.org^, *.org^, .org^, ||org^
74
113
  // Each part explained:
75
114
  const EXACT_DOMAIN_PATTERN = new RegExp(
@@ -139,13 +178,19 @@ const MAX_PATTERN_LENGTH = 5;
139
178
  function validHostname(hostname, ruleText, allowedIP, hasLimitModifier, allowPublicSuffix) {
140
179
  const result = tldts.parse(hostname);
141
180
 
142
- if (!result.hostname) {
143
- consola.debug(`invalid hostname ${hostname} in the rule: ${ruleText}`);
181
+ // tldts v7 may return a non-null hostname for malformed inputs consisting only of dots
182
+ // (".." => hostname: "."). Reject any hostname that does not contain at least one
183
+ // alphanumeric character, since real hostnames always have one.
184
+ // This is separate from the adblock-style `checkChars` filter in validAdblockRule(),
185
+ // which rejects raw Unicode patterns before they reach this function. Raw Unicode
186
+ // adblock rules are expected to pass through ConvertToAscii before validation.
187
+ if (!result.hostname || !HOSTNAME_ALNUM_PATTERN.test(result.hostname)) {
188
+ consola.debug(`malformed hostname ${hostname} in the rule: ${ruleText}`);
144
189
  return false;
145
190
  }
146
191
 
147
192
  if (!allowedIP && result.isIp) {
148
- consola.debug(`invalid hostname ${hostname} in the rule: ${ruleText}`);
193
+ consola.debug(`IP address ${hostname} is not allowed in the rule: ${ruleText}`);
149
194
  return false;
150
195
  }
151
196
 
@@ -291,6 +336,15 @@ function validAdblockRule(ruleText, allowedIP, allowPublicSuffix) {
291
336
  return false;
292
337
  }
293
338
 
339
+ // `denyallow` is only meaningful for domain-based rules.
340
+ // IP-like patterns with `denyallow` do not behave as expected, so reject them.
341
+ const hasDenyallowModifier = props.options
342
+ && props.options.some((option) => option.name === 'denyallow');
343
+ if (hasDenyallowModifier && utils.classifyIpPattern(props.pattern)) {
344
+ consola.debug(`denyallow is not supported for IP patterns: ${ruleText}`);
345
+ return false;
346
+ }
347
+
294
348
  // 3.5. Check if the base pattern is an IP address or IP-like pattern.
295
349
 
296
350
  // Reject IP-suffix patterns (1.1^, 1.1.1^, 1.1.1.1^) ALWAYS — they match string endings unpredictably
@@ -300,12 +354,23 @@ function validAdblockRule(ruleText, allowedIP, allowPublicSuffix) {
300
354
  return false;
301
355
  }
302
356
 
303
- // Reject IP-subnet patterns (||1.1^, ||1.1.2^) only in Validate, allow in ValidateAllowIp
304
- if (!allowedIP && isIpSubnetPattern(props.pattern)) {
305
- consola.debug(`IP-like subnet is not allowed: ${ruleText}`);
357
+ // Reject unsafe IP patterns (1.2., 1.2.*, 192.168.1) ALWAYS — too wide or ambiguous
358
+ // These don't have ^ so they slip through isIpSuffixPattern
359
+ if (isUnsafeIpPattern(props.pattern)) {
360
+ consola.debug(`Unsafe IP pattern is not allowed (too wide or ambiguous): ${ruleText}`);
306
361
  return false;
307
362
  }
308
363
 
364
+ // Reject IP-subnet patterns (||1.1^, ||1.1.2^, ||1.1.) always, except in ValidateAllowIp
365
+ // where only 3-octet subnets with trailing dot/wildcard (||192.168.1., ||192.168.1.*) are valid.
366
+ // 1-2 octet patterns and 3-octet with ^ are rejected even when allowedIP=true.
367
+ if (isIpSubnetPattern(props.pattern)) {
368
+ if (!allowedIP || !is3OctetSubnetWithSuffix(props.pattern)) {
369
+ consola.debug(`IP-like subnet is not allowed: ${ruleText}`);
370
+ return false;
371
+ }
372
+ }
373
+
309
374
  // Check for full 4-octet IP addresses (||1.1.1.1^, |1.1.1.1^)
310
375
  let ipCandidate = props.pattern;
311
376
  if (_.startsWith(ipCandidate, DOMAIN_PREFIX)) {
package/src/utils.js CHANGED
@@ -1,5 +1,105 @@
1
1
  const _ = require('lodash');
2
2
 
3
+ // Prefix for canonical IP/domain patterns
4
+ const IP_PREFIX = '||';
5
+
6
+ /**
7
+ * Regex that matches strings that look like adblock rule modifiers
8
+ * Used to distinguish a real `$modifier` suffix from a `$` that is
9
+ * part of a regex anchor.
10
+ */
11
+ const MODIFIER_REGEX = /^[a-zA-Z0-9,=~_.-]+$/;
12
+
13
+ /**
14
+ * Regex to match a valid IPv4 octet (0-255).
15
+ */
16
+ const OCTET_REGEX = /^\d{1,3}$/;
17
+
18
+ /**
19
+ * Checks if a string is a valid IPv4 octet (0-255).
20
+ *
21
+ * @param {string} s - The string to check.
22
+ * @returns {boolean} True if valid octet.
23
+ */
24
+ function isValidOctet(s) {
25
+ if (!OCTET_REGEX.test(s)) {
26
+ return false;
27
+ }
28
+ const num = Number(s);
29
+ return num >= 0 && num <= 255;
30
+ }
31
+
32
+ /**
33
+ * Parses an IP-like adblock pattern and extracts its structural components.
34
+ * Expects a pattern already stripped of @@ prefix and $modifier suffix
35
+ * (as returned by parseRuleTokens in rule.js).
36
+ *
37
+ * @param {string} pattern - The adblock pattern.
38
+ * @returns {{ prefix: string, octets: string[],
39
+ * hasTrailingDot: boolean, hasTrailingWildcard: boolean,
40
+ * hasCaret: boolean, hasCaretPipe: boolean }|null}
41
+ * Parsed components, or null if the pattern is not a valid IP pattern.
42
+ */
43
+ function parseIpPattern(pattern) {
44
+ let remaining = pattern;
45
+ let prefix = '';
46
+
47
+ // Extract prefix
48
+ if (remaining.startsWith(IP_PREFIX)) {
49
+ prefix = IP_PREFIX;
50
+ remaining = remaining.slice(2);
51
+ } else if (remaining.startsWith('|')) {
52
+ prefix = '|';
53
+ remaining = remaining.slice(1);
54
+ }
55
+
56
+ // Check for ^| or ^ suffix
57
+ let hasCaret = false;
58
+ let hasCaretPipe = false;
59
+ if (remaining.endsWith('^|')) {
60
+ hasCaretPipe = true;
61
+ hasCaret = true;
62
+ remaining = remaining.slice(0, -2);
63
+ } else if (remaining.endsWith('^')) {
64
+ hasCaret = true;
65
+ remaining = remaining.slice(0, -1);
66
+ }
67
+
68
+ // Check for trailing wildcard (.*)
69
+ let hasTrailingWildcard = false;
70
+ if (remaining.endsWith('.*')) {
71
+ hasTrailingWildcard = true;
72
+ remaining = remaining.slice(0, -2);
73
+ }
74
+
75
+ // Check for trailing dot
76
+ let hasTrailingDot = false;
77
+ if (remaining.endsWith('.')) {
78
+ hasTrailingDot = true;
79
+ remaining = remaining.slice(0, -1);
80
+ }
81
+
82
+ // Split into octets
83
+ const parts = remaining.split('.');
84
+ if (parts.length === 0 || parts.length > 4) {
85
+ return null;
86
+ }
87
+
88
+ // All parts must be valid octets
89
+ if (!parts.every(isValidOctet)) {
90
+ return null;
91
+ }
92
+
93
+ return {
94
+ prefix,
95
+ octets: parts,
96
+ hasTrailingDot,
97
+ hasTrailingWildcard,
98
+ hasCaret,
99
+ hasCaretPipe,
100
+ };
101
+ }
102
+
3
103
  /**
4
104
  * Extracts a substring between two tags.
5
105
  *
@@ -136,7 +236,44 @@ class Wildcard {
136
236
  }
137
237
  }
138
238
 
239
+ /**
240
+ * Classifies an IP-like adblock pattern into categories used by validators and normalizers.
241
+ * Returns null when the pattern cannot be parsed as an IP-like pattern.
242
+ *
243
+ * @param {string} pattern - The adblock pattern to classify.
244
+ * @returns {{ prefix: string, octets: string[],
245
+ * hasTrailingDot: boolean, hasTrailingWildcard: boolean,
246
+ * hasCaret: boolean, hasCaretPipe: boolean,
247
+ * octetCount: number, isFullIp: boolean,
248
+ * isSubnetWildcard: boolean, isAmbiguous3Octet: boolean,
249
+ * isTooWide: boolean }|null}
250
+ */
251
+ function classifyIpPattern(pattern) {
252
+ const parsed = parseIpPattern(pattern);
253
+ if (!parsed) {
254
+ return null;
255
+ }
256
+
257
+ return {
258
+ ...parsed,
259
+ octetCount: parsed.octets.length,
260
+ isFullIp: parsed.octets.length === 4
261
+ && !parsed.hasTrailingDot
262
+ && !parsed.hasTrailingWildcard,
263
+ isSubnetWildcard: parsed.octets.length < 4
264
+ && (parsed.hasTrailingDot || parsed.hasTrailingWildcard),
265
+ isAmbiguous3Octet: parsed.octets.length === 3
266
+ && !parsed.hasTrailingDot
267
+ && !parsed.hasTrailingWildcard
268
+ && !parsed.hasCaret,
269
+ isTooWide: parsed.octets.length <= 2,
270
+ };
271
+ }
272
+
139
273
  module.exports = {
274
+ MODIFIER_REGEX,
275
+ parseIpPattern,
276
+ classifyIpPattern,
140
277
  Wildcard,
141
278
  splitByDelimiterWithEscapeCharacter,
142
279
  substringBetween,
@@ -95,7 +95,10 @@ describe('Configuration', () => {
95
95
  it.each([
96
96
  ['Validate', 'ValidateAllowIp'],
97
97
  ['Validate', 'ValidateAllowPublicSuffix'],
98
+ ['Validate', 'ValidateAllowIpAndPublicSuffix'],
98
99
  ['ValidateAllowIp', 'ValidateAllowPublicSuffix'],
100
+ ['ValidateAllowIp', 'ValidateAllowIpAndPublicSuffix'],
101
+ ['ValidateAllowPublicSuffix', 'ValidateAllowIpAndPublicSuffix'],
99
102
  ])('test incompatible top-level transformations configuration: %s + %s', (first, second) => {
100
103
  const ret = config.validateConfiguration({
101
104
  name: 'test',
@@ -117,7 +120,10 @@ describe('Configuration', () => {
117
120
  it.each([
118
121
  ['Validate', 'ValidateAllowIp'],
119
122
  ['Validate', 'ValidateAllowPublicSuffix'],
123
+ ['Validate', 'ValidateAllowIpAndPublicSuffix'],
120
124
  ['ValidateAllowIp', 'ValidateAllowPublicSuffix'],
125
+ ['ValidateAllowIp', 'ValidateAllowIpAndPublicSuffix'],
126
+ ['ValidateAllowPublicSuffix', 'ValidateAllowIpAndPublicSuffix'],
121
127
  ])('test incompatible source transformations configuration: %s + %s', (first, second) => {
122
128
  const ret = config.validateConfiguration({
123
129
  name: 'test',
@@ -0,0 +1,65 @@
1
+ /**
2
+ * IP pipeline alignment tests.
3
+ *
4
+ * Runs the same pattern matrix through processIpRule(), Validate, and
5
+ * ValidateAllowIp to ensure all three agree — and to catch any future
6
+ * divergence caused by editing one module without updating the others.
7
+ */
8
+ const { ACTION, processIpRule } = require('../../src/transformations/ip-normalize');
9
+ const { validate } = require('../../src/transformations/validate');
10
+ const { validateAllowIp } = require('../../src/transformations/validate-allow-ip');
11
+
12
+ /**
13
+ * Each row: [ input, expectedNormalized, survivesValidate, survivesValidateAllowIp ]
14
+ *
15
+ * expectedNormalized: if processIpRule rewrites the rule, this is the canonical
16
+ * form; null means processIpRule leaves the input unchanged (keep/allow).
17
+ * survives*: whether the canonical form is kept by that transformation.
18
+ *
19
+ * Note: Validate (without AllowIp) rejects all IP-like patterns regardless of
20
+ * canonical form — IPs are only permitted in the ValidateAllowIp path.
21
+ */
22
+ const CASES = [
23
+ // Full 4-octet IPs — already canonical; dropped by Validate, kept by ValidateAllowIp
24
+ ['||1.2.3.4^', null, false, true],
25
+ // Full 4-octet IPs — normalizer rewrites to canonical form first
26
+ ['1.2.3.4', '||1.2.3.4^', false, true],
27
+ ['1.2.3.4^', '||1.2.3.4^', false, true],
28
+ ['1.2.3.4^|', '||1.2.3.4^', false, true],
29
+ ['|1.2.3.4^|', '||1.2.3.4^', false, true],
30
+ ['||1.2.3.4^|', '||1.2.3.4^', false, true],
31
+ // 3-octet subnets with || — normalizer keeps them as-is
32
+ ['||192.168.1.', null, false, true],
33
+ ['||192.168.1.*', null, false, true],
34
+ // 3-octet subnets without || — normalizer adds the prefix
35
+ ['192.168.1.', '||192.168.1.', false, true],
36
+ ['192.168.1.*', '||192.168.1.*', false, true],
37
+ ['||192.168.1.^|', null, false, false],
38
+ // Patterns rejected by all validators
39
+ ['||1.2.3^', null, false, false],
40
+ ['||1.2^', null, false, false],
41
+ ['192.168.1', null, false, false],
42
+ ];
43
+
44
+ describe('IP pipeline alignment: processIpRule / Validate / ValidateAllowIp', () => {
45
+ it.each(CASES)(
46
+ 'input=%s, normalized=%s, validate=%s, allowIp=%s',
47
+ (input, expectedNorm, expectsValidate, expectsAllowIp) => {
48
+ // 1. Normalization step
49
+ const result = processIpRule(input);
50
+ const canonical = result.action === ACTION.NORMALIZE ? result.normalized : input;
51
+
52
+ if (expectedNorm !== null) {
53
+ expect(canonical).toBe(expectedNorm);
54
+ } else {
55
+ expect(canonical).toBe(input);
56
+ }
57
+
58
+ // 2. Validate — receives the canonical form (no IP normalization in this path)
59
+ expect(validate([canonical]).includes(canonical)).toBe(expectsValidate);
60
+
61
+ // 3. ValidateAllowIp — receives the raw input (it runs normalization internally)
62
+ expect(validateAllowIp([input]).includes(canonical)).toBe(expectsAllowIp);
63
+ },
64
+ );
65
+ });