@adguard/hostlist-compiler 1.0.39 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +175 -0
- package/CHANGELOG.md +41 -0
- package/DEVELOPMENT.md +172 -0
- package/README.md +142 -95
- package/bamboo-specs/build.yaml +2 -10
- package/bamboo-specs/deploy.yaml +0 -7
- package/bamboo-specs/increment.yaml +1 -1
- package/bamboo-specs/test.yaml +0 -8
- package/package.json +3 -3
- package/src/index.d.ts +6 -2
- package/src/index.js +44 -1
- package/src/schemas/configuration.schema.json +200 -0
- package/src/transformations/ip-normalize.js +182 -0
- package/src/transformations/transform.js +37 -1
- package/src/transformations/validate-allow-ip-and-public-suffix.js +33 -0
- package/src/transformations/validate-allow-ip.js +18 -4
- package/src/transformations/validate-allow-public-suffix.js +20 -0
- package/src/transformations/validate.js +251 -22
- package/src/utils.js +137 -0
- package/test/configuration.test.js +70 -0
- package/test/index.test.js +64 -0
- package/test/transformations/ip-alignment.test.js +65 -0
- package/test/transformations/ip-normalize.test.js +331 -0
- package/test/transformations/transform.test.js +25 -0
- package/test/transformations/validate-allow-ip-and-public-suffix.test.js +167 -0
- package/test/transformations/validate-allow-ip.test.js +72 -1
- package/test/transformations/validate-allow-public-suffix.test.js +284 -0
- package/test/transformations/validate.test.js +251 -0
|
@@ -4,11 +4,136 @@ const tldts = require('tldts');
|
|
|
4
4
|
const utils = require('../utils');
|
|
5
5
|
const ruleUtils = require('../rule');
|
|
6
6
|
|
|
7
|
+
// Prefix and separators for domain patterns
|
|
7
8
|
const DOMAIN_PREFIX = '||';
|
|
8
9
|
const DOMAIN_SEPARATOR = '^';
|
|
9
10
|
const WILDCARD = '*';
|
|
10
11
|
const WILDCARD_DOMAIN_PART = '*.';
|
|
11
12
|
|
|
13
|
+
/**
|
|
14
|
+
* Checks if a pattern is a 3-octet subnet with trailing dot or wildcard AND a || prefix.
|
|
15
|
+
* These are the only subnet patterns that work in AdGuard Home for ValidateAllowIp.
|
|
16
|
+
* Examples: ||192.168.1. ||192.168.1.*
|
|
17
|
+
*
|
|
18
|
+
* @param {string} s - The pattern string to check
|
|
19
|
+
* @returns {boolean}
|
|
20
|
+
*/
|
|
21
|
+
function is3OctetSubnetWithSuffix(s) {
|
|
22
|
+
const c = utils.classifyIpPattern(s);
|
|
23
|
+
// Valid 3-octet subnet: ||prefix, trailing dot or wildcard, no caret
|
|
24
|
+
return c !== null && c.octetCount === 3 && c.isSubnetWildcard && !c.hasCaret && c.prefix === DOMAIN_PREFIX;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Checks if a pattern is an IP-like subnet with | or || prefix: 1, 2, or 3 octets.
|
|
29
|
+
* These patterns should be rejected in Validate:
|
|
30
|
+
* - With ^ separator (||1^, ||1.1^, ||1.1.2^) — do NOT work
|
|
31
|
+
* - With trailing dot (||1.1., ||1.1.2.) — IP wildcard patterns
|
|
32
|
+
* - With trailing wildcard (||1.1.*, ||1.1.2.*) — IP wildcard patterns
|
|
33
|
+
*
|
|
34
|
+
* In ValidateAllowIp, 3-octet wildcards (||192.168.1., ||192.168.1.*) are allowed,
|
|
35
|
+
* but 1-2 octet patterns are still rejected as too wide.
|
|
36
|
+
*
|
|
37
|
+
* @param {string} s - The pattern string to check
|
|
38
|
+
* @returns {boolean} True if the pattern is an IP subnet pattern, false otherwise
|
|
39
|
+
*/
|
|
40
|
+
function isIpSubnetPattern(s) {
|
|
41
|
+
const c = utils.classifyIpPattern(s);
|
|
42
|
+
if (!c || c.prefix === '') {
|
|
43
|
+
return false;
|
|
44
|
+
}
|
|
45
|
+
if (c.octetCount < 4) {
|
|
46
|
+
return true;
|
|
47
|
+
}
|
|
48
|
+
// 4-octet with trailing dot or wildcard is a malformed IP pattern (not a complete address)
|
|
49
|
+
return c.hasTrailingDot || c.hasTrailingWildcard;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Checks if a pattern is an IP-suffix (no | or || prefix): 2, 3, or 4 octets.
|
|
54
|
+
* Examples: 1.1^, 1.1.1^, 1.1.1.1^, 192.168^
|
|
55
|
+
* These patterns match string endings and block unpredictably (e.g., 1.1^ blocks 1.1.1.1, 1.1.111.1, example1.1).
|
|
56
|
+
* They should ALWAYS be rejected (in both Validate and ValidateAllowIp).
|
|
57
|
+
* Note: In ValidateAllowIp, these patterns are normalized to ||ip^ before validation,
|
|
58
|
+
* so this function mainly catches patterns that slip through normalization.
|
|
59
|
+
* @param {string} s - The pattern string to check
|
|
60
|
+
* @returns {boolean} True if the pattern is an IP suffix pattern, false otherwise
|
|
61
|
+
*/
|
|
62
|
+
function isIpSuffixPattern(s) {
|
|
63
|
+
const c = utils.classifyIpPattern(s);
|
|
64
|
+
return c !== null && c.prefix === '' && c.hasCaret && c.octetCount >= 2;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Checks if a pattern is an unsafe/ambiguous IP pattern that should be rejected.
|
|
69
|
+
* These are patterns WITHOUT ^ that look like partial IPs:
|
|
70
|
+
* - 1-2 octet patterns (1.2., 1.2.*, 192.168) — too wide
|
|
71
|
+
* - 3 octet patterns without trailing dot/wildcard (192.168.1) — ambiguous
|
|
72
|
+
*
|
|
73
|
+
* These should be rejected in ALL validators (Validate, ValidateAllowIp, ValidateAllowPublicSuffix).
|
|
74
|
+
*
|
|
75
|
+
* @param {string} s - The pattern string to check
|
|
76
|
+
* @returns {boolean} True if the pattern is unsafe, false otherwise
|
|
77
|
+
*/
|
|
78
|
+
function isUnsafeIpPattern(s) {
|
|
79
|
+
// Skip if has ^ (handled by isIpSuffixPattern)
|
|
80
|
+
if (s.includes('^')) {
|
|
81
|
+
return false;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const c = utils.classifyIpPattern(s);
|
|
85
|
+
if (!c) {
|
|
86
|
+
return false;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// 1-2 octets are always too wide
|
|
90
|
+
if (c.isTooWide) {
|
|
91
|
+
return true;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// 3 octets without trailing dot/wildcard are ambiguous
|
|
95
|
+
if (c.isAmbiguous3Octet) {
|
|
96
|
+
return true;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// 3 or 4-octet with trailing dot or wildcard, no prefix (e.g. 10.10.34., 10.10.34.*, 1.2.3.4., 1.2.3.4.*)
|
|
100
|
+
// Patterns with `||` prefix are handled by isIpSubnetPattern; in ValidateAllowIp,
|
|
101
|
+
// prefix-less 3-octet ones are pre-normalized to `||ip.` by ip-normalize.js before reaching here.
|
|
102
|
+
if ((c.octetCount === 3 || c.octetCount === 4)
|
|
103
|
+
&& (c.hasTrailingDot || c.hasTrailingWildcard)
|
|
104
|
+
&& c.prefix === '') {
|
|
105
|
+
return true;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
return false;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const HOSTNAME_ALNUM_PATTERN = /[a-zA-Z0-9]/;
|
|
112
|
+
// Matches exact domain-style adblock patterns: ||example.org^, *.org^, .org^, ||org^
|
|
113
|
+
// Each part explained:
|
|
114
|
+
const EXACT_DOMAIN_PATTERN = new RegExp(
|
|
115
|
+
// optional || prefix: matches ||example.org^, does not require it for .org^ or *.org^
|
|
116
|
+
'^(?:\\|\\|)?'
|
|
117
|
+
// optional leading *. or .: matches *.org^, .org^; absent for plain ||example.org^
|
|
118
|
+
+ '(?:\\*\\.|\\.)?'
|
|
119
|
+
// first label: alnum start/end, hyphens only inside
|
|
120
|
+
// valid: example, foo-bar, xn--d1acufc
|
|
121
|
+
// invalid: -example, example-, (empty)
|
|
122
|
+
+ '([a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?'
|
|
123
|
+
// additional dot-separated labels with same constraints
|
|
124
|
+
// valid: example.org, sub.example.org, foo-bar.baz
|
|
125
|
+
// invalid: example..org, example.-org, example.org-
|
|
126
|
+
+ '(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?)*'
|
|
127
|
+
// optional trailing dot: FQDN notation, normalized away before tldts
|
|
128
|
+
// valid: foo.bar. (treated as foo.bar)
|
|
129
|
+
// invalid: foo.. (two dots — not matched)
|
|
130
|
+
+ '\\.?)'
|
|
131
|
+
// required ^ separator; optional trailing | for exception-end anchors
|
|
132
|
+
// valid: ||example.org^, ||example.org^|
|
|
133
|
+
// invalid: ||example.org, ||example.org^abc
|
|
134
|
+
+ '\\^\\|?$',
|
|
135
|
+
);
|
|
136
|
+
|
|
12
137
|
// TODO: Remove lodash from the project if possible
|
|
13
138
|
|
|
14
139
|
/**
|
|
@@ -27,7 +152,6 @@ const ANY_PATTERN_MODIFIER = [
|
|
|
27
152
|
const SUPPORTED_MODIFIERS = [
|
|
28
153
|
'important',
|
|
29
154
|
'~important',
|
|
30
|
-
'ctag',
|
|
31
155
|
// DNS-related modifiers.
|
|
32
156
|
'dnstype',
|
|
33
157
|
'dnsrewrite',
|
|
@@ -48,29 +172,63 @@ const MAX_PATTERN_LENGTH = 5;
|
|
|
48
172
|
* @param {string} ruleText - original rule text (for logging)
|
|
49
173
|
* @param {boolean} allowedIP - flag to determine if IP validation is allowed
|
|
50
174
|
* @param {boolean} hasLimitModifier - flag to determine if the rule has a limit modifier, e.g. denyallow
|
|
175
|
+
* @param {boolean} allowPublicSuffix - flag to determine if matching whole public suffixes is allowed
|
|
51
176
|
* @returns {boolean} true if the hostname is okay to be in the blocklist.
|
|
52
177
|
*/
|
|
53
|
-
function validHostname(hostname, ruleText, allowedIP, hasLimitModifier) {
|
|
178
|
+
function validHostname(hostname, ruleText, allowedIP, hasLimitModifier, allowPublicSuffix) {
|
|
54
179
|
const result = tldts.parse(hostname);
|
|
55
180
|
|
|
56
|
-
|
|
57
|
-
|
|
181
|
+
// tldts v7 may return a non-null hostname for malformed inputs consisting only of dots
|
|
182
|
+
// (".." => hostname: "."). Reject any hostname that does not contain at least one
|
|
183
|
+
// alphanumeric character, since real hostnames always have one.
|
|
184
|
+
// This is separate from the adblock-style `checkChars` filter in validAdblockRule(),
|
|
185
|
+
// which rejects raw Unicode patterns before they reach this function. Raw Unicode
|
|
186
|
+
// adblock rules are expected to pass through ConvertToAscii before validation.
|
|
187
|
+
if (!result.hostname || !HOSTNAME_ALNUM_PATTERN.test(result.hostname)) {
|
|
188
|
+
consola.debug(`malformed hostname ${hostname} in the rule: ${ruleText}`);
|
|
58
189
|
return false;
|
|
59
190
|
}
|
|
60
191
|
|
|
61
192
|
if (!allowedIP && result.isIp) {
|
|
62
|
-
consola.debug(`
|
|
193
|
+
consola.debug(`IP address ${hostname} is not allowed in the rule: ${ruleText}`);
|
|
63
194
|
return false;
|
|
64
195
|
}
|
|
65
196
|
|
|
66
197
|
if (result.hostname === result.publicSuffix && !hasLimitModifier) {
|
|
67
|
-
|
|
68
|
-
|
|
198
|
+
if (!allowPublicSuffix) {
|
|
199
|
+
consola.debug(`matching the whole public suffix ${hostname} is not allowed: ${ruleText}`);
|
|
200
|
+
return false;
|
|
201
|
+
}
|
|
202
|
+
// Even with allowPublicSuffix, reject hostnames that tldts does not recognise
|
|
203
|
+
// as real ICANN or private public suffixes. Without this check, any single-label
|
|
204
|
+
// garbage like "a", "aa", "1" passes because tldts defaults hostname === publicSuffix
|
|
205
|
+
// for unknown labels.
|
|
206
|
+
if (!result.isIcann && !result.isPrivate) {
|
|
207
|
+
consola.debug(`hostname ${hostname} is not a known public suffix: ${ruleText}`);
|
|
208
|
+
return false;
|
|
209
|
+
}
|
|
69
210
|
}
|
|
70
211
|
|
|
71
212
|
return true;
|
|
72
213
|
}
|
|
73
214
|
|
|
215
|
+
/**
|
|
216
|
+
* Extracts a domain-like pattern from rules such as ||org^, ||*.org^, .org^ or *.org^.
|
|
217
|
+
*
|
|
218
|
+
* @param {string} pattern - adblock rule pattern without modifiers
|
|
219
|
+
* @returns {string|null} normalized hostname to validate or null if the pattern does not match
|
|
220
|
+
*/
|
|
221
|
+
function extractDomainPattern(pattern) {
|
|
222
|
+
const match = pattern.match(EXACT_DOMAIN_PATTERN);
|
|
223
|
+
if (!match) {
|
|
224
|
+
return null;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
const hostname = match[1];
|
|
228
|
+
// Allow trailing dot as absolute FQDN marker by normalizing it away for validation.
|
|
229
|
+
return hostname.endsWith('.') ? hostname.slice(0, -1) : hostname;
|
|
230
|
+
}
|
|
231
|
+
|
|
74
232
|
/**
|
|
75
233
|
* Validates an /etc/hosts rule.
|
|
76
234
|
*
|
|
@@ -81,9 +239,10 @@ function validHostname(hostname, ruleText, allowedIP, hasLimitModifier) {
|
|
|
81
239
|
*
|
|
82
240
|
* @param {string} ruleText - rule text
|
|
83
241
|
* @param {boolean} allowIP - flag to determine if IP validation is allowed
|
|
242
|
+
* @param {boolean} allowPublicSuffix - flag to determine if matching whole public suffixes is allowed
|
|
84
243
|
* @returns {boolean} true if the rule is a valid /etc/hosts rule
|
|
85
244
|
*/
|
|
86
|
-
function validEtcHostsRule(ruleText, allowIP) {
|
|
245
|
+
function validEtcHostsRule(ruleText, allowIP, allowPublicSuffix) {
|
|
87
246
|
let props;
|
|
88
247
|
try {
|
|
89
248
|
props = ruleUtils.loadEtcHostsRuleProperties(ruleText);
|
|
@@ -96,7 +255,7 @@ function validEtcHostsRule(ruleText, allowIP) {
|
|
|
96
255
|
return false;
|
|
97
256
|
}
|
|
98
257
|
|
|
99
|
-
if (props.hostnames.some((h) => !validHostname(h, ruleText, allowIP))) {
|
|
258
|
+
if (props.hostnames.some((h) => !validHostname(h, ruleText, allowIP, false, allowPublicSuffix))) {
|
|
100
259
|
return false;
|
|
101
260
|
}
|
|
102
261
|
|
|
@@ -114,9 +273,10 @@ function validEtcHostsRule(ruleText, allowIP) {
|
|
|
114
273
|
*
|
|
115
274
|
* @param {string} ruleText - rule text
|
|
116
275
|
* @param {boolean} allowedIP - flag to determine if IP validation is allowed
|
|
276
|
+
* @param {boolean} allowPublicSuffix - flag to determine if matching whole public suffixes is allowed
|
|
117
277
|
* @returns {boolean} - adblock-style rule
|
|
118
278
|
*/
|
|
119
|
-
function validAdblockRule(ruleText, allowedIP) {
|
|
279
|
+
function validAdblockRule(ruleText, allowedIP, allowPublicSuffix) {
|
|
120
280
|
let props;
|
|
121
281
|
try {
|
|
122
282
|
props = ruleUtils.loadAdblockRuleProperties(ruleText);
|
|
@@ -144,7 +304,10 @@ function validAdblockRule(ruleText, allowedIP) {
|
|
|
144
304
|
}
|
|
145
305
|
|
|
146
306
|
// 2. It checks whether the pattern is not too wide (should be at least 5 characters).
|
|
147
|
-
|
|
307
|
+
// Exception: exact domain-style patterns (||uk^, .uk^, etc.) bypass this check even if shorter,
|
|
308
|
+
// because they must reach step 4 to be validated as public suffixes by tldts.
|
|
309
|
+
const exactDomainPattern = extractDomainPattern(props.pattern);
|
|
310
|
+
if (props.pattern.length < MAX_PATTERN_LENGTH && !exactDomainPattern) {
|
|
148
311
|
consola.debug(`The rule is too short: ${ruleText}`);
|
|
149
312
|
return false;
|
|
150
313
|
}
|
|
@@ -173,6 +336,59 @@ function validAdblockRule(ruleText, allowedIP) {
|
|
|
173
336
|
return false;
|
|
174
337
|
}
|
|
175
338
|
|
|
339
|
+
// `denyallow` is only meaningful for domain-based rules.
|
|
340
|
+
// IP-like patterns with `denyallow` do not behave as expected, so reject them.
|
|
341
|
+
const hasDenyallowModifier = props.options
|
|
342
|
+
&& props.options.some((option) => option.name === 'denyallow');
|
|
343
|
+
if (hasDenyallowModifier && utils.classifyIpPattern(props.pattern)) {
|
|
344
|
+
consola.debug(`denyallow is not supported for IP patterns: ${ruleText}`);
|
|
345
|
+
return false;
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
// 3.5. Check if the base pattern is an IP address or IP-like pattern.
|
|
349
|
+
|
|
350
|
+
// Reject IP-suffix patterns (1.1^, 1.1.1^, 1.1.1.1^) ALWAYS — they match string endings unpredictably
|
|
351
|
+
// Must check BEFORE tldts.isIp because 4-octet suffixes would be recognized as valid IPs
|
|
352
|
+
if (isIpSuffixPattern(props.pattern)) {
|
|
353
|
+
consola.debug(`IP-suffix pattern is not allowed (blocks unpredictably): ${ruleText}`);
|
|
354
|
+
return false;
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
// Reject unsafe IP patterns (1.2., 1.2.*, 192.168.1) ALWAYS — too wide or ambiguous
|
|
358
|
+
// These don't have ^ so they slip through isIpSuffixPattern
|
|
359
|
+
if (isUnsafeIpPattern(props.pattern)) {
|
|
360
|
+
consola.debug(`Unsafe IP pattern is not allowed (too wide or ambiguous): ${ruleText}`);
|
|
361
|
+
return false;
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
// Reject IP-subnet patterns (||1.1^, ||1.1.2^, ||1.1.) always, except in ValidateAllowIp
|
|
365
|
+
// where only 3-octet subnets with trailing dot/wildcard (||192.168.1., ||192.168.1.*) are valid.
|
|
366
|
+
// 1-2 octet patterns and 3-octet with ^ are rejected even when allowedIP=true.
|
|
367
|
+
if (isIpSubnetPattern(props.pattern)) {
|
|
368
|
+
if (!allowedIP || !is3OctetSubnetWithSuffix(props.pattern)) {
|
|
369
|
+
consola.debug(`IP-like subnet is not allowed: ${ruleText}`);
|
|
370
|
+
return false;
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
// Check for full 4-octet IP addresses (||1.1.1.1^, |1.1.1.1^)
|
|
375
|
+
let ipCandidate = props.pattern;
|
|
376
|
+
if (_.startsWith(ipCandidate, DOMAIN_PREFIX)) {
|
|
377
|
+
ipCandidate = ipCandidate.slice(DOMAIN_PREFIX.length);
|
|
378
|
+
} else if (_.startsWith(ipCandidate, '|')) {
|
|
379
|
+
ipCandidate = ipCandidate.slice(1);
|
|
380
|
+
}
|
|
381
|
+
ipCandidate = ipCandidate.replace(/\^\|?$/, '');
|
|
382
|
+
|
|
383
|
+
if (tldts.parse(ipCandidate).isIp) {
|
|
384
|
+
if (!allowedIP) {
|
|
385
|
+
consola.debug(`IP address is not allowed: ${ruleText}`);
|
|
386
|
+
return false;
|
|
387
|
+
}
|
|
388
|
+
// IP is allowed — no further domain validation needed
|
|
389
|
+
return true;
|
|
390
|
+
}
|
|
391
|
+
|
|
176
392
|
// 4. Validate domain name
|
|
177
393
|
// Note that we don't check rules that contain wildcard characters
|
|
178
394
|
const sepIdx = props.pattern.indexOf(DOMAIN_SEPARATOR);
|
|
@@ -182,13 +398,18 @@ function validAdblockRule(ruleText, allowedIP) {
|
|
|
182
398
|
return false;
|
|
183
399
|
}
|
|
184
400
|
|
|
401
|
+
if (exactDomainPattern) {
|
|
402
|
+
return validHostname(exactDomainPattern, ruleText, allowedIP, hasLimitModifier, allowPublicSuffix);
|
|
403
|
+
}
|
|
404
|
+
|
|
185
405
|
// Check if the pattern does not start with the domain prefix and does not contain a domain separator
|
|
186
406
|
if (!_.startsWith(props.pattern, DOMAIN_PREFIX) || sepIdx === -1) {
|
|
187
407
|
return true;
|
|
188
408
|
}
|
|
189
409
|
|
|
190
|
-
// Extract the domain to check from the
|
|
191
|
-
|
|
410
|
+
// Extract the domain to check from the parsed pattern (not ruleText) to stay consistent
|
|
411
|
+
// with sepIdx/wildcardIdx which are also computed from props.pattern.
|
|
412
|
+
const domainToCheck = utils.substringBetween(props.pattern, DOMAIN_PREFIX, DOMAIN_SEPARATOR);
|
|
192
413
|
|
|
193
414
|
// If there are wildcard characters in the pattern
|
|
194
415
|
if (wildcardIdx !== -1) {
|
|
@@ -201,14 +422,14 @@ function validAdblockRule(ruleText, allowedIP) {
|
|
|
201
422
|
// If it's a wildcard with TLD, validate the cleaned TLD
|
|
202
423
|
if (startsWithWildcard && isOnlyTLD) {
|
|
203
424
|
const cleanedDomain = domainToCheck.replace(WILDCARD_DOMAIN_PART, '');
|
|
204
|
-
return validHostname(cleanedDomain, ruleText, allowedIP, hasLimitModifier);
|
|
425
|
+
return validHostname(cleanedDomain, ruleText, allowedIP, hasLimitModifier, allowPublicSuffix);
|
|
205
426
|
}
|
|
206
427
|
// If the rule has wildcard characters but is not a TLD (e.g., ||*.example.org^)
|
|
207
428
|
return true;
|
|
208
429
|
}
|
|
209
430
|
|
|
210
431
|
// Validate the domain
|
|
211
|
-
if (!validHostname(domainToCheck, ruleText, allowedIP, hasLimitModifier)) {
|
|
432
|
+
if (!validHostname(domainToCheck, ruleText, allowedIP, hasLimitModifier, allowPublicSuffix)) {
|
|
212
433
|
return false;
|
|
213
434
|
}
|
|
214
435
|
|
|
@@ -230,18 +451,19 @@ function validAdblockRule(ruleText, allowedIP) {
|
|
|
230
451
|
*
|
|
231
452
|
* @param {string} ruleText - rule to check
|
|
232
453
|
* @param {boolean} allowedIP - flag to determine if IP validation is allowed
|
|
454
|
+
* @param {boolean} allowPublicSuffix - flag to determine if matching whole public suffixes is allowed
|
|
233
455
|
* @returns {boolean} - true if the rule is valid, false otherwise
|
|
234
456
|
*/
|
|
235
|
-
function valid(ruleText, allowedIP) {
|
|
457
|
+
function valid(ruleText, allowedIP, allowPublicSuffix) {
|
|
236
458
|
if (ruleUtils.isComment(ruleText) || _.isEmpty(_.trim(ruleText))) {
|
|
237
459
|
return true;
|
|
238
460
|
}
|
|
239
461
|
|
|
240
462
|
if (ruleUtils.isEtcHostsRule(ruleText)) {
|
|
241
|
-
return validEtcHostsRule(ruleText, allowedIP);
|
|
463
|
+
return validEtcHostsRule(ruleText, allowedIP, allowPublicSuffix);
|
|
242
464
|
}
|
|
243
465
|
|
|
244
|
-
return validAdblockRule(ruleText, allowedIP);
|
|
466
|
+
return validAdblockRule(ruleText, allowedIP, allowPublicSuffix);
|
|
245
467
|
}
|
|
246
468
|
|
|
247
469
|
/**
|
|
@@ -251,8 +473,9 @@ class Validator {
|
|
|
251
473
|
/**
|
|
252
474
|
* Creates a new rule validator.
|
|
253
475
|
* @param {boolean} allowedIP - Flag indicating whether IP addresses should be considered valid.
|
|
476
|
+
* @param {boolean} allowPublicSuffix - Flag indicating whether rules matching whole public suffix are allowed.
|
|
254
477
|
*/
|
|
255
|
-
constructor(allowedIP) {
|
|
478
|
+
constructor(allowedIP, allowPublicSuffix) {
|
|
256
479
|
/**
|
|
257
480
|
* Indicates that a rule was previously removed (the iteration processed an invalid rule).
|
|
258
481
|
* Used to remove preceding comments or empty lines.
|
|
@@ -265,6 +488,12 @@ class Validator {
|
|
|
265
488
|
* @type {boolean}
|
|
266
489
|
*/
|
|
267
490
|
this.allowedIP = allowedIP;
|
|
491
|
+
|
|
492
|
+
/**
|
|
493
|
+
* Flag to allow or disallow validation when hostname equals public suffix.
|
|
494
|
+
* @type {boolean}
|
|
495
|
+
*/
|
|
496
|
+
this.allowPublicSuffix = allowPublicSuffix;
|
|
268
497
|
}
|
|
269
498
|
|
|
270
499
|
/**
|
|
@@ -279,7 +508,7 @@ class Validator {
|
|
|
279
508
|
// Iterate from the end to the beginning so we can remove
|
|
280
509
|
// preceding comments/empty lines if needed
|
|
281
510
|
for (let i = filtered.length - 1; i >= 0; i -= 1) {
|
|
282
|
-
const isValidRule = valid(filtered[i], this.allowedIP);
|
|
511
|
+
const isValidRule = valid(filtered[i], this.allowedIP, this.allowPublicSuffix);
|
|
283
512
|
const isCommentOrEmptyLine = ruleUtils.isComment(filtered[i]) || _.isEmpty(filtered[i]);
|
|
284
513
|
|
|
285
514
|
if (!isValidRule) {
|
|
@@ -304,12 +533,12 @@ class Validator {
|
|
|
304
533
|
* Validates a list of rules.
|
|
305
534
|
*
|
|
306
535
|
* @param {Array<string>} rules - The array of rules to validate.
|
|
307
|
-
* @param {boolean} allowedIP - flag to allow IP validation.
|
|
308
536
|
* @returns {Array<string>} The filtered list of valid rules.
|
|
309
537
|
*/
|
|
310
538
|
function validate(rules) {
|
|
311
|
-
const validator = new Validator(false);
|
|
539
|
+
const validator = new Validator(false, false);
|
|
312
540
|
// ip is not allowed for default validation
|
|
541
|
+
// public suffix matching is not allowed for default validation
|
|
313
542
|
return validator.validate(rules);
|
|
314
543
|
}
|
|
315
544
|
|
package/src/utils.js
CHANGED
|
@@ -1,5 +1,105 @@
|
|
|
1
1
|
const _ = require('lodash');
|
|
2
2
|
|
|
3
|
+
// Prefix for canonical IP/domain patterns
|
|
4
|
+
const IP_PREFIX = '||';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Regex that matches strings that look like adblock rule modifiers
|
|
8
|
+
* Used to distinguish a real `$modifier` suffix from a `$` that is
|
|
9
|
+
* part of a regex anchor.
|
|
10
|
+
*/
|
|
11
|
+
const MODIFIER_REGEX = /^[a-zA-Z0-9,=~_.-]+$/;
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Regex to match a valid IPv4 octet (0-255).
|
|
15
|
+
*/
|
|
16
|
+
const OCTET_REGEX = /^\d{1,3}$/;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Checks if a string is a valid IPv4 octet (0-255).
|
|
20
|
+
*
|
|
21
|
+
* @param {string} s - The string to check.
|
|
22
|
+
* @returns {boolean} True if valid octet.
|
|
23
|
+
*/
|
|
24
|
+
function isValidOctet(s) {
|
|
25
|
+
if (!OCTET_REGEX.test(s)) {
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
const num = Number(s);
|
|
29
|
+
return num >= 0 && num <= 255;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Parses an IP-like adblock pattern and extracts its structural components.
|
|
34
|
+
* Expects a pattern already stripped of @@ prefix and $modifier suffix
|
|
35
|
+
* (as returned by parseRuleTokens in rule.js).
|
|
36
|
+
*
|
|
37
|
+
* @param {string} pattern - The adblock pattern.
|
|
38
|
+
* @returns {{ prefix: string, octets: string[],
|
|
39
|
+
* hasTrailingDot: boolean, hasTrailingWildcard: boolean,
|
|
40
|
+
* hasCaret: boolean, hasCaretPipe: boolean }|null}
|
|
41
|
+
* Parsed components, or null if the pattern is not a valid IP pattern.
|
|
42
|
+
*/
|
|
43
|
+
function parseIpPattern(pattern) {
|
|
44
|
+
let remaining = pattern;
|
|
45
|
+
let prefix = '';
|
|
46
|
+
|
|
47
|
+
// Extract prefix
|
|
48
|
+
if (remaining.startsWith(IP_PREFIX)) {
|
|
49
|
+
prefix = IP_PREFIX;
|
|
50
|
+
remaining = remaining.slice(2);
|
|
51
|
+
} else if (remaining.startsWith('|')) {
|
|
52
|
+
prefix = '|';
|
|
53
|
+
remaining = remaining.slice(1);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// Check for ^| or ^ suffix
|
|
57
|
+
let hasCaret = false;
|
|
58
|
+
let hasCaretPipe = false;
|
|
59
|
+
if (remaining.endsWith('^|')) {
|
|
60
|
+
hasCaretPipe = true;
|
|
61
|
+
hasCaret = true;
|
|
62
|
+
remaining = remaining.slice(0, -2);
|
|
63
|
+
} else if (remaining.endsWith('^')) {
|
|
64
|
+
hasCaret = true;
|
|
65
|
+
remaining = remaining.slice(0, -1);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// Check for trailing wildcard (.*)
|
|
69
|
+
let hasTrailingWildcard = false;
|
|
70
|
+
if (remaining.endsWith('.*')) {
|
|
71
|
+
hasTrailingWildcard = true;
|
|
72
|
+
remaining = remaining.slice(0, -2);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Check for trailing dot
|
|
76
|
+
let hasTrailingDot = false;
|
|
77
|
+
if (remaining.endsWith('.')) {
|
|
78
|
+
hasTrailingDot = true;
|
|
79
|
+
remaining = remaining.slice(0, -1);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Split into octets
|
|
83
|
+
const parts = remaining.split('.');
|
|
84
|
+
if (parts.length === 0 || parts.length > 4) {
|
|
85
|
+
return null;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// All parts must be valid octets
|
|
89
|
+
if (!parts.every(isValidOctet)) {
|
|
90
|
+
return null;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
return {
|
|
94
|
+
prefix,
|
|
95
|
+
octets: parts,
|
|
96
|
+
hasTrailingDot,
|
|
97
|
+
hasTrailingWildcard,
|
|
98
|
+
hasCaret,
|
|
99
|
+
hasCaretPipe,
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
3
103
|
/**
|
|
4
104
|
* Extracts a substring between two tags.
|
|
5
105
|
*
|
|
@@ -136,7 +236,44 @@ class Wildcard {
|
|
|
136
236
|
}
|
|
137
237
|
}
|
|
138
238
|
|
|
239
|
+
/**
|
|
240
|
+
* Classifies an IP-like adblock pattern into categories used by validators and normalizers.
|
|
241
|
+
* Returns null when the pattern cannot be parsed as an IP-like pattern.
|
|
242
|
+
*
|
|
243
|
+
* @param {string} pattern - The adblock pattern to classify.
|
|
244
|
+
* @returns {{ prefix: string, octets: string[],
|
|
245
|
+
* hasTrailingDot: boolean, hasTrailingWildcard: boolean,
|
|
246
|
+
* hasCaret: boolean, hasCaretPipe: boolean,
|
|
247
|
+
* octetCount: number, isFullIp: boolean,
|
|
248
|
+
* isSubnetWildcard: boolean, isAmbiguous3Octet: boolean,
|
|
249
|
+
* isTooWide: boolean }|null}
|
|
250
|
+
*/
|
|
251
|
+
function classifyIpPattern(pattern) {
|
|
252
|
+
const parsed = parseIpPattern(pattern);
|
|
253
|
+
if (!parsed) {
|
|
254
|
+
return null;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
return {
|
|
258
|
+
...parsed,
|
|
259
|
+
octetCount: parsed.octets.length,
|
|
260
|
+
isFullIp: parsed.octets.length === 4
|
|
261
|
+
&& !parsed.hasTrailingDot
|
|
262
|
+
&& !parsed.hasTrailingWildcard,
|
|
263
|
+
isSubnetWildcard: parsed.octets.length < 4
|
|
264
|
+
&& (parsed.hasTrailingDot || parsed.hasTrailingWildcard),
|
|
265
|
+
isAmbiguous3Octet: parsed.octets.length === 3
|
|
266
|
+
&& !parsed.hasTrailingDot
|
|
267
|
+
&& !parsed.hasTrailingWildcard
|
|
268
|
+
&& !parsed.hasCaret,
|
|
269
|
+
isTooWide: parsed.octets.length <= 2,
|
|
270
|
+
};
|
|
271
|
+
}
|
|
272
|
+
|
|
139
273
|
module.exports = {
|
|
274
|
+
MODIFIER_REGEX,
|
|
275
|
+
parseIpPattern,
|
|
276
|
+
classifyIpPattern,
|
|
140
277
|
Wildcard,
|
|
141
278
|
splitByDelimiterWithEscapeCharacter,
|
|
142
279
|
substringBetween,
|
|
@@ -71,4 +71,74 @@ describe('Configuration', () => {
|
|
|
71
71
|
expect(ret.valid).toBe(false);
|
|
72
72
|
expect(ret.errorsText).toBeTruthy();
|
|
73
73
|
});
|
|
74
|
+
|
|
75
|
+
it('test ValidateAllowPublicSuffix transformation in configuration', () => {
|
|
76
|
+
const ret = config.validateConfiguration({
|
|
77
|
+
name: 'test',
|
|
78
|
+
sources: [
|
|
79
|
+
{
|
|
80
|
+
source: 'test.txt',
|
|
81
|
+
transformations: [
|
|
82
|
+
'ValidateAllowPublicSuffix',
|
|
83
|
+
],
|
|
84
|
+
},
|
|
85
|
+
],
|
|
86
|
+
transformations: [
|
|
87
|
+
'ValidateAllowPublicSuffix',
|
|
88
|
+
],
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
expect(ret.valid).toBe(true);
|
|
92
|
+
expect(ret.errorsText).toBeNull();
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
it.each([
|
|
96
|
+
['Validate', 'ValidateAllowIp'],
|
|
97
|
+
['Validate', 'ValidateAllowPublicSuffix'],
|
|
98
|
+
['Validate', 'ValidateAllowIpAndPublicSuffix'],
|
|
99
|
+
['ValidateAllowIp', 'ValidateAllowPublicSuffix'],
|
|
100
|
+
['ValidateAllowIp', 'ValidateAllowIpAndPublicSuffix'],
|
|
101
|
+
['ValidateAllowPublicSuffix', 'ValidateAllowIpAndPublicSuffix'],
|
|
102
|
+
])('test incompatible top-level transformations configuration: %s + %s', (first, second) => {
|
|
103
|
+
const ret = config.validateConfiguration({
|
|
104
|
+
name: 'test',
|
|
105
|
+
sources: [
|
|
106
|
+
{
|
|
107
|
+
source: 'test.txt',
|
|
108
|
+
},
|
|
109
|
+
],
|
|
110
|
+
transformations: [
|
|
111
|
+
first,
|
|
112
|
+
second,
|
|
113
|
+
],
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
expect(ret.valid).toBe(false);
|
|
117
|
+
expect(ret.errorsText).toBeTruthy();
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
it.each([
|
|
121
|
+
['Validate', 'ValidateAllowIp'],
|
|
122
|
+
['Validate', 'ValidateAllowPublicSuffix'],
|
|
123
|
+
['Validate', 'ValidateAllowIpAndPublicSuffix'],
|
|
124
|
+
['ValidateAllowIp', 'ValidateAllowPublicSuffix'],
|
|
125
|
+
['ValidateAllowIp', 'ValidateAllowIpAndPublicSuffix'],
|
|
126
|
+
['ValidateAllowPublicSuffix', 'ValidateAllowIpAndPublicSuffix'],
|
|
127
|
+
])('test incompatible source transformations configuration: %s + %s', (first, second) => {
|
|
128
|
+
const ret = config.validateConfiguration({
|
|
129
|
+
name: 'test',
|
|
130
|
+
sources: [
|
|
131
|
+
{
|
|
132
|
+
source: 'test.txt',
|
|
133
|
+
transformations: [
|
|
134
|
+
first,
|
|
135
|
+
second,
|
|
136
|
+
],
|
|
137
|
+
},
|
|
138
|
+
],
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
expect(ret.valid).toBe(false);
|
|
142
|
+
expect(ret.errorsText).toBeTruthy();
|
|
143
|
+
});
|
|
74
144
|
});
|