@adguard/hostlist-compiler 1.0.26 → 1.0.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md ADDED
@@ -0,0 +1,24 @@
1
+ # Hostlist Compiler Changelog
2
+
3
+ All notable changes to this project will be documented in this file.
4
+
5
+ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
+ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
+
8
+
9
+ ## [1.0.28] - 2024-09-25
10
+
11
+ ### Fix
12
+
13
+ - add `ValidateAllowIp` to validate configuration schema [#69]
14
+
15
+ [1.0.28]: https://github.com/AdguardTeam/HostlistCompiler/compare/v1.0.27...v1.0.28
16
+
17
+ ## [1.0.27] - 2024-09-25
18
+
19
+ ### Added
20
+
21
+ - `ValidateAllowIp` transformation option [#69]
22
+
23
+ [1.0.27]: https://github.com/AdguardTeam/HostlistCompiler/compare/v1.0.26...v1.0.27
24
+ [#69]: https://github.com/AdguardTeam/FiltersCompiler/issues/69
package/README.md CHANGED
@@ -13,6 +13,7 @@ This is a simple tool that makes it easier to compile a [hosts blocklist](https:
13
13
  - [Compress](#compress)
14
14
  - [RemoveModifiers](#remove-modifiers)
15
15
  - [Validate](#validate)
16
+ - [ValidateAllowIp](#validate-allow-ip)
16
17
  - [Deduplicate](#deduplicate)
17
18
  - [InvertAllow](#invertallow)
18
19
  - [RemoveEmptyLines](#removeemptylines)
@@ -255,14 +256,15 @@ import { writeFileSync } from 'fs';
255
256
  Here is the full list of transformations that are available:
256
257
 
257
258
  1. `RemoveComments`
258
- 2. `Compress`
259
- 3. `RemoveModifiers`
260
- 4. `Validate`
261
- 5. `Deduplicate`
262
- 6. `InvertAllow`
263
- 7. `RemoveEmptyLines`
264
- 8. `TrimLines`
265
- 9. `InsertFinalNewLine`
259
+ 1. `Compress`
260
+ 1. `RemoveModifiers`
261
+ 1. `Validate`
262
+ 1. `ValidateAllowIp`
263
+ 1. `Deduplicate`
264
+ 1. `InvertAllow`
265
+ 1. `RemoveEmptyLines`
266
+ 1. `TrimLines`
267
+ 1. `InsertFinalNewLine`
266
268
 
267
269
  Please note that these transformations are are always applied in the order specified here.
268
270
 
@@ -306,9 +308,14 @@ So here's what it does:
306
308
  - Discards domain-specific rules (e.g. `||example.org^$domain=example.com`). You don't want to have domain-specific rules working globally.
307
309
  - Discards rules with unsupported modifiers. [Click here](https://github.com/AdguardTeam/AdGuardHome/wiki/Hosts-Blocklists#-adblock-style-syntax) to learn more about which modifiers are supported.
308
310
  - Discards rules that are too short.
311
+ - Discards IP addresses. If you need to keep IP addresses, use [ValidateAllowIp](#validate-allow-ip) instead.
309
312
 
310
313
  If there are comments preceding the invalid rule, they will be removed as well.
311
314
 
315
+ ### <a name="validate-allow-ip"></a> ValidateAllowIp
316
+
317
+ This transformation exactly repeats the behavior of [Validate](#validate), but leaves the IP addresses in the lists.
318
+
312
319
  ### <a name="deduplicate"></a> Deduplicate
313
320
 
314
321
  This transformation simply removes the duplicates from the specified source.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@adguard/hostlist-compiler",
3
- "version": "1.0.26",
3
+ "version": "1.0.28",
4
4
  "description": "A simple tool that compiles hosts blocklists from multiple sources",
5
5
  "main": "src/index.js",
6
6
  "types": "src/index.d.ts",
@@ -44,6 +44,7 @@
44
44
  "RemoveModifiers",
45
45
  "Compress",
46
46
  "Validate",
47
+ "ValidateAllowIp",
47
48
  "Deduplicate",
48
49
  "InvertAllow",
49
50
  "RemoveEmptyLines",
@@ -119,6 +120,7 @@
119
120
  "RemoveModifiers",
120
121
  "Compress",
121
122
  "Validate",
123
+ "ValidateAllowIp",
122
124
  "Deduplicate",
123
125
  "InvertAllow",
124
126
  "RemoveEmptyLines",
@@ -1,6 +1,7 @@
1
1
  const removeComments = require('./remove-comments');
2
2
  const removeModifiers = require('./remove-modifiers');
3
- const validate = require('./validate');
3
+ const { validate } = require('./validate');
4
+ const { validateAllowIp } = require('./validate-allow-ip');
4
5
  const exclude = require('./exclude');
5
6
  const include = require('./include');
6
7
  const deduplicate = require('./deduplicate');
@@ -18,6 +19,7 @@ const TRANSFORMATIONS = Object.freeze({
18
19
  Compress: 'Compress',
19
20
  RemoveModifiers: 'RemoveModifiers',
20
21
  Validate: 'Validate',
22
+ ValidateAllowIp: 'ValidateAllowIp',
21
23
  Deduplicate: 'Deduplicate',
22
24
  InvertAllow: 'InvertAllow',
23
25
  RemoveEmptyLines: 'RemoveEmptyLines',
@@ -71,6 +73,9 @@ async function transform(rules, configuration, transformations) {
71
73
  if (transformations.indexOf(TRANSFORMATIONS.Validate) !== -1) {
72
74
  transformed = validate(transformed);
73
75
  }
76
+ if (transformations.indexOf(TRANSFORMATIONS.ValidateAllowIp) !== -1) {
77
+ transformed = validateAllowIp(transformed);
78
+ }
74
79
  if (transformations.indexOf(TRANSFORMATIONS.Deduplicate) !== -1) {
75
80
  transformed = deduplicate(transformed);
76
81
  }
@@ -0,0 +1,19 @@
1
+ const { Validator } = require('./validate');
2
+
3
+ /**
4
+ * Validates a list of rules and allows IP validation.
5
+ *
6
+ * This function creates an instance of the `Validator` class, enabling IP address validation,
7
+ * and returns a filtered list of valid rules.
8
+ *
9
+ * @param {Array<string>} rules - The array of rules to validate.
10
+ * @returns {Array<string>} - The filtered array of valid rules, allowing IP addresses.
11
+ */
12
+ function validateAllowIp(rules) {
13
+ // ip is not allowed by default
14
+ const validator = new Validator(true);
15
+ return validator.validate(rules);
16
+ }
17
+ module.exports = {
18
+ validateAllowIp,
19
+ };
@@ -28,14 +28,20 @@ const MAX_PATTERN_LENGTH = 5;
28
28
  /**
29
29
  * Checks if the specified hostname is valid for a blocklist.
30
30
  *
31
- * @param {String} hostname - hostname to check
32
- * @param {String} ruleText - original rule text (for logging)
33
- * @returns {Boolean} true if the hostname is okay to be in the blocklist.
31
+ * @param {string} hostname - hostname to check
32
+ * @param {string} ruleText - original rule text (for logging)
33
+ * @param {boolean} allowedIP - flag to determine if IP validation is allowed
34
+ * @returns {boolean} true if the hostname is okay to be in the blocklist.
34
35
  */
35
- function validHostname(hostname, ruleText) {
36
+ function validHostname(hostname, ruleText, allowedIP) {
36
37
  const result = tldts.parse(hostname);
37
38
 
38
- if (!result.hostname || result.isIp) {
39
+ if (!result.hostname) {
40
+ consola.debug(`invalid hostname ${hostname} in the rule: ${ruleText}`);
41
+ return false;
42
+ }
43
+
44
+ if (!allowedIP && result.isIp) {
39
45
  consola.debug(`invalid hostname ${hostname} in the rule: ${ruleText}`);
40
46
  return false;
41
47
  }
@@ -56,10 +62,11 @@ function validHostname(hostname, ruleText) {
56
62
  * 2. Prohibit rules that block the whole public suffix
57
63
  * 3. Prohibit rules that contain invalid domain names
58
64
  *
59
- * @param {String} ruleText - rule text
60
- * @returns {Boolean} true if the rule is a valid /etc/hosts rule
65
+ * @param {string} ruleText - rule text
66
+ * @param {boolean} allowIP - flag to determine if IP validation is allowed
67
+ * @returns {boolean} true if the rule is a valid /etc/hosts rule
61
68
  */
62
- function validEtcHostsRule(ruleText) {
69
+ function validEtcHostsRule(ruleText, allowIP) {
63
70
  let props;
64
71
  try {
65
72
  props = ruleUtils.loadEtcHostsRuleProperties(ruleText);
@@ -72,7 +79,7 @@ function validEtcHostsRule(ruleText) {
72
79
  return false;
73
80
  }
74
81
 
75
- if (props.hostnames.some((h) => !validHostname(h, ruleText))) {
82
+ if (props.hostnames.some((h) => !validHostname(h, ruleText, allowIP))) {
76
83
  return false;
77
84
  }
78
85
 
@@ -88,10 +95,11 @@ function validEtcHostsRule(ruleText) {
88
95
  * 4. For domain-blocking rules like ||domain^ it checks that the domain is
89
96
  * valid and does not block too much.
90
97
  *
91
- * @param {String} ruleText - rule text
92
- * @returns {Boolean} - adblock-style rule
98
+ * @param {string} ruleText - rule text
99
+ * @param {boolean} allowedIP - flag to determine if IP validation is allowed
100
+ * @returns {boolean} - adblock-style rule
93
101
  */
94
- function validAdblockRule(ruleText) {
102
+ function validAdblockRule(ruleText, allowedIP) {
95
103
  let props;
96
104
  try {
97
105
  props = ruleUtils.loadAdblockRuleProperties(ruleText);
@@ -154,7 +162,7 @@ function validAdblockRule(ruleText) {
154
162
  && sepIdx !== -1
155
163
  && wildcardIdx === -1) {
156
164
  const hostname = utils.substringBetween(ruleText, '||', '^');
157
- if (!validHostname(hostname, ruleText)) {
165
+ if (!validHostname(hostname, ruleText, allowedIP)) {
158
166
  return false;
159
167
  }
160
168
 
@@ -172,53 +180,89 @@ function validAdblockRule(ruleText) {
172
180
  /**
173
181
  * Validates the rule.
174
182
  *
175
- * Emptry strings and comments are considered valid.
183
+ * Empty strings and comments are considered valid.
176
184
  *
177
- * For /etc/hosts rules: {@see validEtcHostsRule}
178
- * For adblock-style rules: {@see validAdblockRule}
185
+ * For /etc/hosts rules: @see {@link validEtcHostsRule}
186
+ * For adblock-style rules: @see {@link validAdblockRule}
179
187
  *
180
- * @param {String} ruleText - rule to check
181
- * @returns {Boolean} true if rule is a comment
188
+ * @param {string} ruleText - rule to check
189
+ * @param {boolean} allowedIP - flag to determine if IP validation is allowed
190
+ * @returns {boolean} - true if the rule is valid, false otherwise
182
191
  */
183
- function valid(ruleText) {
192
+ function valid(ruleText, allowedIP) {
184
193
  if (ruleUtils.isComment(ruleText) || _.isEmpty(_.trim(ruleText))) {
185
194
  return true;
186
195
  }
187
196
 
188
197
  if (ruleUtils.isEtcHostsRule(ruleText)) {
189
- return validEtcHostsRule(ruleText);
198
+ return validEtcHostsRule(ruleText, allowedIP);
190
199
  }
191
- return validAdblockRule(ruleText);
200
+ return validAdblockRule(ruleText, allowedIP);
192
201
  }
193
202
 
203
+ /**
204
+ * Class representing a validator for a list of rules.
205
+ */
206
+ class Validator {
207
+ constructor(allowedIP) {
208
+ /**
209
+ * Indicates whether the previous rule was removed.
210
+ * @type {boolean}
211
+ */
212
+ this.prevRuleRemoved = false;
213
+ /**
214
+ * Flag to allow IP validation.
215
+ * @type {boolean}
216
+ */
217
+ this.allowedIP = allowedIP;
218
+ }
219
+
220
+ /**
221
+ * Validates the list of rules and removes invalid rules.
222
+ * If a rule is invalid, any preceding comments or empty lines are also removed.
223
+ *
224
+ * @returns {Array<string>} The filtered list of valid rules.
225
+ * @param {Array<string>} rules - An array of rules to validate.
226
+ */
227
+ validate(rules) {
228
+ /**
229
+ * A filtered list of rules after validation.
230
+ * @type {Array<string>}
231
+ */
232
+ const filtered = [...rules];
233
+ for (let iFiltered = filtered.length - 1; iFiltered >= 0; iFiltered -= 1) {
234
+ const ruleText = filtered[iFiltered];
235
+
236
+ if (!valid(ruleText, this.allowedIP)) {
237
+ this.prevRuleRemoved = true;
238
+ filtered.splice(iFiltered, 1);
239
+ } else if (this.prevRuleRemoved && (ruleUtils.isComment(ruleText) || _.isEmpty(ruleText))) {
240
+ // Remove preceding comments and empty lines
241
+ consola.debug(`Removing a comment preceding invalid rule: ${ruleText}`);
242
+ filtered.splice(iFiltered, 1);
243
+ } else {
244
+ // Stop removing comments
245
+ this.prevRuleRemoved = false;
246
+ }
247
+ }
248
+
249
+ return filtered;
250
+ }
251
+ }
194
252
  /**
195
253
  * Validates a list of rules.
196
254
  *
197
- * @param {Array<string>} rules - an array of rules to validate
198
- * @returns {Array<string>} an array of rules without invalid
255
+ * @param {Array<string>} rules - The array of rules to validate.
256
+ * @param {boolean} allowedIP - flag to allow IP validation.
257
+ * @returns {Array<string>} The filtered list of valid rules.
199
258
  */
200
259
  function validate(rules) {
201
- // Clone the original array before modifying it
202
- const filtered = [...rules];
203
- let prevRuleRemoved = false;
204
-
205
- for (let iFiltered = filtered.length - 1; iFiltered >= 0; iFiltered -= 1) {
206
- const ruleText = filtered[iFiltered];
207
-
208
- if (!valid(ruleText)) {
209
- prevRuleRemoved = true;
210
- filtered.splice(iFiltered, 1);
211
- } else if (prevRuleRemoved && (ruleUtils.isComment(ruleText) || _.isEmpty(ruleText))) {
212
- // Remove preceding comments and empty lines
213
- consola.debug(`Removing a comment preceding invalid rule: ${ruleText}`);
214
- filtered.splice(iFiltered, 1);
215
- } else {
216
- // Stop removing comments
217
- prevRuleRemoved = false;
218
- }
219
- }
220
-
221
- return filtered;
260
+ const validator = new Validator(false);
261
+ // ip is not allowed for default validation
262
+ return validator.validate(rules);
222
263
  }
223
264
 
224
- module.exports = validate;
265
+ module.exports = {
266
+ validate,
267
+ Validator,
268
+ };
@@ -35,4 +35,24 @@ dupl1
35
35
  expect(filtered).toHaveLength(2);
36
36
  expect(filtered).toEqual(['rule1', 'dupl1']);
37
37
  });
38
+ it('simple transformations with ValidateAllowIp', async () => {
39
+ const rules = `! test comment
40
+ rule1
41
+ rule2
42
+ ! dup1 comment
43
+ dupl1
44
+ dupl1
45
+ ||185.149.120.173^
46
+ # another comment`.split(/\r?\n/);
47
+ const exclusions = [
48
+ 'rule2',
49
+ '', // empty exclusions are ignored
50
+ ];
51
+ const configuration = {
52
+ exclusions,
53
+ };
54
+ const filtered = await transform(rules, configuration, ['RemoveComments', 'ValidateAllowIp', 'Deduplicate']);
55
+ // expect(filtered).toHaveLength(3);
56
+ expect(filtered).toEqual(['rule1', 'dupl1', '||185.149.120.173^']);
57
+ });
38
58
  });
@@ -0,0 +1,71 @@
1
+ const { validateAllowIp } = require('../../src/transformations/validate-allow-ip');
2
+
3
+ describe('validateAllowIp', () => {
4
+ it('simple /etc/hosts rule', () => {
5
+ const rules = '0.0.0.0 example.org'.split(/\r?\n/);
6
+ const filtered = validateAllowIp(rules);
7
+
8
+ expect(filtered).toHaveLength(1);
9
+ expect(filtered).toContain('0.0.0.0 example.org');
10
+ });
11
+
12
+ it('/etc/hosts rules', () => {
13
+ const rules = `0.0.0.0 example.org
14
+ 0.0.0.0 co.uk
15
+ 0.0.0.0 doubleclick.net doubleclick.com`.split(/\r?\n/);
16
+ const filtered = validateAllowIp(rules);
17
+
18
+ expect(filtered).toHaveLength(2);
19
+ expect(filtered).toContain('0.0.0.0 example.org');
20
+ expect(filtered).toContain('0.0.0.0 doubleclick.net doubleclick.com');
21
+ });
22
+
23
+ it('remove preceding comments', () => {
24
+ const rules = `! rule comment
25
+
26
+ ||invalid/rule
27
+ ! comment
28
+ ||valid.com^`.split(/\r?\n/);
29
+ const filtered = validateAllowIp(rules);
30
+
31
+ expect(filtered).toEqual([
32
+ '! comment',
33
+ '||valid.com^',
34
+ ]);
35
+ });
36
+
37
+ it('adblock-style rules', () => {
38
+ const rules = `! here goes a comment
39
+
40
+ ||example.org^
41
+ ||185.149.120.173^
42
+ ! invalid rule comment will be removed
43
+ ||example.com/atata
44
+ ||ex*.org^
45
+ ||org^
46
+ ||example.org^$third-party
47
+ ||example.org^$important
48
+ ||*.ga^$denyallow=example1.ga|example2.ga
49
+ ://ww4.$denyallow=ww4.example.com
50
+ ://example.org
51
+ ||example.org^|
52
+ @@||example.org^|$important
53
+ @@||example.com^*-tracking.js
54
+ @@||example.com^-tracking.js`.split(/\r?\n/);
55
+ const filtered = validateAllowIp(rules);
56
+
57
+ expect(filtered).toEqual([
58
+ '! here goes a comment',
59
+ '',
60
+ '||example.org^',
61
+ '||185.149.120.173^', // valid because IP addresses is allowed
62
+ '||ex*.org^', // valid because contains special characters
63
+ '||example.org^$important',
64
+ '||*.ga^$denyallow=example1.ga|example2.ga',
65
+ '://ww4.$denyallow=ww4.example.com',
66
+ '://example.org',
67
+ '||example.org^|',
68
+ '@@||example.org^|$important',
69
+ ]);
70
+ });
71
+ });
@@ -1,4 +1,4 @@
1
- const validate = require('../../src/transformations/validate');
1
+ const { validate } = require('../../src/transformations/validate');
2
2
 
3
3
  describe('Validate', () => {
4
4
  it('simple /etc/hosts rule', () => {
@@ -38,6 +38,7 @@ describe('Validate', () => {
38
38
  const rules = `! here goes a comment
39
39
 
40
40
  ||example.org^
41
+ ||185.149.120.173^
41
42
  ! invalid rule comment will be removed
42
43
  ||example.com/atata
43
44
  ||ex*.org^