@adguard/hostlist-compiler 1.0.24 → 1.0.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md ADDED
@@ -0,0 +1,16 @@
1
+ # Hostlist Compiler Changelog
2
+
3
+ All notable changes to this project will be documented in this file.
4
+
5
+ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
+ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
+
8
+
9
+ ## [1.0.27] - 2024-09-25
10
+
11
+ ### Added
12
+
13
+ - `ValidateAllowIp` transformation option [#69]
14
+
15
+ [1.0.27]: https://github.com/AdguardTeam/HostlistCompiler/compare/v1.0.26...v1.0.27
16
+ [#69]: https://github.com/AdguardTeam/FiltersCompiler/issues/69
package/README.md CHANGED
@@ -13,6 +13,7 @@ This is a simple tool that makes it easier to compile a [hosts blocklist](https:
13
13
  - [Compress](#compress)
14
14
  - [RemoveModifiers](#remove-modifiers)
15
15
  - [Validate](#validate)
16
+ - [ValidateAllowIp](#validate-allow-ip)
16
17
  - [Deduplicate](#deduplicate)
17
18
  - [InvertAllow](#invertallow)
18
19
  - [RemoveEmptyLines](#removeemptylines)
@@ -81,6 +82,7 @@ Here is an example of this configuration:
81
82
  "description": "List description",
82
83
  "homepage": "https://example.org/",
83
84
  "license": "GPLv3",
85
+ "version": "1.0.0.0",
84
86
  "sources": [
85
87
  {
86
88
  "name": "Local rules",
@@ -111,6 +113,7 @@ Here is an example of this configuration:
111
113
  - `description` - (optional) the list description.
112
114
  - `homepage` - (optional) URL to the list homepage.
113
115
  - `license` - (optional) Filter list license.
116
+ - `version` - (optional) Filter list version.
114
117
  - `sources` - (mandatory) array of the list sources.
115
118
  - `.source` - (mandatory) path or URL of the source. It can be a traditional filter list or a hosts file.
116
119
  - `.name` - (optional) name of the source.
@@ -253,14 +256,15 @@ import { writeFileSync } from 'fs';
253
256
  Here is the full list of transformations that are available:
254
257
 
255
258
  1. `RemoveComments`
256
- 2. `Compress`
257
- 3. `RemoveModifiers`
258
- 4. `Validate`
259
- 5. `Deduplicate`
260
- 6. `InvertAllow`
261
- 7. `RemoveEmptyLines`
262
- 8. `TrimLines`
263
- 9. `InsertFinalNewLine`
259
+ 1. `Compress`
260
+ 1. `RemoveModifiers`
261
+ 1. `Validate`
262
+ 1. `ValidateAllowIp`
263
+ 1. `Deduplicate`
264
+ 1. `InvertAllow`
265
+ 1. `RemoveEmptyLines`
266
+ 1. `TrimLines`
267
+ 1. `InsertFinalNewLine`
264
268
 
265
269
  Please note that these transformations are are always applied in the order specified here.
266
270
 
@@ -270,7 +274,8 @@ This is a very simple transformation that simply removes comments (e.g. all rule
270
274
 
271
275
  ### <a name="compress"></a> Compress
272
276
 
273
- > **IMPORTANT:** this transformation converts `hosts` lists into `adblock` lists.
277
+ > [!IMPORTANT]
278
+ > This transformation converts `hosts` lists into `adblock` lists.
274
279
 
275
280
  Here's what it does:
276
281
 
@@ -288,7 +293,9 @@ Here is the list of modifiers that will be removed:
288
293
  - `$all` modifier
289
294
  - `$popup` modifier
290
295
 
291
- **IMPORTANT:** please, be cautious with this. Blindly removing `$third-party` from traditional ad blocking rules leads to lots of false-positives. This is exactly why there is an option to exclude rules - you may need to use it.
296
+ > [!CAUTION]
297
+ > Blindly removing `$third-party` from traditional ad blocking rules leads to lots of false-positives.
298
+ >> This is exactly why there is an option to exclude rules - you may need to use it.
292
299
 
293
300
  ### <a name="validate"></a> Validate
294
301
 
@@ -301,9 +308,14 @@ So here's what it does:
301
308
  - Discards domain-specific rules (e.g. `||example.org^$domain=example.com`). You don't want to have domain-specific rules working globally.
302
309
  - Discards rules with unsupported modifiers. [Click here](https://github.com/AdguardTeam/AdGuardHome/wiki/Hosts-Blocklists#-adblock-style-syntax) to learn more about which modifiers are supported.
303
310
  - Discards rules that are too short.
311
+ - Discards IP addresses. If you need to keep IP addresses, use [ValidateAllowIp](#validate-allow-ip) instead.
304
312
 
305
313
  If there are comments preceding the invalid rule, they will be removed as well.
306
314
 
315
+ ### <a name="validate-allow-ip"></a> ValidateAllowIp
316
+
317
+ This transformation exactly repeats the behavior of [Validate](#validate), but leaves the IP addresses in the lists.
318
+
307
319
  ### <a name="deduplicate"></a> Deduplicate
308
320
 
309
321
  This transformation simply removes the duplicates from the specified source.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@adguard/hostlist-compiler",
3
- "version": "1.0.24",
3
+ "version": "1.0.27",
4
4
  "description": "A simple tool that compiles hosts blocklists from multiple sources",
5
5
  "main": "src/index.js",
6
6
  "types": "src/index.d.ts",
package/src/index.d.ts CHANGED
@@ -42,6 +42,8 @@ declare module '@adguard/hostlist-compiler' {
42
42
  homepage?: string;
43
43
  /** Filter list license */
44
44
  license?: string;
45
+ /** Filter list version */
46
+ version?: string;
45
47
  /** An array of the filter list sources */
46
48
  sources: ISource[];
47
49
  /** A list of the transformations that will be applied */
package/src/index.js CHANGED
@@ -21,6 +21,9 @@ function prepareHeader(configuration) {
21
21
  if (!_.isEmpty(configuration.description)) {
22
22
  lines.push(`! Description: ${configuration.description}`);
23
23
  }
24
+ if (!_.isEmpty(configuration.version)) {
25
+ lines.push(`! Version: ${configuration.version}`);
26
+ }
24
27
  if (!_.isEmpty(configuration.homepage)) {
25
28
  lines.push(`! Homepage: ${configuration.homepage}`);
26
29
  }
@@ -22,6 +22,10 @@
22
22
  "description": "Filter list license",
23
23
  "type": "string"
24
24
  },
25
+ "version": {
26
+ "description": "Filter version",
27
+ "type": "string"
28
+ },
25
29
  "sources": {
26
30
  "description": "An array of the filter list sources",
27
31
  "type": "array",
@@ -1,6 +1,7 @@
1
1
  const removeComments = require('./remove-comments');
2
2
  const removeModifiers = require('./remove-modifiers');
3
- const validate = require('./validate');
3
+ const { validate } = require('./validate');
4
+ const { validateAllowIp } = require('./validate-allow-ip');
4
5
  const exclude = require('./exclude');
5
6
  const include = require('./include');
6
7
  const deduplicate = require('./deduplicate');
@@ -18,6 +19,7 @@ const TRANSFORMATIONS = Object.freeze({
18
19
  Compress: 'Compress',
19
20
  RemoveModifiers: 'RemoveModifiers',
20
21
  Validate: 'Validate',
22
+ ValidateAllowIp: 'ValidateAllowIp',
21
23
  Deduplicate: 'Deduplicate',
22
24
  InvertAllow: 'InvertAllow',
23
25
  RemoveEmptyLines: 'RemoveEmptyLines',
@@ -71,6 +73,9 @@ async function transform(rules, configuration, transformations) {
71
73
  if (transformations.indexOf(TRANSFORMATIONS.Validate) !== -1) {
72
74
  transformed = validate(transformed);
73
75
  }
76
+ if (transformations.indexOf(TRANSFORMATIONS.ValidateAllowIp) !== -1) {
77
+ transformed = validateAllowIp(transformed);
78
+ }
74
79
  if (transformations.indexOf(TRANSFORMATIONS.Deduplicate) !== -1) {
75
80
  transformed = deduplicate(transformed);
76
81
  }
@@ -0,0 +1,19 @@
1
+ const { Validator } = require('./validate');
2
+
3
+ /**
4
+ * Validates a list of rules and allows IP validation.
5
+ *
6
+ * This function creates an instance of the `Validator` class, enabling IP address validation,
7
+ * and returns a filtered list of valid rules.
8
+ *
9
+ * @param {Array<string>} rules - The array of rules to validate.
10
+ * @returns {Array<string>} - The filtered array of valid rules, allowing IP addresses.
11
+ */
12
+ function validateAllowIp(rules) {
13
+ // ip is not allowed by default
14
+ const validator = new Validator(true);
15
+ return validator.validate(rules);
16
+ }
17
+ module.exports = {
18
+ validateAllowIp,
19
+ };
@@ -28,14 +28,20 @@ const MAX_PATTERN_LENGTH = 5;
28
28
  /**
29
29
  * Checks if the specified hostname is valid for a blocklist.
30
30
  *
31
- * @param {String} hostname - hostname to check
32
- * @param {String} ruleText - original rule text (for logging)
33
- * @returns {Boolean} true if the hostname is okay to be in the blocklist.
31
+ * @param {string} hostname - hostname to check
32
+ * @param {string} ruleText - original rule text (for logging)
33
+ * @param {boolean} allowedIP - flag to determine if IP validation is allowed
34
+ * @returns {boolean} true if the hostname is okay to be in the blocklist.
34
35
  */
35
- function validHostname(hostname, ruleText) {
36
+ function validHostname(hostname, ruleText, allowedIP) {
36
37
  const result = tldts.parse(hostname);
37
38
 
38
- if (!result.hostname || result.isIp) {
39
+ if (!result.hostname) {
40
+ consola.debug(`invalid hostname ${hostname} in the rule: ${ruleText}`);
41
+ return false;
42
+ }
43
+
44
+ if (!allowedIP && result.isIp) {
39
45
  consola.debug(`invalid hostname ${hostname} in the rule: ${ruleText}`);
40
46
  return false;
41
47
  }
@@ -56,10 +62,11 @@ function validHostname(hostname, ruleText) {
56
62
  * 2. Prohibit rules that block the whole public suffix
57
63
  * 3. Prohibit rules that contain invalid domain names
58
64
  *
59
- * @param {String} ruleText - rule text
60
- * @returns {Boolean} true if the rule is a valid /etc/hosts rule
65
+ * @param {string} ruleText - rule text
66
+ * @param {boolean} allowIP - flag to determine if IP validation is allowed
67
+ * @returns {boolean} true if the rule is a valid /etc/hosts rule
61
68
  */
62
- function validEtcHostsRule(ruleText) {
69
+ function validEtcHostsRule(ruleText, allowIP) {
63
70
  let props;
64
71
  try {
65
72
  props = ruleUtils.loadEtcHostsRuleProperties(ruleText);
@@ -72,7 +79,7 @@ function validEtcHostsRule(ruleText) {
72
79
  return false;
73
80
  }
74
81
 
75
- if (props.hostnames.some((h) => !validHostname(h, ruleText))) {
82
+ if (props.hostnames.some((h) => !validHostname(h, ruleText, allowIP))) {
76
83
  return false;
77
84
  }
78
85
 
@@ -88,10 +95,11 @@ function validEtcHostsRule(ruleText) {
88
95
  * 4. For domain-blocking rules like ||domain^ it checks that the domain is
89
96
  * valid and does not block too much.
90
97
  *
91
- * @param {String} ruleText - rule text
92
- * @returns {Boolean} - adblock-style rule
98
+ * @param {string} ruleText - rule text
99
+ * @param {boolean} allowedIP - flag to determine if IP validation is allowed
100
+ * @returns {boolean} - adblock-style rule
93
101
  */
94
- function validAdblockRule(ruleText) {
102
+ function validAdblockRule(ruleText, allowedIP) {
95
103
  let props;
96
104
  try {
97
105
  props = ruleUtils.loadAdblockRuleProperties(ruleText);
@@ -154,7 +162,7 @@ function validAdblockRule(ruleText) {
154
162
  && sepIdx !== -1
155
163
  && wildcardIdx === -1) {
156
164
  const hostname = utils.substringBetween(ruleText, '||', '^');
157
- if (!validHostname(hostname, ruleText)) {
165
+ if (!validHostname(hostname, ruleText, allowedIP)) {
158
166
  return false;
159
167
  }
160
168
 
@@ -172,53 +180,89 @@ function validAdblockRule(ruleText) {
172
180
  /**
173
181
  * Validates the rule.
174
182
  *
175
- * Emptry strings and comments are considered valid.
183
+ * Empty strings and comments are considered valid.
176
184
  *
177
- * For /etc/hosts rules: {@see validEtcHostsRule}
178
- * For adblock-style rules: {@see validAdblockRule}
185
+ * For /etc/hosts rules: @see {@link validEtcHostsRule}
186
+ * For adblock-style rules: @see {@link validAdblockRule}
179
187
  *
180
- * @param {String} ruleText - rule to check
181
- * @returns {Boolean} true if rule is a comment
188
+ * @param {string} ruleText - rule to check
189
+ * @param {boolean} allowedIP - flag to determine if IP validation is allowed
190
+ * @returns {boolean} - true if the rule is valid, false otherwise
182
191
  */
183
- function valid(ruleText) {
192
+ function valid(ruleText, allowedIP) {
184
193
  if (ruleUtils.isComment(ruleText) || _.isEmpty(_.trim(ruleText))) {
185
194
  return true;
186
195
  }
187
196
 
188
197
  if (ruleUtils.isEtcHostsRule(ruleText)) {
189
- return validEtcHostsRule(ruleText);
198
+ return validEtcHostsRule(ruleText, allowedIP);
190
199
  }
191
- return validAdblockRule(ruleText);
200
+ return validAdblockRule(ruleText, allowedIP);
192
201
  }
193
202
 
203
+ /**
204
+ * Class representing a validator for a list of rules.
205
+ */
206
+ class Validator {
207
+ constructor(allowedIP) {
208
+ /**
209
+ * Indicates whether the previous rule was removed.
210
+ * @type {boolean}
211
+ */
212
+ this.prevRuleRemoved = false;
213
+ /**
214
+ * Flag to allow IP validation.
215
+ * @type {boolean}
216
+ */
217
+ this.allowedIP = allowedIP;
218
+ }
219
+
220
+ /**
221
+ * Validates the list of rules and removes invalid rules.
222
+ * If a rule is invalid, any preceding comments or empty lines are also removed.
223
+ *
224
+ * @returns {Array<string>} The filtered list of valid rules.
225
+ * @param {Array<string>} rules - An array of rules to validate.
226
+ */
227
+ validate(rules) {
228
+ /**
229
+ * A filtered list of rules after validation.
230
+ * @type {Array<string>}
231
+ */
232
+ const filtered = [...rules];
233
+ for (let iFiltered = filtered.length - 1; iFiltered >= 0; iFiltered -= 1) {
234
+ const ruleText = filtered[iFiltered];
235
+
236
+ if (!valid(ruleText, this.allowedIP)) {
237
+ this.prevRuleRemoved = true;
238
+ filtered.splice(iFiltered, 1);
239
+ } else if (this.prevRuleRemoved && (ruleUtils.isComment(ruleText) || _.isEmpty(ruleText))) {
240
+ // Remove preceding comments and empty lines
241
+ consola.debug(`Removing a comment preceding invalid rule: ${ruleText}`);
242
+ filtered.splice(iFiltered, 1);
243
+ } else {
244
+ // Stop removing comments
245
+ this.prevRuleRemoved = false;
246
+ }
247
+ }
248
+
249
+ return filtered;
250
+ }
251
+ }
194
252
  /**
195
253
  * Validates a list of rules.
196
254
  *
197
- * @param {Array<string>} rules - an array of rules to validate
198
- * @returns {Array<string>} an array of rules without invalid
255
+ * @param {Array<string>} rules - The array of rules to validate.
256
+ * @param {boolean} allowedIP - flag to allow IP validation.
257
+ * @returns {Array<string>} The filtered list of valid rules.
199
258
  */
200
259
  function validate(rules) {
201
- // Clone the original array before modifying it
202
- const filtered = [...rules];
203
- let prevRuleRemoved = false;
204
-
205
- for (let iFiltered = filtered.length - 1; iFiltered >= 0; iFiltered -= 1) {
206
- const ruleText = filtered[iFiltered];
207
-
208
- if (!valid(ruleText)) {
209
- prevRuleRemoved = true;
210
- filtered.splice(iFiltered, 1);
211
- } else if (prevRuleRemoved && (ruleUtils.isComment(ruleText) || _.isEmpty(ruleText))) {
212
- // Remove preceding comments and empty lines
213
- consola.debug(`Removing a comment preceding invalid rule: ${ruleText}`);
214
- filtered.splice(iFiltered, 1);
215
- } else {
216
- // Stop removing comments
217
- prevRuleRemoved = false;
218
- }
219
- }
220
-
221
- return filtered;
260
+ const validator = new Validator(false);
261
+ // ip is not allowed for default validation
262
+ return validator.validate(rules);
222
263
  }
223
264
 
224
- module.exports = validate;
265
+ module.exports = {
266
+ validate,
267
+ Validator,
268
+ };
@@ -15,6 +15,7 @@ describe('Hostlist compiler', () => {
15
15
  const configuration = {
16
16
  name: 'Test filter',
17
17
  description: 'Our test filter',
18
+ version: '1.0.0.9',
18
19
  sources: [
19
20
  {
20
21
  name: 'source 1',
@@ -33,6 +34,7 @@ describe('Hostlist compiler', () => {
33
34
  // assert
34
35
  expect(list).toContain('||example.org');
35
36
  expect(list).toContain('||example.com');
37
+ expect(list).toContain('! Version: 1.0.0.9');
36
38
 
37
39
  const str = list.join('\n');
38
40
  consola.info(str);
@@ -35,4 +35,24 @@ dupl1
35
35
  expect(filtered).toHaveLength(2);
36
36
  expect(filtered).toEqual(['rule1', 'dupl1']);
37
37
  });
38
+ it('simple transformations with ValidateAllowIp', async () => {
39
+ const rules = `! test comment
40
+ rule1
41
+ rule2
42
+ ! dup1 comment
43
+ dupl1
44
+ dupl1
45
+ ||185.149.120.173^
46
+ # another comment`.split(/\r?\n/);
47
+ const exclusions = [
48
+ 'rule2',
49
+ '', // empty exclusions are ignored
50
+ ];
51
+ const configuration = {
52
+ exclusions,
53
+ };
54
+ const filtered = await transform(rules, configuration, ['RemoveComments', 'ValidateAllowIp', 'Deduplicate']);
55
+ // expect(filtered).toHaveLength(3);
56
+ expect(filtered).toEqual(['rule1', 'dupl1', '||185.149.120.173^']);
57
+ });
38
58
  });
@@ -0,0 +1,71 @@
1
+ const { validateAllowIp } = require('../../src/transformations/validate-allow-ip');
2
+
3
+ describe('validateAllowIp', () => {
4
+ it('simple /etc/hosts rule', () => {
5
+ const rules = '0.0.0.0 example.org'.split(/\r?\n/);
6
+ const filtered = validateAllowIp(rules);
7
+
8
+ expect(filtered).toHaveLength(1);
9
+ expect(filtered).toContain('0.0.0.0 example.org');
10
+ });
11
+
12
+ it('/etc/hosts rules', () => {
13
+ const rules = `0.0.0.0 example.org
14
+ 0.0.0.0 co.uk
15
+ 0.0.0.0 doubleclick.net doubleclick.com`.split(/\r?\n/);
16
+ const filtered = validateAllowIp(rules);
17
+
18
+ expect(filtered).toHaveLength(2);
19
+ expect(filtered).toContain('0.0.0.0 example.org');
20
+ expect(filtered).toContain('0.0.0.0 doubleclick.net doubleclick.com');
21
+ });
22
+
23
+ it('remove preceding comments', () => {
24
+ const rules = `! rule comment
25
+
26
+ ||invalid/rule
27
+ ! comment
28
+ ||valid.com^`.split(/\r?\n/);
29
+ const filtered = validateAllowIp(rules);
30
+
31
+ expect(filtered).toEqual([
32
+ '! comment',
33
+ '||valid.com^',
34
+ ]);
35
+ });
36
+
37
+ it('adblock-style rules', () => {
38
+ const rules = `! here goes a comment
39
+
40
+ ||example.org^
41
+ ||185.149.120.173^
42
+ ! invalid rule comment will be removed
43
+ ||example.com/atata
44
+ ||ex*.org^
45
+ ||org^
46
+ ||example.org^$third-party
47
+ ||example.org^$important
48
+ ||*.ga^$denyallow=example1.ga|example2.ga
49
+ ://ww4.$denyallow=ww4.example.com
50
+ ://example.org
51
+ ||example.org^|
52
+ @@||example.org^|$important
53
+ @@||example.com^*-tracking.js
54
+ @@||example.com^-tracking.js`.split(/\r?\n/);
55
+ const filtered = validateAllowIp(rules);
56
+
57
+ expect(filtered).toEqual([
58
+ '! here goes a comment',
59
+ '',
60
+ '||example.org^',
61
+ '||185.149.120.173^', // valid because IP addresses is allowed
62
+ '||ex*.org^', // valid because contains special characters
63
+ '||example.org^$important',
64
+ '||*.ga^$denyallow=example1.ga|example2.ga',
65
+ '://ww4.$denyallow=ww4.example.com',
66
+ '://example.org',
67
+ '||example.org^|',
68
+ '@@||example.org^|$important',
69
+ ]);
70
+ });
71
+ });
@@ -1,4 +1,4 @@
1
- const validate = require('../../src/transformations/validate');
1
+ const { validate } = require('../../src/transformations/validate');
2
2
 
3
3
  describe('Validate', () => {
4
4
  it('simple /etc/hosts rule', () => {
@@ -38,6 +38,7 @@ describe('Validate', () => {
38
38
  const rules = `! here goes a comment
39
39
 
40
40
  ||example.org^
41
+ ||185.149.120.173^
41
42
  ! invalid rule comment will be removed
42
43
  ||example.com/atata
43
44
  ||ex*.org^