@adguard/hostlist-compiler 1.0.39 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -9,18 +9,20 @@ This is a simple tool that makes it easier to compile a [hosts blocklist](https:
9
9
  - [Command-line](#command-line)
10
10
  - [API](#api)
11
11
  - [Transformations](#transformations)
12
+ - [ConvertToAscii](#convert-to-ascii)
13
+ - [TrimLines](#trimlines)
12
14
  - [RemoveComments](#remove-comments)
13
15
  - [Compress](#compress)
14
16
  - [RemoveModifiers](#remove-modifiers)
17
+ - [InvertAllow](#invertallow)
15
18
  - [Validate](#validate)
16
19
  - [ValidateAllowIp](#validate-allow-ip)
20
+ - [ValidateAllowPublicSuffix](#validate-allow-public-suffix)
21
+ - [ValidateAllowIpAndPublicSuffix](#validate-allow-ip-and-public-suffix)
17
22
  - [Deduplicate](#deduplicate)
18
- - [InvertAllow](#invertallow)
19
23
  - [RemoveEmptyLines](#removeemptylines)
20
- - [TrimLines](#trimlines)
21
24
  - [InsertFinalNewLine](#insertfinalnewline)
22
- - [ConvertToAscii](#convert-to-ascii)
23
- - [How to build](#how-to-build)
25
+ - [Documentation](#documentation)
24
26
 
25
27
  ## <a name="usage"></a> Usage
26
28
 
@@ -162,8 +164,8 @@ Here is an example:
162
164
  Rules in HOSTS syntax: `/hosts.txt`
163
165
 
164
166
  ```txt
165
- 0.0.0.0 ads.example.com
166
- 0.0.0.0 tracking.example1.com
167
+ 0.0.0.0 ads.example.com
168
+ 0.0.0.0 tracking.example1.com
167
169
  0.0.0.0 example.com
168
170
  ```
169
171
 
@@ -195,7 +197,7 @@ Configuration of the final list:
195
197
  Final filter output of `/hosts.txt` after applying the `Compress` transformation and exclusions:
196
198
 
197
199
  ```txt
198
- ||ads.example.com^
200
+ ||ads.example.com^
199
201
  ||tracking.example1.com^
200
202
  ```
201
203
 
@@ -305,18 +307,67 @@ import { writeFileSync } from 'fs';
305
307
 
306
308
  Here is the full list of transformations that are available:
307
309
 
310
+ 1. `ConvertToAscii`
311
+ 1. `TrimLines`
308
312
  1. `RemoveComments`
309
313
  1. `Compress`
310
314
  1. `RemoveModifiers`
315
+ 1. `InvertAllow`
311
316
  1. `Validate`
312
317
  1. `ValidateAllowIp`
318
+ 1. `ValidateAllowPublicSuffix`
319
+ 1. `ValidateAllowIpAndPublicSuffix`
313
320
  1. `Deduplicate`
314
- 1. `InvertAllow`
315
321
  1. `RemoveEmptyLines`
316
- 1. `TrimLines`
317
322
  1. `InsertFinalNewLine`
318
323
 
319
- Please note that these transformations are are always applied in the order specified here.
324
+ Please note that these transformations are always applied in the order specified here.
325
+
326
+ ### <a name="convert-to-ascii"></a> ConvertToAscii
327
+
328
+ This transformation converts all non-ASCII characters to their ASCII equivalents. It is always performed first.
329
+
330
+ **Example:**
331
+
332
+ Original list:
333
+
334
+ ```
335
+ ||*.рус^
336
+ ||*.कॉम^
337
+ ||*.セール^
338
+ ```
339
+
340
+ Here's what we will have after applying this transformation:
341
+
342
+ ```
343
+ ||*.xn--p1acf^
344
+ ||*.xn--11b4c3d^
345
+ ||*.xn--1qqw23a^
346
+ ```
347
+
348
+ ### <a name="trimlines"></a> TrimLines
349
+
350
+ This is a very simple transformation that removes leading and trailing spaces/tabs.
351
+
352
+ **Example:**
353
+
354
+ Original list:
355
+
356
+ ```
357
+ rule1
358
+ rule2
359
+ rule3
360
+ rule4
361
+ ```
362
+
363
+ Here's what we will have after applying this transformation:
364
+
365
+ ```
366
+ rule1
367
+ rule2
368
+ rule3
369
+ rule4
370
+ ```
320
371
 
321
372
  ### <a name="remove-comments"></a> RemoveComments
322
373
 
@@ -348,6 +399,39 @@ Here is the list of modifiers that will be removed:
348
399
  > Blindly removing `$third-party` from traditional ad blocking rules leads to lots of false-positives.
349
400
  >> This is exactly why there is an option to exclude rules - you may need to use it.
350
401
 
402
+ ### <a name="invertallow"></a> InvertAllow
403
+
404
+ This transformation converts blocking rules to "allow" rules. Note, that it does nothing to /etc/hosts rules (unless they were previously converted to adblock-style syntax by a different transformation, for example [Compress](#compress)).
405
+
406
+ There are two important notes about this transformation:
407
+
408
+ 1. It keeps the original rules order.
409
+ 2. It ignores comments, empty lines, /etc/hosts rules and existing "allow" rules.
410
+
411
+ **Example:**
412
+
413
+ Original list:
414
+
415
+ ```
416
+ ! comment 1
417
+ rule1
418
+
419
+ # comment 2
420
+ 192.168.11.11 test.local
421
+ @@rule2
422
+ ```
423
+
424
+ Here's what we will have after applying this transformation:
425
+
426
+ ```
427
+ ! comment 1
428
+ @@rule1
429
+
430
+ # comment 2
431
+ 192.168.11.11 test.local
432
+ @@rule2
433
+ ```
434
+
351
435
  ### <a name="validate"></a> Validate
352
436
 
353
437
  This transformation is really crucial if you're using a filter list for a traditional ad blocker as a source.
@@ -360,16 +444,59 @@ So here's what it does:
360
444
  - Discards rules with unsupported modifiers. [Click here](https://github.com/AdguardTeam/AdGuardHome/wiki/Hosts-Blocklists#-adblock-style-syntax) to learn more about which modifiers are supported.
361
445
  - Discards rules that are too short.
362
446
  - Discards IP addresses. If you need to keep IP addresses, use [ValidateAllowIp](#validate-allow-ip) instead.
447
+
448
+ #### <a name="rejected-ip-patterns"></a>Rejected IP Patterns
449
+
450
+ The following IP patterns are rejected by all validation transformations (`Validate`, `ValidateAllowIp`, `ValidateAllowPublicSuffix`, `ValidateAllowIpAndPublicSuffix`) as they are either unsafe or ambiguous:
451
+
452
+ - `||192.168.1^` — 3-octet with `^` - does not work
453
+ - `192.168.1` — Ambiguous: would match `192.168.11`, `192.168.111`, etc.
454
+ - `1.2.` or `1.2.*` — Too wide (1-2 octets), use regex instead
455
+
363
456
  - Removes rules that block entire top-level domains (TLDs) like `||*.org^`, unless they have specific limiting modifiers such as `$denyallow`, `$badfilter`, or `$client`.
364
457
  Examples:
365
458
  - `||*.org^` - this rule will be removed
366
459
  - `||*.org^$denyallow=example.com` - this rule will be kept because it has a limiting modifier
460
+ If such rules must be saved, use [ValidateAllowPublicSuffix](#validate-allow-public-suffix) or [ValidateAllowIpAndPublicSuffix](#validate-allow-ip-and-public-suffix).
367
461
 
368
462
  If there are comments preceding the invalid rule, they will be removed as well.
369
463
 
370
464
  ### <a name="validate-allow-ip"></a> ValidateAllowIp
371
465
 
372
- This transformation exactly repeats the behavior of [Validate](#validate), but leaves the IP addresses in the lists.
466
+ This transformation extends [Validate](#validate) to allow IP addresses in the lists. It also **normalizes IP rules** in Adblock-style to the safe format `||ip^`.
467
+
468
+ #### IP Rule Normalization
469
+
470
+ - `1.2.3.4` → `||1.2.3.4^` (add both separators)
471
+ - `1.2.3.4^` → `||1.2.3.4^` (add left anchor)
472
+ - `|1.2.3.4` → `||1.2.3.4^` (replace `|` with `||`, add `^`)
473
+ - `|1.2.3.4^` → `||1.2.3.4^` (replace `|` with `||`)
474
+ - `||1.2.3.4` → `||1.2.3.4^` (add right separator)
475
+ - `||1.2.3.4^` → (no change, already canonical)
476
+ - `192.168.1.` → `||192.168.1.` (3-octet subnet wildcard)
477
+ - `192.168.1.*` → `||192.168.1.*` (3-octet subnet wildcard)
478
+
479
+ Modifiers like `$important`, `$client`, `$denyallow`, `$badfilter` are preserved during normalization.
480
+
481
+ > **Note:** Invalid IP patterns are rejected solely by the base [Validate](#validate) logic — normalization only converts valid patterns to canonical form and passes everything else through unchanged. See [Rejected IP Patterns](#rejected-ip-patterns) for details.
482
+
483
+ ### <a name="validate-allow-public-suffix"></a> ValidateAllowPublicSuffix
484
+
485
+ This transformation exactly repeats the behavior of [Validate](#validate), but leaves rules that match whole public suffixes (e.g. `||hl.cn^`, `||org^`) in the list.
486
+
487
+ It still filters out invalid syntax rules and unsupported modifiers, but does not reject public-suffix rules unless the rule itself is malformed.
488
+
489
+ > **Note:** Combining any `Validate`, `ValidateAllowIp`, `ValidateAllowPublicSuffix`, and `ValidateAllowIpAndPublicSuffix` in one transformation list is not allowed and will result in an error. Each runs its own validator on the already-filtered output of the previous one, so allow-modes become silently ineffective.
490
+
491
+ > **Important:** Validation transformations also cannot be used at both source-level and top-level simultaneously. For example, if a source uses `ValidateAllowPublicSuffix` and the top-level configuration uses `Validate`, the compiler will throw an error. This is because the top-level `Validate` would override the source-level validation, making `ValidateAllowPublicSuffix` ineffective. Use validation transformations at only one level.
492
+
493
+ ### <a name="validate-allow-ip-and-public-suffix"></a> ValidateAllowIpAndPublicSuffix
494
+
495
+ This transformation combines the behavior of [ValidateAllowIp](#validate-allow-ip) and [ValidateAllowPublicSuffix](#validate-allow-public-suffix). It allows both IP addresses and public suffix rules in the list.
496
+
497
+ Like `ValidateAllowIp`, it normalizes incomplete IP rules to the safe format `||ip^` before validation. See [IP Rule Normalization](#ip-rule-normalization) for details.
498
+
499
+ Like `ValidateAllowPublicSuffix`, it keeps rules that match whole public suffixes (e.g. `||hl.cn^`, `||org^`).
373
500
 
374
501
  ### <a name="deduplicate"></a> Deduplicate
375
502
 
@@ -396,39 +523,6 @@ Here's what will be left after the transformation:
396
523
  rule1
397
524
  ```
398
525
 
399
- ### <a name="invertallow"></a> InvertAllow
400
-
401
- This transformation converts blocking rules to "allow" rules. Note, that it does nothing to /etc/hosts rules (unless they were previously converted to adblock-style syntax by a different transformation, for example [Compress](#compress)).
402
-
403
- There are two important notes about this transformation:
404
-
405
- 1. It keeps the original rules order.
406
- 2. It ignores comments, empty lines, /etc/hosts rules and existing "allow" rules.
407
-
408
- **Example:**
409
-
410
- Original list:
411
-
412
- ```
413
- ! comment 1
414
- rule1
415
-
416
- # comment 2
417
- 192.168.11.11 test.local
418
- @@rule2
419
- ```
420
-
421
- Here's what we will have after applying this transformation:
422
-
423
- ```
424
- ! comment 1
425
- @@rule1
426
-
427
- # comment 2
428
- 192.168.11.11 test.local
429
- @@rule2
430
- ```
431
-
432
526
  ### <a name="removeemptylines"></a> RemoveEmptyLines
433
527
 
434
528
  This is a very simple transformation that removes empty lines.
@@ -454,30 +548,6 @@ rule2
454
548
  rule3
455
549
  ```
456
550
 
457
- ### <a name="trimlines"></a> TrimLines
458
-
459
- This is a very simple transformation that removes leading and trailing spaces/tabs.
460
-
461
- **Example:**
462
-
463
- Original list:
464
-
465
- ```
466
- rule1
467
- rule2
468
- rule3
469
- rule4
470
- ```
471
-
472
- Here's what we will have after applying this transformation:
473
-
474
- ```
475
- rule1
476
- rule2
477
- rule3
478
- rule4
479
- ```
480
-
481
551
  ### <a name="insertfinalnewline"></a> InsertFinalNewLine
482
552
 
483
553
  This is a very simple transformation that inserts a final newline.
@@ -503,31 +573,8 @@ rule3
503
573
 
504
574
  `RemoveEmptyLines` doesn't delete this empty row due to the execution order.
505
575
 
506
- ### <a name="convert-to-ascii"></a> ConvertToAscii
507
-
508
- This transformation converts all non-ASCII characters to their ASCII equivalents. It is always performed first.
509
-
510
- **Example:**
511
-
512
- Original list:
513
-
514
- ```
515
- ||*.рус^
516
- ||*.कॉम^
517
- ||*.セール^
518
- ```
519
-
520
- Here's what we will have after applying this transformation:
521
-
522
- ```
523
- ||*.xn--p1acf^
524
- ||*.xn--11b4c3d^
525
- ||*.xn--1qqw23a^
526
- ```
527
-
528
- ## <a name="how-to-build"></a> How to build
576
+ ## <a name="documentation"></a> Documentation
529
577
 
530
- - `yarn install` - installs dependencies
531
- - `yarn lint` - runs eslint
532
- - `yarn test` - runs tests
533
- - `node src/cli.js -c examples/sdn/configuration.json -o filter.txt` - runs compiler with the example configuration
578
+ - [Development guide](DEVELOPMENT.md) — how to set up the environment, build, test, and contribute
579
+ - [LLM agent rules](AGENTS.md) — code guidelines and project conventions
580
+ - [Changelog](CHANGELOG.md) — release history
@@ -54,10 +54,10 @@ Build:
54
54
  # Fix mixed logs
55
55
  exec 2>&1
56
56
 
57
- # Fail if version was not incremented
57
+ # Fail if version was not bumped
58
58
  if [ "$(git tag -l "v${bamboo.inject.version}")" ]; then
59
59
  echo "Build failed!"
60
- echo "Error: Tag v${bamboo.inject.version} already exists. Increment version before build"
60
+ echo "Error: Tag v${bamboo.inject.version} already exists. Bump the version in package.json before building"
61
61
  exit 1
62
62
  fi
63
63
 
@@ -86,14 +86,6 @@ branches:
86
86
  delete: never
87
87
  link-to-jira: true
88
88
 
89
- notifications:
90
- - events:
91
- - plan-status-changed
92
- recipients:
93
- - webhook:
94
- name: Build webhook
95
- url: http://prod.jirahub.service.eu.consul/v1/webhook/bamboo
96
-
97
89
  labels: []
98
90
 
99
91
  other:
@@ -36,10 +36,3 @@ npmjs:
36
36
  npm publish hostlist-compiler.tgz --access public
37
37
  requirements:
38
38
  - adg-docker: true
39
- notifications:
40
- - events:
41
- - deployment-started-and-finished
42
- recipients:
43
- - webhook:
44
- name: Deploy webhook
45
- url: http://prod.jirahub.service.eu.consul/v1/webhook/bamboo
@@ -9,7 +9,7 @@ variables:
9
9
 
10
10
  stages:
11
11
  - Increment:
12
- manual: false
12
+ manual: true
13
13
  final: false
14
14
  jobs:
15
15
  - Increment
@@ -51,14 +51,6 @@ branches:
51
51
  after-inactive-days: '5'
52
52
  link-to-jira: true
53
53
 
54
- notifications:
55
- - events:
56
- - plan-status-changed
57
- recipients:
58
- - webhook:
59
- name: Build webhook
60
- url: http://prod.jirahub.service.eu.consul/v1/webhook/bamboo
61
-
62
54
  labels: []
63
55
  other:
64
56
  concurrent-build-plugin: system-default
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@adguard/hostlist-compiler",
3
- "version": "1.0.39",
3
+ "version": "2.1.0",
4
4
  "description": "A simple tool that compiles hosts blocklists from multiple sources",
5
5
  "main": "src/index.js",
6
6
  "types": "src/index.d.ts",
@@ -31,11 +31,11 @@
31
31
  "@adguard/filters-downloader": "^2.3.1",
32
32
  "ajv": "^6.12.0",
33
33
  "ajv-errors": "^1.0.1",
34
- "axios": "^0.19.2",
34
+ "axios": "^1.15.0",
35
35
  "better-ajv-errors": "^0.6.7",
36
36
  "consola": "^2.11.3",
37
37
  "lodash": "^4.17.15",
38
- "tldts": "^5.6.10",
38
+ "tldts": "7.0.28",
39
39
  "yargs": "^15.3.0"
40
40
  }
41
41
  }
package/src/index.d.ts CHANGED
@@ -2,14 +2,18 @@ declare module '@adguard/hostlist-compiler' {
2
2
  export type SourceType = 'adblock' | 'hosts';
3
3
 
4
4
  export type Transformation =
5
+ 'ConvertToAscii' |
6
+ 'TrimLines' |
5
7
  'RemoveComments' |
6
8
  'Compress' |
7
9
  'RemoveModifiers' |
10
+ 'InvertAllow' |
8
11
  'Validate' |
12
+ 'ValidateAllowIp' |
13
+ 'ValidateAllowPublicSuffix' |
14
+ 'ValidateAllowIpAndPublicSuffix' |
9
15
  'Deduplicate' |
10
- 'InvertAllow' |
11
16
  'RemoveEmptyLines' |
12
- 'TrimLines' |
13
17
  'InsertFinalNewLine';
14
18
 
15
19
  /** A source for the filter list */
package/src/index.js CHANGED
@@ -2,7 +2,7 @@ const _ = require('lodash');
2
2
  const consola = require('consola');
3
3
  const config = require('./configuration');
4
4
  const compileSource = require('./compile-source');
5
- const { transform } = require('./transformations/transform');
5
+ const { transform, TRANSFORMATIONS } = require('./transformations/transform');
6
6
  const packageJson = require('../package.json');
7
7
 
8
8
  /**
@@ -63,6 +63,47 @@ function prepareSourceHeader(source) {
63
63
  return lines;
64
64
  }
65
65
 
66
+ /**
67
+ * Checks for conflicting validation transformations across source-level and top-level.
68
+ * Throws if a validation transformation is used at both levels, as this causes silent data loss.
69
+ *
70
+ * @param {*} configuration - compilation configuration.
71
+ */
72
+ function checkCrossLevelValidationConflicts(configuration) {
73
+ const VALIDATION_TRANSFORMATIONS = [
74
+ TRANSFORMATIONS.Validate,
75
+ TRANSFORMATIONS.ValidateAllowIp,
76
+ TRANSFORMATIONS.ValidateAllowPublicSuffix,
77
+ TRANSFORMATIONS.ValidateAllowIpAndPublicSuffix,
78
+ ];
79
+
80
+ const topLevelTransformations = configuration.transformations || [];
81
+ const topLevelValidations = VALIDATION_TRANSFORMATIONS
82
+ .filter((t) => topLevelTransformations.indexOf(t) !== -1);
83
+
84
+ if (topLevelValidations.length === 0) {
85
+ return;
86
+ }
87
+
88
+ // Check each source for conflicting validations
89
+ // eslint-disable-next-line no-restricted-syntax
90
+ for (const source of configuration.sources) {
91
+ const sourceTransformations = source.transformations || [];
92
+ const sourceValidations = VALIDATION_TRANSFORMATIONS
93
+ .filter((t) => sourceTransformations.indexOf(t) !== -1);
94
+
95
+ if (sourceValidations.length > 0) {
96
+ throw new Error(
97
+ 'Validation transformations cannot be used at both source and top level. '
98
+ + `Source "${source.source}" uses [${sourceValidations.join(', ')}], `
99
+ + `but top level uses [${topLevelValidations.join(', ')}]. `
100
+ + 'This causes the source-level validation to be overridden by the top-level one, '
101
+ + 'making the source-level validation ineffective.',
102
+ );
103
+ }
104
+ }
105
+ }
106
+
66
107
  /**
67
108
  * Compiles a filter list using the specified configuration.
68
109
  *
@@ -78,6 +119,8 @@ async function compile(configuration) {
78
119
  throw new Error('Failed to validate configuration');
79
120
  }
80
121
 
122
+ checkCrossLevelValidationConflicts(configuration);
123
+
81
124
  consola.info(`Configuration: ${JSON.stringify(configuration, 0, 4)}`);
82
125
 
83
126
  // This will be the final list of rules