@adguard/hostlist-compiler 2.0.0 → 2.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/.dockerignore +29 -0
  2. package/.github/workflows/ci.yml +22 -0
  3. package/.github/workflows/mirror.yml +24 -0
  4. package/.github/workflows/prepare-release.yml +26 -0
  5. package/.github/workflows/publish-release.yml +44 -0
  6. package/AGENTS.md +15 -16
  7. package/CHANGELOG.md +44 -0
  8. package/DEVELOPMENT.md +16 -17
  9. package/README.md +42 -3
  10. package/package.json +10 -11
  11. package/src/cli.js +6 -2
  12. package/src/index.d.ts +1 -0
  13. package/src/index.js +6 -1
  14. package/src/schemas/configuration.schema.json +98 -0
  15. package/src/transformations/ip-normalize.js +182 -0
  16. package/src/transformations/transform.js +6 -4
  17. package/src/transformations/validate-allow-ip-and-public-suffix.js +33 -0
  18. package/src/transformations/validate-allow-ip.js +16 -3
  19. package/src/transformations/validate.js +118 -53
  20. package/src/utils.js +137 -0
  21. package/test/configuration.test.js +6 -0
  22. package/test/transformations/ip-alignment.test.js +65 -0
  23. package/test/transformations/ip-normalize.test.js +331 -0
  24. package/test/transformations/transform.test.js +6 -0
  25. package/test/transformations/validate-allow-ip-and-public-suffix.test.js +167 -0
  26. package/test/transformations/validate-allow-ip.test.js +46 -18
  27. package/test/transformations/validate-allow-public-suffix.test.js +20 -0
  28. package/test/transformations/validate.test.js +57 -0
  29. package/bamboo-specs/bamboo.yaml +0 -14
  30. package/bamboo-specs/build.yaml +0 -92
  31. package/bamboo-specs/deploy.yaml +0 -38
  32. package/bamboo-specs/increment.yaml +0 -54
  33. package/bamboo-specs/permissions.yaml +0 -16
  34. package/bamboo-specs/test.yaml +0 -56
  35. package/tools/build-txt.js +0 -19
@@ -0,0 +1,167 @@
1
+ const { validateAllowIpAndPublicSuffix } = require('../../src/transformations/validate-allow-ip-and-public-suffix');
2
+
3
+ describe('ValidateAllowIpAndPublicSuffix', () => {
4
+ it('keeps valid domain rules', () => {
5
+ const rules = [
6
+ '||example.org^',
7
+ '||example.com^',
8
+ ];
9
+ const filtered = validateAllowIpAndPublicSuffix(rules);
10
+
11
+ expect(filtered).toEqual([
12
+ '||example.org^',
13
+ '||example.com^',
14
+ ]);
15
+ });
16
+
17
+ it('keeps IP address rules (like ValidateAllowIp)', () => {
18
+ const rules = [
19
+ '||185.149.120.173^',
20
+ '0.0.0.0 example.org',
21
+ ];
22
+ const filtered = validateAllowIpAndPublicSuffix(rules);
23
+
24
+ expect(filtered).toEqual([
25
+ '||185.149.120.173^',
26
+ '0.0.0.0 example.org',
27
+ ]);
28
+ });
29
+
30
+ it('keeps public suffix rules (like ValidateAllowPublicSuffix)', () => {
31
+ const rules = [
32
+ '||hl.cn^',
33
+ '||org^',
34
+ '||*.org^',
35
+ '.org^',
36
+ ];
37
+ const filtered = validateAllowIpAndPublicSuffix(rules);
38
+
39
+ expect(filtered).toEqual([
40
+ '||hl.cn^',
41
+ '||org^',
42
+ '||*.org^',
43
+ '.org^',
44
+ ]);
45
+ });
46
+
47
+ it('keeps both IP and public suffix rules together', () => {
48
+ const rules = [
49
+ '||185.149.120.173^',
50
+ '||org^',
51
+ '||example.org^',
52
+ '||hl.cn^',
53
+ '0.0.0.0 example.com',
54
+ ];
55
+ const filtered = validateAllowIpAndPublicSuffix(rules);
56
+
57
+ expect(filtered).toEqual([
58
+ '||185.149.120.173^',
59
+ '||org^',
60
+ '||example.org^',
61
+ '||hl.cn^',
62
+ '0.0.0.0 example.com',
63
+ ]);
64
+ });
65
+
66
+ it('normalizes IP patterns (like ValidateAllowIp)', () => {
67
+ const rules = [
68
+ '1.1.1.1^',
69
+ '192.168.1.1^',
70
+ '|1.1.1.1^',
71
+ '||192.168.1.',
72
+ '||192.168.1.*',
73
+ '||example.org^',
74
+ ];
75
+ const filtered = validateAllowIpAndPublicSuffix(rules);
76
+
77
+ expect(filtered).toEqual([
78
+ '||1.1.1.1^',
79
+ '||192.168.1.1^',
80
+ '||1.1.1.1^',
81
+ '||192.168.1.',
82
+ '||192.168.1.*',
83
+ '||example.org^',
84
+ ]);
85
+ });
86
+
87
+ it('rejects invalid rules', () => {
88
+ const rules = [
89
+ '||invalid/rule',
90
+ '||example.org^$domain=example.com',
91
+ '||example.org^',
92
+ ];
93
+ const filtered = validateAllowIpAndPublicSuffix(rules);
94
+
95
+ expect(filtered).toEqual([
96
+ '||example.org^',
97
+ ]);
98
+ });
99
+
100
+ it('rejects invalid IP patterns', () => {
101
+ const rules = [
102
+ // 3-octet without trailing dot — rejected (ambiguous)
103
+ '192.168.1',
104
+ '||192.168.1',
105
+ // 3-octet with ^ — rejected (doesn't work)
106
+ '||192.168.1^',
107
+ // 2-octet — rejected (too wide)
108
+ '||1.1.',
109
+ '1.1.',
110
+ // valid entries — kept
111
+ '||example.org^',
112
+ '||1.2.3.4^',
113
+ ];
114
+ const filtered = validateAllowIpAndPublicSuffix(rules);
115
+
116
+ expect(filtered).toEqual([
117
+ '||example.org^',
118
+ '||1.2.3.4^',
119
+ ]);
120
+ });
121
+
122
+ it('rejects terminated IDN TLDs (like ValidateAllowPublicSuffix)', () => {
123
+ const rules = [
124
+ '||*.xn--jlq61u9w7b^',
125
+ '||xn--jlq61u9w7b^',
126
+ '.xn--p1acf^',
127
+ '||*.xn--p1acf^',
128
+ ];
129
+ const filtered = validateAllowIpAndPublicSuffix(rules);
130
+
131
+ expect(filtered).toEqual([
132
+ '.xn--p1acf^',
133
+ '||*.xn--p1acf^',
134
+ ]);
135
+ });
136
+
137
+ it('keeps known ICANN TLDs but rejects unknown ones', () => {
138
+ const rules = [
139
+ '||aaa^',
140
+ '||aaaa^',
141
+ '||foo^',
142
+ '||zzzz^',
143
+ '||example.org^',
144
+ ];
145
+ const filtered = validateAllowIpAndPublicSuffix(rules);
146
+
147
+ expect(filtered).toEqual([
148
+ '||aaa^',
149
+ '||foo^',
150
+ '||example.org^',
151
+ ]);
152
+ });
153
+
154
+ it('removes preceding comments for invalid rules', () => {
155
+ const rules = `! rule comment
156
+
157
+ ||invalid/rule
158
+ ! comment
159
+ ||valid.com^`.split(/\r?\n/);
160
+ const filtered = validateAllowIpAndPublicSuffix(rules);
161
+
162
+ expect(filtered).toEqual([
163
+ '! comment',
164
+ '||valid.com^',
165
+ ]);
166
+ });
167
+ });
@@ -46,6 +46,8 @@ describe('ValidateAllowIp', () => {
46
46
  ||example.org^$third-party
47
47
  ||example.org^$important
48
48
  ||*.ga^$denyallow=example1.ga|example2.ga
49
+ 1.1.1.1$denyallow=001114.cn
50
+ ||1.1.1.1^$denyallow=001114.cn
49
51
  ://ww4.$denyallow=ww4.example.com
50
52
  ://example.org
51
53
  ||example.org^|
@@ -69,44 +71,70 @@ describe('ValidateAllowIp', () => {
69
71
  ]);
70
72
  });
71
73
 
72
- it('allows IP-subnet patterns but rejects IP-suffix patterns', () => {
73
- // ValidateAllowIp allows IP-subnet patterns (with | or || prefix) because they block specific subnets.
74
- // But IP-suffix patterns (no prefix) are ALWAYS rejected because they block unpredictably
75
- // (e.g., 1.1^ blocks 1.1.1.1, 1.1.111.1, example1.1).
74
+ it('normalizes and validates IP patterns', () => {
75
+ // ValidateAllowIp now normalizes incomplete IP rules to ||ip^ format
76
+ // and rejects patterns that don't work in AdGuard Home.
76
77
  const rules = [
77
- // IP-suffix (no prefix) — rejected ALWAYS (blocks unpredictably)
78
+ // IP-suffix (no prefix) with ^ — normalized to ||ip^ for 4 octets
79
+ '1.1.1.1^',
80
+ '192.168.1.1^',
81
+ // IP-suffix (no prefix) with ^ — rejected for <4 octets (doesn't work)
78
82
  '1.1^',
79
83
  '1.1.1^',
80
- '1.1.1.1^',
81
84
  '192.168^',
82
85
  '10.0.1^',
83
- '192.168.1.1^',
84
- // IP-subnet (with prefix) — allowed in ValidateAllowIp
86
+ // IP-subnet with ^ and <4 octets — rejected (doesn't work in AdGuard Home)
85
87
  '||1.1^',
86
88
  '|1.1^',
87
89
  '||1.1.1^',
88
90
  '|1.1.1^',
89
- '||1.1.1.1^',
90
- '|1.1.1.1^',
91
91
  '||192.168^',
92
92
  '||10.0.1^',
93
+ // Full 4-octet IPs — normalized to ||ip^
94
+ '||1.1.1.1^',
95
+ '|1.1.1.1^',
93
96
  '||192.168.1.1^',
97
+ '1.2.3.4^|',
98
+ '|1.2.3.4^|',
99
+ '||1.2.3.4^|',
100
+ // 3-octet subnet wildcards — allowed (all prefix forms normalize to ||)
101
+ '||192.168.1.',
102
+ '||192.168.1.*',
103
+ '|192.168.1.',
104
+ '|192.168.1.*',
105
+ '192.168.1.',
106
+ // 3-octet without trailing dot — rejected (ambiguous)
107
+ '192.168.1',
108
+ '||192.168.1',
109
+ // 2-octet patterns — rejected (too wide)
110
+ '||1.1.',
111
+ '1.1.',
112
+ // 4-octet with trailing dot or wildcard — rejected (malformed IP, not a complete address)
113
+ '1.2.3.4.',
114
+ '1.2.3.4.*',
115
+ '||1.2.3.4.',
116
+ '||1.2.3.4.*',
94
117
  // valid domain — kept
95
118
  '||example.org^',
96
119
  ];
97
120
  const filtered = validateAllowIp(rules);
98
121
 
99
122
  expect(filtered).toEqual([
100
- // IP-subnet patterns are allowed
101
- '||1.1^',
102
- '|1.1^',
103
- '||1.1.1^',
104
- '|1.1.1^',
123
+ // 4-octet IPs normalized to ||ip^
124
+ '||1.1.1.1^',
125
+ '||192.168.1.1^',
126
+ '||1.1.1.1^',
105
127
  '||1.1.1.1^',
106
- '|1.1.1.1^',
107
- '||192.168^',
108
- '||10.0.1^',
109
128
  '||192.168.1.1^',
129
+ '||1.2.3.4^',
130
+ '||1.2.3.4^',
131
+ '||1.2.3.4^',
132
+ // 3-octet subnet wildcards (all prefix forms normalized to ||)
133
+ '||192.168.1.',
134
+ '||192.168.1.*',
135
+ '||192.168.1.',
136
+ '||192.168.1.*',
137
+ '||192.168.1.',
110
138
  // valid domain
111
139
  '||example.org^',
112
140
  ]);
@@ -261,4 +261,24 @@ describe('validateAllowPublicSuffix', () => {
261
261
  '@@.xyz123notreal.org^',
262
262
  ]);
263
263
  });
264
+
265
+ it('rejects terminated/unknown IDN TLDs (xn-- prefixed) not in ICANN list', () => {
266
+ // xn--jlq61u9w7b (.蔚来) was terminated by ICANN in 2024.
267
+ // tldts reports it as non-ICANN public suffix, so whole-TLD rules should be rejected
268
+ // just like any other unknown single-label pattern.
269
+ const rules = [
270
+ '||*.xn--jlq61u9w7b^', // terminated IDN TLD — rejected
271
+ '||xn--jlq61u9w7b^', // same, without wildcard — rejected
272
+ '.xn--jlq61u9w7b^', // same, dot-prefix — rejected
273
+ '*.xn--jlq61u9w7b^', // same, wildcard-prefix — rejected
274
+ '.xn--p1acf^', // .рус — real ICANN TLD — kept
275
+ '||*.xn--p1acf^', // .рус — real ICANN TLD — kept
276
+ ];
277
+ const filtered = validateAllowPublicSuffix(rules);
278
+
279
+ expect(filtered).toEqual([
280
+ '.xn--p1acf^',
281
+ '||*.xn--p1acf^',
282
+ ]);
283
+ });
264
284
  });
@@ -46,6 +46,8 @@ describe('Validate', () => {
46
46
  ||example.org^$third-party
47
47
  ||example.org^$important
48
48
  ||*.ga^$denyallow=example1.ga|example2.ga
49
+ 1.1.1.1$denyallow=001114.cn
50
+ ||1.1.1.1^$denyallow=001114.cn
49
51
  ://ww4.$denyallow=ww4.example.com
50
52
  ://example.org
51
53
  ||example.org^|
@@ -327,4 +329,59 @@ describe('Validate', () => {
327
329
  '||a.b.c.d^',
328
330
  ]);
329
331
  });
332
+
333
+ it('rejects unsafe IP patterns without ^ (too wide or ambiguous)', () => {
334
+ // These patterns don't have ^ but are still dangerous:
335
+ // - 1-2 octet patterns (too wide)
336
+ // - 3 octet patterns without trailing dot/wildcard (ambiguous)
337
+ const rules = [
338
+ // 1-2 octets — too wide
339
+ '1.2.',
340
+ '1.2.*',
341
+ '||1.2.',
342
+ '||1.2.*',
343
+ '192.168',
344
+ '||192.168',
345
+ // 3 octets without trailing dot/wildcard — ambiguous
346
+ '192.168.1',
347
+ '||192.168.1',
348
+ '10.0.1',
349
+ // 3 octets WITH trailing dot/wildcard, no prefix — unsafe (no || means subnet is unanchored)
350
+ '192.168.1.',
351
+ '192.168.1.*',
352
+ '10.10.34.',
353
+ '10.10.34.*',
354
+ // 3 octets WITH trailing dot/wildcard, with || prefix — rejected in Validate (IP subnet)
355
+ '||192.168.1.',
356
+ '||192.168.1.*',
357
+ // 4 octets WITH trailing dot or wildcard — malformed IP, rejected in all validators
358
+ '1.2.3.4.',
359
+ '1.2.3.4.*',
360
+ '|1.2.3.4.',
361
+ '|1.2.3.4.*',
362
+ '||1.2.3.4.',
363
+ '||1.2.3.4.*',
364
+ // valid domains — kept
365
+ '||example.org^',
366
+ ];
367
+ const filtered = validate(rules);
368
+
369
+ expect(filtered).toEqual([
370
+ '||example.org^',
371
+ ]);
372
+ });
373
+
374
+ it('allows domains with IP-like subdomains', () => {
375
+ // Patterns like ||1.2.3.4.example.com^ are valid domain rules, not IP patterns.
376
+ // parseIpPattern returns null for them (> 4 parts or non-numeric labels),
377
+ // so they pass through all IP checks and are validated as regular domain rules.
378
+ const rules = [
379
+ '||1-2-3-4.example.com^',
380
+ '||111.22.33.44.host.example.com^',
381
+ '||111.22.33.44.ll.host.example.com^',
382
+ ];
383
+ const filtered = validate(rules);
384
+
385
+ expect(filtered).toEqual(rules);
386
+ });
330
387
  });
@@ -1,14 +0,0 @@
1
- ---
2
- !include 'build.yaml'
3
-
4
- ---
5
- !include 'deploy.yaml'
6
-
7
- ---
8
- !include 'increment.yaml'
9
-
10
- ---
11
- !include 'permissions.yaml'
12
-
13
- ---
14
- !include 'test.yaml'
@@ -1,92 +0,0 @@
1
- ---
2
- version: 2
3
- plan:
4
- project-key: AJL
5
- key: HOSTLISTCMPLRBUILD
6
- name: hostlist-compiler - build
7
- variables:
8
- dockerContainer: adguard/node-ssh:18.13--0
9
-
10
- stages:
11
- - Build:
12
- manual: false
13
- final: false
14
- jobs:
15
- - Build
16
-
17
- Build:
18
- key: BUILD
19
- other:
20
- clean-working-dir: true
21
- docker:
22
- image: "${bamboo.dockerContainer}"
23
- volumes:
24
- ${system.YARN_DIR}: "${bamboo.cacheYarn}"
25
- tasks:
26
- - checkout:
27
- force-clean-build: true
28
- - script:
29
- interpreter: SHELL
30
- scripts:
31
- - |-
32
- set -e
33
- set -x
34
-
35
- # Fix mixed logs
36
- exec 2>&1
37
-
38
- # install dependencies at the beginning
39
- yarn install ${bamboo.varsYarn}
40
-
41
- yarn build-txt
42
- - inject-variables:
43
- file: dist/build.txt
44
- scope: RESULT
45
- namespace: inject
46
- - script:
47
- interpreter: SHELL
48
- scripts:
49
- - |-
50
- #!/bin/bash
51
- set -e
52
- set -x
53
-
54
- # Fix mixed logs
55
- exec 2>&1
56
-
57
- # Fail if version was not bumped
58
- if [ "$(git tag -l "v${bamboo.inject.version}")" ]; then
59
- echo "Build failed!"
60
- echo "Error: Tag v${bamboo.inject.version} already exists. Bump the version in package.json before building"
61
- exit 1
62
- fi
63
-
64
- yarn pack --filename hostlist-compiler.tgz
65
-
66
- # clean up at the end
67
- rm -rf node_modules
68
- - any-task:
69
- plugin-key: com.atlassian.bamboo.plugins.vcs:task.vcs.tagging
70
- configuration:
71
- selectedRepository: defaultRepository
72
- tagName: v${bamboo.inject.version}
73
- artifacts:
74
- - name: hostlist-compiler.tgz
75
- location: ./
76
- pattern: hostlist-compiler.tgz
77
- shared: true
78
- required: true
79
- requirements:
80
- - adg-docker: true
81
-
82
- triggers: []
83
-
84
- branches:
85
- create: manually
86
- delete: never
87
- link-to-jira: true
88
-
89
- labels: []
90
-
91
- other:
92
- concurrent-build-plugin: system-default
@@ -1,38 +0,0 @@
1
- ---
2
- version: 2
3
- deployment:
4
- name: hostlist-compiler - deploy
5
- source-plan: AJL-HOSTLISTCMPLRBUILD
6
- release-naming: ${bamboo.inject.version}
7
- environments:
8
- - npmjs
9
-
10
- npmjs:
11
- docker:
12
- image: adguard/node-ssh:18.13--0
13
- volumes:
14
- ${system.YARN_DIR}: "${bamboo.cacheYarn}"
15
- triggers: []
16
- tasks:
17
- - checkout:
18
- force-clean-build: true
19
- - artifact-download:
20
- artifacts:
21
- - name: hostlist-compiler.tgz
22
- - script:
23
- interpreter: SHELL
24
- scripts:
25
- - |-
26
- set -e
27
- set -x
28
-
29
- # Fix mixed logs
30
- exec 2>&1
31
-
32
- ls -alt
33
-
34
- export NPM_TOKEN=${bamboo.npmSecretToken}
35
- echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > .npmrc
36
- npm publish hostlist-compiler.tgz --access public
37
- requirements:
38
- - adg-docker: true
@@ -1,54 +0,0 @@
1
- ---
2
- version: 2
3
- plan:
4
- project-key: AJL
5
- key: HOSTLISTCMPLRINCR
6
- name: hostlist-compiler - increment
7
- variables:
8
- dockerContainer: adguard/node-ssh:18.13--0
9
-
10
- stages:
11
- - Increment:
12
- manual: true
13
- final: false
14
- jobs:
15
- - Increment
16
-
17
- Increment:
18
- key: INCR
19
- docker:
20
- image: "${bamboo.dockerContainer}"
21
- volumes:
22
- ${system.YARN_DIR}: "${bamboo.cacheYarn}"
23
- other:
24
- clean-working-dir: true
25
- tasks:
26
- - checkout:
27
- force-clean-build: true
28
- - script:
29
- interpreter: SHELL
30
- scripts:
31
- - |-
32
- set -e
33
- set -x
34
-
35
- # Fix mixed logs
36
- exec 2>&1
37
-
38
- yarn increment
39
- - any-task:
40
- plugin-key: com.atlassian.bamboo.plugins.vcs:task.vcs.commit
41
- configuration:
42
- commitMessage: 'skipci: Automatic increment build number'
43
- selectedRepository: defaultRepository
44
- requirements:
45
- - adg-docker: true
46
-
47
- branches:
48
- create: manually
49
- delete: never
50
- link-to-jira: true
51
-
52
- labels: []
53
- other:
54
- concurrent-build-plugin: system-default
@@ -1,16 +0,0 @@
1
- ---
2
- version: 2
3
- deployment:
4
- name: hostlist-compiler - deploy
5
- deployment-permissions:
6
- - groups:
7
- - extensions-developers
8
- permissions:
9
- - view
10
- environment-permissions:
11
- - npmjs:
12
- - groups:
13
- - extensions-developers
14
- permissions:
15
- - view
16
- - deploy
@@ -1,56 +0,0 @@
1
- ---
2
- version: 2
3
- plan:
4
- project-key: AJL
5
- key: HOSTLISTCMPLRTEST
6
- name: hostlist-compiler - tests
7
- variables:
8
- dockerContainer: adguard/node-ssh:18.13--0
9
-
10
- stages:
11
- - Test:
12
- manual: false
13
- final: false
14
- jobs:
15
- - Test
16
-
17
- Test:
18
- key: TEST
19
- docker:
20
- image: "${bamboo.dockerContainer}"
21
- volumes:
22
- ${system.YARN_DIR}: "${bamboo.cacheYarn}"
23
- tasks:
24
- - checkout:
25
- force-clean-build: true
26
- - script:
27
- interpreter: SHELL
28
- scripts:
29
- - |-
30
- set -e
31
- set -x
32
-
33
- # Fix mixed logs
34
- exec 2>&1
35
-
36
- yarn install ${bamboo.varsYarn}
37
-
38
- yarn lint
39
- yarn test
40
-
41
- rm -rf node_modules
42
- final-tasks:
43
- - clean
44
- requirements:
45
- - adg-docker: true
46
-
47
- branches:
48
- create: for-pull-request
49
- delete:
50
- after-deleted-days: '1'
51
- after-inactive-days: '5'
52
- link-to-jira: true
53
-
54
- labels: []
55
- other:
56
- concurrent-build-plugin: system-default
@@ -1,19 +0,0 @@
1
- const fs = require('fs');
2
- const path = require('path');
3
- const { version } = require('../package.json');
4
-
5
- const PATH = '../dist';
6
- const FILENAME = 'build.txt';
7
-
8
- const buildTxt = () => {
9
- const dir = path.resolve(__dirname, PATH);
10
-
11
- if (!fs.existsSync(dir)) {
12
- fs.mkdirSync(dir);
13
- }
14
-
15
- const content = `version=${version}`;
16
- fs.writeFileSync(path.resolve(__dirname, PATH, FILENAME), content);
17
- };
18
-
19
- buildTxt();