@adguard/hostlist-compiler 2.1.0 → 3.0.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/.dockerignore +29 -0
  2. package/.github/workflows/ci.yml +22 -0
  3. package/.github/workflows/mirror.yml +24 -0
  4. package/.github/workflows/prepare-release.yml +26 -0
  5. package/.github/workflows/publish-release.yml +44 -0
  6. package/.markdownlint.json +29 -0
  7. package/.markdownlintignore +1 -0
  8. package/AGENTS.md +314 -103
  9. package/CHANGELOG.md +76 -7
  10. package/DEPLOYMENT.md +171 -0
  11. package/DEVELOPMENT.md +174 -76
  12. package/README.md +240 -140
  13. package/examples/china/README.md +2 -2
  14. package/examples/energized/README.md +4 -2
  15. package/examples/sdn/README.md +2 -2
  16. package/examples/whitelist/README.md +2 -2
  17. package/package.json +29 -24
  18. package/src/cli.js +59 -33
  19. package/src/compile-source.js +4 -3
  20. package/src/configuration.js +132 -2
  21. package/src/download.js +27 -0
  22. package/src/filter.js +2 -2
  23. package/src/index.d.ts +2 -0
  24. package/src/index.js +21 -51
  25. package/src/{transformations/ip-normalize.js → ip-normalize.js} +2 -2
  26. package/src/schemas/configuration.schema.js +137 -0
  27. package/src/source-types.js +17 -0
  28. package/src/transformations/enum.js +49 -0
  29. package/src/transformations/include.js +1 -1
  30. package/src/transformations/transform.js +40 -74
  31. package/src/transformations/validate-allow-ip-and-public-suffix.js +1 -1
  32. package/src/transformations/validate-allow-ip.js +1 -1
  33. package/src/utils.js +1284 -0
  34. package/src/validation-conflicts.js +106 -0
  35. package/test/cli-error-output.test.js +305 -0
  36. package/test/cli.test.js +47 -0
  37. package/test/compile-source.test.js +167 -7
  38. package/test/configuration.test.js +438 -0
  39. package/test/download.test.js +260 -0
  40. package/test/helpers/http-mock.js +94 -0
  41. package/test/index.test.js +497 -4
  42. package/test/{transformations/ip-normalize.test.js → ip-normalize.test.js} +1 -1
  43. package/test/resolve-include.test.js +8 -8
  44. package/test/transformations/covert-to-ascii.test.js +1 -1
  45. package/test/transformations/exclude.test.js +2 -2
  46. package/test/transformations/include.test.js +9 -3
  47. package/test/transformations/ip-alignment.test.js +1 -1
  48. package/test/transformations/transform.test.js +66 -1
  49. package/test/utils.test.js +1559 -0
  50. package/test/validation-conflicts.test.js +230 -0
  51. package/bamboo-specs/bamboo.yaml +0 -14
  52. package/bamboo-specs/build.yaml +0 -92
  53. package/bamboo-specs/deploy.yaml +0 -38
  54. package/bamboo-specs/increment.yaml +0 -54
  55. package/bamboo-specs/permissions.yaml +0 -16
  56. package/bamboo-specs/test.yaml +0 -56
  57. package/src/schemas/configuration.schema.json +0 -369
  58. package/tools/build-txt.js +0 -19
@@ -0,0 +1,230 @@
1
+ const Ajv = require('ajv');
2
+ const { TRANSFORMATIONS, VALIDATION_TRANSFORMATIONS } = require('../src/transformations/enum');
3
+ const {
4
+ checkIncompatibleValidationTransformations,
5
+ checkCrossLevelValidationConflicts,
6
+ buildSchemaConflictRules,
7
+ } = require('../src/validation-conflicts');
8
+
9
+ describe('Validation conflicts', () => {
10
+ describe('checkIncompatibleValidationTransformations', () => {
11
+ it.each([
12
+ ['Validate', 'ValidateAllowIp'],
13
+ ['Validate', 'ValidateAllowPublicSuffix'],
14
+ ['Validate', 'ValidateAllowIpAndPublicSuffix'],
15
+ ['ValidateAllowIp', 'ValidateAllowPublicSuffix'],
16
+ ['ValidateAllowIp', 'ValidateAllowIpAndPublicSuffix'],
17
+ ['ValidateAllowPublicSuffix', 'ValidateAllowIpAndPublicSuffix'],
18
+ ])('throws when incompatible validation transformations are combined: %s + %s', (first, second) => {
19
+ const transformations = [first, second];
20
+
21
+ expect(() => checkIncompatibleValidationTransformations(transformations))
22
+ .toThrow(`Validation transformations cannot be combined: ${transformations.join(', ')}.`);
23
+ });
24
+
25
+ it('throws when three validation transformations are combined', () => {
26
+ const transformations = ['Validate', 'ValidateAllowIp', 'ValidateAllowPublicSuffix'];
27
+
28
+ expect(() => checkIncompatibleValidationTransformations(transformations))
29
+ .toThrow(`Validation transformations cannot be combined: ${transformations.join(', ')}.`);
30
+ });
31
+
32
+ it('does not throw for a single validation transformation', () => {
33
+ expect(() => checkIncompatibleValidationTransformations(['Validate'])).not.toThrow();
34
+ });
35
+
36
+ it('does not throw without validation transformations', () => {
37
+ expect(() => checkIncompatibleValidationTransformations(['RemoveComments', 'Compress'])).not.toThrow();
38
+ expect(() => checkIncompatibleValidationTransformations([])).not.toThrow();
39
+ });
40
+ });
41
+
42
+ describe('checkCrossLevelValidationConflicts', () => {
43
+ it('throws when validation transformations are used at both source and top level', () => {
44
+ const configuration = {
45
+ transformations: ['Validate'],
46
+ sources: [
47
+ {
48
+ source: 'test.txt',
49
+ transformations: ['ValidateAllowPublicSuffix'],
50
+ },
51
+ ],
52
+ };
53
+
54
+ expect(() => checkCrossLevelValidationConflicts(configuration)).toThrow(
55
+ /Validation transformations cannot be used at both source and top level/,
56
+ );
57
+ });
58
+
59
+ it('does not include source credentials in the conflict error', () => {
60
+ const configuration = {
61
+ transformations: ['Validate'],
62
+ sources: [
63
+ {
64
+ source: 'https://user:secret@example.org/test.txt',
65
+ transformations: ['ValidateAllowPublicSuffix'],
66
+ },
67
+ ],
68
+ };
69
+
70
+ let message = '';
71
+ try {
72
+ checkCrossLevelValidationConflicts(configuration);
73
+ } catch (error) {
74
+ message = error.message;
75
+ }
76
+
77
+ expect(message).toContain('Source "https://example.org/test.txt"');
78
+ expect(message).not.toContain('secret');
79
+ });
80
+
81
+ it('does not include a quote or space in the username in the conflict error', () => {
82
+ const configuration = {
83
+ transformations: ['Validate'],
84
+ sources: [
85
+ {
86
+ source: "https://o'brien:secret@example.org/test.txt",
87
+ transformations: ['ValidateAllowPublicSuffix'],
88
+ },
89
+ ],
90
+ };
91
+
92
+ let message = '';
93
+ try {
94
+ checkCrossLevelValidationConflicts(configuration);
95
+ } catch (error) {
96
+ message = error.message;
97
+ }
98
+
99
+ expect(message).toContain('Source "https://example.org/test.txt"');
100
+ expect(message).not.toContain('secret');
101
+ });
102
+
103
+ it('does not include credentials from a prefixed source in the conflict error', () => {
104
+ const configuration = {
105
+ transformations: ['Validate'],
106
+ sources: [
107
+ {
108
+ source: ' https://user:secret@example.org/test.txt',
109
+ transformations: ['ValidateAllowPublicSuffix'],
110
+ },
111
+ ],
112
+ };
113
+
114
+ let message = '';
115
+ try {
116
+ checkCrossLevelValidationConflicts(configuration);
117
+ } catch (error) {
118
+ message = error.message;
119
+ }
120
+
121
+ expect(message).toContain('Source " https://example.org/test.txt"');
122
+ expect(message).not.toContain('secret');
123
+ });
124
+
125
+ it('does not include a literal @ password from a path-less source in the conflict error', () => {
126
+ const configuration = {
127
+ transformations: ['Validate'],
128
+ sources: [
129
+ {
130
+ source: 'https://user:p@s s@example.org',
131
+ transformations: ['ValidateAllowPublicSuffix'],
132
+ },
133
+ ],
134
+ };
135
+
136
+ let message = '';
137
+ try {
138
+ checkCrossLevelValidationConflicts(configuration);
139
+ } catch (error) {
140
+ message = error.message;
141
+ }
142
+
143
+ expect(message).toContain('Source "https://example.org"');
144
+ expect(message).not.toContain('p@s s');
145
+ });
146
+
147
+ it('does not throw when a validation transformation is used at source level only', () => {
148
+ const configuration = {
149
+ sources: [
150
+ {
151
+ source: 'test.txt',
152
+ transformations: ['ValidateAllowPublicSuffix'],
153
+ },
154
+ ],
155
+ };
156
+
157
+ expect(() => checkCrossLevelValidationConflicts(configuration)).not.toThrow();
158
+ });
159
+
160
+ it('does not throw when a validation transformation is used at top level only', () => {
161
+ const configuration = {
162
+ transformations: ['Validate'],
163
+ sources: [
164
+ {
165
+ source: 'test.txt',
166
+ },
167
+ ],
168
+ };
169
+
170
+ expect(() => checkCrossLevelValidationConflicts(configuration)).not.toThrow();
171
+ });
172
+
173
+ it('does not throw without validation transformations', () => {
174
+ const configuration = {
175
+ sources: [
176
+ {
177
+ source: 'test.txt',
178
+ transformations: ['RemoveComments'],
179
+ },
180
+ ],
181
+ };
182
+
183
+ expect(() => checkCrossLevelValidationConflicts(configuration)).not.toThrow();
184
+ });
185
+ });
186
+
187
+ describe('buildSchemaConflictRules', () => {
188
+ it('generates a rule for each unordered pair of validation transformations', () => {
189
+ // A literal list, independent of the implementation's pairing logic.
190
+ const expectedPairs = [
191
+ ['Validate', 'ValidateAllowIp'],
192
+ ['Validate', 'ValidateAllowPublicSuffix'],
193
+ ['Validate', 'ValidateAllowIpAndPublicSuffix'],
194
+ ['ValidateAllowIp', 'ValidateAllowPublicSuffix'],
195
+ ['ValidateAllowIp', 'ValidateAllowIpAndPublicSuffix'],
196
+ ['ValidateAllowPublicSuffix', 'ValidateAllowIpAndPublicSuffix'],
197
+ ];
198
+
199
+ const generatedPairs = buildSchemaConflictRules().map((rule) => rule.not.allOf
200
+ .map((part) => part.contains.const));
201
+
202
+ expect(generatedPairs).toEqual(expectedPairs);
203
+ });
204
+
205
+ it('rejects a transformations list with multiple validation transformations', () => {
206
+ const ajv = new Ajv();
207
+ const validate = ajv.compile({ type: 'array', allOf: buildSchemaConflictRules() });
208
+
209
+ expect(validate(['Validate', 'ValidateAllowIp'])).toBe(false);
210
+ expect(validate(['Validate', 'ValidateAllowIp', 'ValidateAllowPublicSuffix'])).toBe(false);
211
+ expect(validate(['Validate'])).toBe(true);
212
+ expect(validate(['RemoveComments', 'Compress'])).toBe(true);
213
+ });
214
+ });
215
+
216
+ it('keeps every validation transformation frozen', () => {
217
+ expect(Object.isFrozen(VALIDATION_TRANSFORMATIONS)).toBe(true);
218
+ });
219
+
220
+ it('keeps every validation transformation a member of the TRANSFORMATIONS enum', () => {
221
+ // Regression guard: VALIDATION_TRANSFORMATIONS is built from TRANSFORMATIONS.*
222
+ // references (see src/transformations/enum.js), so a hand-typed literal or a
223
+ // renamed enum key would produce an entry that is not a member of the enum.
224
+ const transformationNames = Object.values(TRANSFORMATIONS);
225
+
226
+ VALIDATION_TRANSFORMATIONS.forEach((name) => {
227
+ expect(transformationNames).toContain(name);
228
+ });
229
+ });
230
+ });
@@ -1,14 +0,0 @@
1
- ---
2
- !include 'build.yaml'
3
-
4
- ---
5
- !include 'deploy.yaml'
6
-
7
- ---
8
- !include 'increment.yaml'
9
-
10
- ---
11
- !include 'permissions.yaml'
12
-
13
- ---
14
- !include 'test.yaml'
@@ -1,92 +0,0 @@
1
- ---
2
- version: 2
3
- plan:
4
- project-key: AJL
5
- key: HOSTLISTCMPLRBUILD
6
- name: hostlist-compiler - build
7
- variables:
8
- dockerContainer: adguard/node-ssh:18.13--0
9
-
10
- stages:
11
- - Build:
12
- manual: false
13
- final: false
14
- jobs:
15
- - Build
16
-
17
- Build:
18
- key: BUILD
19
- other:
20
- clean-working-dir: true
21
- docker:
22
- image: "${bamboo.dockerContainer}"
23
- volumes:
24
- ${system.YARN_DIR}: "${bamboo.cacheYarn}"
25
- tasks:
26
- - checkout:
27
- force-clean-build: true
28
- - script:
29
- interpreter: SHELL
30
- scripts:
31
- - |-
32
- set -e
33
- set -x
34
-
35
- # Fix mixed logs
36
- exec 2>&1
37
-
38
- # install dependencies at the beginning
39
- yarn install ${bamboo.varsYarn}
40
-
41
- yarn build-txt
42
- - inject-variables:
43
- file: dist/build.txt
44
- scope: RESULT
45
- namespace: inject
46
- - script:
47
- interpreter: SHELL
48
- scripts:
49
- - |-
50
- #!/bin/bash
51
- set -e
52
- set -x
53
-
54
- # Fix mixed logs
55
- exec 2>&1
56
-
57
- # Fail if version was not bumped
58
- if [ "$(git tag -l "v${bamboo.inject.version}")" ]; then
59
- echo "Build failed!"
60
- echo "Error: Tag v${bamboo.inject.version} already exists. Bump the version in package.json before building"
61
- exit 1
62
- fi
63
-
64
- yarn pack --filename hostlist-compiler.tgz
65
-
66
- # clean up at the end
67
- rm -rf node_modules
68
- - any-task:
69
- plugin-key: com.atlassian.bamboo.plugins.vcs:task.vcs.tagging
70
- configuration:
71
- selectedRepository: defaultRepository
72
- tagName: v${bamboo.inject.version}
73
- artifacts:
74
- - name: hostlist-compiler.tgz
75
- location: ./
76
- pattern: hostlist-compiler.tgz
77
- shared: true
78
- required: true
79
- requirements:
80
- - adg-docker: true
81
-
82
- triggers: []
83
-
84
- branches:
85
- create: manually
86
- delete: never
87
- link-to-jira: true
88
-
89
- labels: []
90
-
91
- other:
92
- concurrent-build-plugin: system-default
@@ -1,38 +0,0 @@
1
- ---
2
- version: 2
3
- deployment:
4
- name: hostlist-compiler - deploy
5
- source-plan: AJL-HOSTLISTCMPLRBUILD
6
- release-naming: ${bamboo.inject.version}
7
- environments:
8
- - npmjs
9
-
10
- npmjs:
11
- docker:
12
- image: adguard/node-ssh:18.13--0
13
- volumes:
14
- ${system.YARN_DIR}: "${bamboo.cacheYarn}"
15
- triggers: []
16
- tasks:
17
- - checkout:
18
- force-clean-build: true
19
- - artifact-download:
20
- artifacts:
21
- - name: hostlist-compiler.tgz
22
- - script:
23
- interpreter: SHELL
24
- scripts:
25
- - |-
26
- set -e
27
- set -x
28
-
29
- # Fix mixed logs
30
- exec 2>&1
31
-
32
- ls -alt
33
-
34
- export NPM_TOKEN=${bamboo.npmSecretToken}
35
- echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > .npmrc
36
- npm publish hostlist-compiler.tgz --access public
37
- requirements:
38
- - adg-docker: true
@@ -1,54 +0,0 @@
1
- ---
2
- version: 2
3
- plan:
4
- project-key: AJL
5
- key: HOSTLISTCMPLRINCR
6
- name: hostlist-compiler - increment
7
- variables:
8
- dockerContainer: adguard/node-ssh:18.13--0
9
-
10
- stages:
11
- - Increment:
12
- manual: true
13
- final: false
14
- jobs:
15
- - Increment
16
-
17
- Increment:
18
- key: INCR
19
- docker:
20
- image: "${bamboo.dockerContainer}"
21
- volumes:
22
- ${system.YARN_DIR}: "${bamboo.cacheYarn}"
23
- other:
24
- clean-working-dir: true
25
- tasks:
26
- - checkout:
27
- force-clean-build: true
28
- - script:
29
- interpreter: SHELL
30
- scripts:
31
- - |-
32
- set -e
33
- set -x
34
-
35
- # Fix mixed logs
36
- exec 2>&1
37
-
38
- yarn increment
39
- - any-task:
40
- plugin-key: com.atlassian.bamboo.plugins.vcs:task.vcs.commit
41
- configuration:
42
- commitMessage: 'skipci: Automatic increment build number'
43
- selectedRepository: defaultRepository
44
- requirements:
45
- - adg-docker: true
46
-
47
- branches:
48
- create: manually
49
- delete: never
50
- link-to-jira: true
51
-
52
- labels: []
53
- other:
54
- concurrent-build-plugin: system-default
@@ -1,16 +0,0 @@
1
- ---
2
- version: 2
3
- deployment:
4
- name: hostlist-compiler - deploy
5
- deployment-permissions:
6
- - groups:
7
- - extensions-developers
8
- permissions:
9
- - view
10
- environment-permissions:
11
- - npmjs:
12
- - groups:
13
- - extensions-developers
14
- permissions:
15
- - view
16
- - deploy
@@ -1,56 +0,0 @@
1
- ---
2
- version: 2
3
- plan:
4
- project-key: AJL
5
- key: HOSTLISTCMPLRTEST
6
- name: hostlist-compiler - tests
7
- variables:
8
- dockerContainer: adguard/node-ssh:18.13--0
9
-
10
- stages:
11
- - Test:
12
- manual: false
13
- final: false
14
- jobs:
15
- - Test
16
-
17
- Test:
18
- key: TEST
19
- docker:
20
- image: "${bamboo.dockerContainer}"
21
- volumes:
22
- ${system.YARN_DIR}: "${bamboo.cacheYarn}"
23
- tasks:
24
- - checkout:
25
- force-clean-build: true
26
- - script:
27
- interpreter: SHELL
28
- scripts:
29
- - |-
30
- set -e
31
- set -x
32
-
33
- # Fix mixed logs
34
- exec 2>&1
35
-
36
- yarn install ${bamboo.varsYarn}
37
-
38
- yarn lint
39
- yarn test
40
-
41
- rm -rf node_modules
42
- final-tasks:
43
- - clean
44
- requirements:
45
- - adg-docker: true
46
-
47
- branches:
48
- create: for-pull-request
49
- delete:
50
- after-deleted-days: '1'
51
- after-inactive-days: '5'
52
- link-to-jira: true
53
-
54
- labels: []
55
- other:
56
- concurrent-build-plugin: system-default