@adguard/hostlist-compiler 1.0.38 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,7 +9,7 @@ variables:
9
9
 
10
10
  stages:
11
11
  - Increment:
12
- manual: false
12
+ manual: true
13
13
  final: false
14
14
  jobs:
15
15
  - Increment
@@ -51,14 +51,6 @@ branches:
51
51
  after-inactive-days: '5'
52
52
  link-to-jira: true
53
53
 
54
- notifications:
55
- - events:
56
- - plan-status-changed
57
- recipients:
58
- - webhook:
59
- name: Build webhook
60
- url: http://prod.jirahub.service.eu.consul/v1/webhook/bamboo
61
-
62
54
  labels: []
63
55
  other:
64
56
  concurrent-build-plugin: system-default
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@adguard/hostlist-compiler",
3
- "version": "1.0.38",
3
+ "version": "2.0.0",
4
4
  "description": "A simple tool that compiles hosts blocklists from multiple sources",
5
5
  "main": "src/index.js",
6
6
  "types": "src/index.d.ts",
@@ -28,7 +28,7 @@
28
28
  "nock": "^12.0.2"
29
29
  },
30
30
  "dependencies": {
31
- "@adguard/filters-downloader": "^2.3.0",
31
+ "@adguard/filters-downloader": "^2.3.1",
32
32
  "ajv": "^6.12.0",
33
33
  "ajv-errors": "^1.0.1",
34
34
  "axios": "^0.19.2",
@@ -21,7 +21,7 @@ const { transform } = require('./transformations/transform');
21
21
  */
22
22
  async function compileSource(source) {
23
23
  consola.info(`Start compiling ${source.source}`);
24
- let rules = await FiltersDownloader.download(source.source);
24
+ let rules = await FiltersDownloader.download(source.source, {}, { allowEmptyResponse: true });
25
25
  consola.info(`Original length is ${rules.length}`);
26
26
 
27
27
  rules = await transform(rules, source, source.transformations);
package/src/filter.js CHANGED
@@ -16,8 +16,8 @@ async function downloadAll(sources) {
16
16
  return list;
17
17
  }
18
18
 
19
- await Promise.all(sources.map(async (s) => {
20
- const rulesStr = await FiltersDownloader.download(s);
19
+ await Promise.all(sources.map(async (source) => {
20
+ const rulesStr = await FiltersDownloader.download(source, {}, { allowEmptyResponse: true });
21
21
  const rules = rulesStr
22
22
  .filter((el) => el.trim().length > 0 && !ruleUtils.isComment(el));
23
23
  list = list.concat(rules);
package/src/index.d.ts CHANGED
@@ -2,14 +2,17 @@ declare module '@adguard/hostlist-compiler' {
2
2
  export type SourceType = 'adblock' | 'hosts';
3
3
 
4
4
  export type Transformation =
5
+ 'ConvertToAscii' |
6
+ 'TrimLines' |
5
7
  'RemoveComments' |
6
8
  'Compress' |
7
9
  'RemoveModifiers' |
10
+ 'InvertAllow' |
8
11
  'Validate' |
12
+ 'ValidateAllowIp' |
13
+ 'ValidateAllowPublicSuffix' |
9
14
  'Deduplicate' |
10
- 'InvertAllow' |
11
15
  'RemoveEmptyLines' |
12
- 'TrimLines' |
13
16
  'InsertFinalNewLine';
14
17
 
15
18
  /** A source for the filter list */
package/src/index.js CHANGED
@@ -2,7 +2,7 @@ const _ = require('lodash');
2
2
  const consola = require('consola');
3
3
  const config = require('./configuration');
4
4
  const compileSource = require('./compile-source');
5
- const { transform } = require('./transformations/transform');
5
+ const { transform, TRANSFORMATIONS } = require('./transformations/transform');
6
6
  const packageJson = require('../package.json');
7
7
 
8
8
  /**
@@ -63,6 +63,46 @@ function prepareSourceHeader(source) {
63
63
  return lines;
64
64
  }
65
65
 
66
+ /**
67
+ * Checks for conflicting validation transformations across source-level and top-level.
68
+ * Throws if a validation transformation is used at both levels, as this causes silent data loss.
69
+ *
70
+ * @param {*} configuration - compilation configuration.
71
+ */
72
+ function checkCrossLevelValidationConflicts(configuration) {
73
+ const VALIDATION_TRANSFORMATIONS = [
74
+ TRANSFORMATIONS.Validate,
75
+ TRANSFORMATIONS.ValidateAllowIp,
76
+ TRANSFORMATIONS.ValidateAllowPublicSuffix,
77
+ ];
78
+
79
+ const topLevelTransformations = configuration.transformations || [];
80
+ const topLevelValidations = VALIDATION_TRANSFORMATIONS
81
+ .filter((t) => topLevelTransformations.indexOf(t) !== -1);
82
+
83
+ if (topLevelValidations.length === 0) {
84
+ return;
85
+ }
86
+
87
+ // Check each source for conflicting validations
88
+ // eslint-disable-next-line no-restricted-syntax
89
+ for (const source of configuration.sources) {
90
+ const sourceTransformations = source.transformations || [];
91
+ const sourceValidations = VALIDATION_TRANSFORMATIONS
92
+ .filter((t) => sourceTransformations.indexOf(t) !== -1);
93
+
94
+ if (sourceValidations.length > 0) {
95
+ throw new Error(
96
+ 'Validation transformations cannot be used at both source and top level. '
97
+ + `Source "${source.source}" uses [${sourceValidations.join(', ')}], `
98
+ + `but top level uses [${topLevelValidations.join(', ')}]. `
99
+ + 'This causes the source-level validation to be overridden by the top-level one, '
100
+ + 'making the source-level validation ineffective.',
101
+ );
102
+ }
103
+ }
104
+ }
105
+
66
106
  /**
67
107
  * Compiles a filter list using the specified configuration.
68
108
  *
@@ -78,6 +118,8 @@ async function compile(configuration) {
78
118
  throw new Error('Failed to validate configuration');
79
119
  }
80
120
 
121
+ checkCrossLevelValidationConflicts(configuration);
122
+
81
123
  consola.info(`Configuration: ${JSON.stringify(configuration, 0, 4)}`);
82
124
 
83
125
  // This will be the final list of rules
@@ -37,6 +37,56 @@
37
37
  "transformations": {
38
38
  "description": "A list of the transformations that will be applied",
39
39
  "type": "array",
40
+ "allOf": [
41
+ {
42
+ "not": {
43
+ "allOf": [
44
+ {
45
+ "contains": {
46
+ "const": "Validate"
47
+ }
48
+ },
49
+ {
50
+ "contains": {
51
+ "const": "ValidateAllowIp"
52
+ }
53
+ }
54
+ ]
55
+ }
56
+ },
57
+ {
58
+ "not": {
59
+ "allOf": [
60
+ {
61
+ "contains": {
62
+ "const": "Validate"
63
+ }
64
+ },
65
+ {
66
+ "contains": {
67
+ "const": "ValidateAllowPublicSuffix"
68
+ }
69
+ }
70
+ ]
71
+ }
72
+ },
73
+ {
74
+ "not": {
75
+ "allOf": [
76
+ {
77
+ "contains": {
78
+ "const": "ValidateAllowIp"
79
+ }
80
+ },
81
+ {
82
+ "contains": {
83
+ "const": "ValidateAllowPublicSuffix"
84
+ }
85
+ }
86
+ ]
87
+ }
88
+ }
89
+ ],
40
90
  "items": {
41
91
  "type": "string",
42
92
  "enum": [
@@ -45,6 +95,7 @@
45
95
  "Compress",
46
96
  "Validate",
47
97
  "ValidateAllowIp",
98
+ "ValidateAllowPublicSuffix",
48
99
  "Deduplicate",
49
100
  "InvertAllow",
50
101
  "RemoveEmptyLines",
@@ -114,6 +165,56 @@
114
165
  "transformations": {
115
166
  "description": "A list of the transformations that will be applied",
116
167
  "type": "array",
168
+ "allOf": [
169
+ {
170
+ "not": {
171
+ "allOf": [
172
+ {
173
+ "contains": {
174
+ "const": "Validate"
175
+ }
176
+ },
177
+ {
178
+ "contains": {
179
+ "const": "ValidateAllowIp"
180
+ }
181
+ }
182
+ ]
183
+ }
184
+ },
185
+ {
186
+ "not": {
187
+ "allOf": [
188
+ {
189
+ "contains": {
190
+ "const": "Validate"
191
+ }
192
+ },
193
+ {
194
+ "contains": {
195
+ "const": "ValidateAllowPublicSuffix"
196
+ }
197
+ }
198
+ ]
199
+ }
200
+ },
201
+ {
202
+ "not": {
203
+ "allOf": [
204
+ {
205
+ "contains": {
206
+ "const": "ValidateAllowIp"
207
+ }
208
+ },
209
+ {
210
+ "contains": {
211
+ "const": "ValidateAllowPublicSuffix"
212
+ }
213
+ }
214
+ ]
215
+ }
216
+ }
217
+ ],
117
218
  "items": {
118
219
  "type": "string",
119
220
  "enum": [
@@ -122,6 +223,7 @@
122
223
  "Compress",
123
224
  "Validate",
124
225
  "ValidateAllowIp",
226
+ "ValidateAllowPublicSuffix",
125
227
  "Deduplicate",
126
228
  "InvertAllow",
127
229
  "RemoveEmptyLines",
@@ -2,6 +2,7 @@ const removeComments = require('./remove-comments');
2
2
  const removeModifiers = require('./remove-modifiers');
3
3
  const { validate } = require('./validate');
4
4
  const { validateAllowIp } = require('./validate-allow-ip');
5
+ const { validateAllowPublicSuffix } = require('./validate-allow-public-suffix');
5
6
  const exclude = require('./exclude');
6
7
  const include = require('./include');
7
8
  const deduplicate = require('./deduplicate');
@@ -21,6 +22,7 @@ const TRANSFORMATIONS = Object.freeze({
21
22
  RemoveModifiers: 'RemoveModifiers',
22
23
  Validate: 'Validate',
23
24
  ValidateAllowIp: 'ValidateAllowIp',
25
+ ValidateAllowPublicSuffix: 'ValidateAllowPublicSuffix',
24
26
  Deduplicate: 'Deduplicate',
25
27
  InvertAllow: 'InvertAllow',
26
28
  RemoveEmptyLines: 'RemoveEmptyLines',
@@ -29,6 +31,32 @@ const TRANSFORMATIONS = Object.freeze({
29
31
  ConvertToAscii: 'ConvertToAscii',
30
32
  });
31
33
 
34
+ const VALIDATION_TRANSFORMATIONS = [
35
+ TRANSFORMATIONS.Validate,
36
+ TRANSFORMATIONS.ValidateAllowIp,
37
+ TRANSFORMATIONS.ValidateAllowPublicSuffix,
38
+ ];
39
+
40
+ /**
41
+ * Throws when multiple validation transformations are selected together.
42
+ * Combining them causes silent data loss: each validator runs on the already-filtered
43
+ * output of the previous one, so allow-modes (AllowIp, AllowPublicSuffix) become ineffective.
44
+ *
45
+ * TODO: When a combined mode for ValidateAllowIp + ValidateAllowPublicSuffix is implemented
46
+ * (see https://github.com/AdguardTeam/HostlistCompiler/issues/126),
47
+ * update this check to allow that specific pair and run Validator(true, true) instead.
48
+ *
49
+ * @param {Array<string>} transformations - configured transformations.
50
+ */
51
+ function checkIncompatibleValidationTransformations(transformations) {
52
+ const selectedValidations = VALIDATION_TRANSFORMATIONS
53
+ .filter((transformation) => transformations.indexOf(transformation) !== -1);
54
+
55
+ if (selectedValidations.length > 1) {
56
+ throw new Error(`Validation transformations cannot be combined: ${selectedValidations.join(', ')}.`);
57
+ }
58
+ }
59
+
32
60
  /**
33
61
  * Applies the specified transformations to the list of rules in the proper order.
34
62
  *
@@ -38,12 +66,14 @@ const TRANSFORMATIONS = Object.freeze({
38
66
  * @returns {Promise<Array<string>>} rules after applying all transformations.
39
67
  */
40
68
  async function transform(rules, configuration, transformations) {
41
- // If none specified -- apply all transformationss
69
+ // If none specified -- apply all transformations
42
70
  if (!transformations) {
43
71
  // eslint-disable-next-line no-param-reassign
44
72
  transformations = [];
45
73
  }
46
74
 
75
+ checkIncompatibleValidationTransformations(transformations);
76
+
47
77
  let transformed = rules;
48
78
 
49
79
  transformed = await exclude(
@@ -57,6 +87,7 @@ async function transform(rules, configuration, transformations) {
57
87
  configuration.inclusions_sources,
58
88
  );
59
89
 
90
+ // When updating the list of transformations, be sure to follow the same order in README.md
60
91
  if (transformations.indexOf(TRANSFORMATIONS.ConvertToAscii) !== -1) {
61
92
  transformed = convertToAscii(transformed);
62
93
  }
@@ -81,6 +112,9 @@ async function transform(rules, configuration, transformations) {
81
112
  if (transformations.indexOf(TRANSFORMATIONS.ValidateAllowIp) !== -1) {
82
113
  transformed = validateAllowIp(transformed);
83
114
  }
115
+ if (transformations.indexOf(TRANSFORMATIONS.ValidateAllowPublicSuffix) !== -1) {
116
+ transformed = validateAllowPublicSuffix(transformed);
117
+ }
84
118
  if (transformations.indexOf(TRANSFORMATIONS.Deduplicate) !== -1) {
85
119
  transformed = deduplicate(transformed);
86
120
  }
@@ -10,8 +10,9 @@ const { Validator } = require('./validate');
10
10
  * @returns {Array<string>} - The filtered array of valid rules, allowing IP addresses.
11
11
  */
12
12
  function validateAllowIp(rules) {
13
- // ip is not allowed by default
14
- const validator = new Validator(true);
13
+ // ip is allowed for this transformation
14
+ // tld is not allowed by default here
15
+ const validator = new Validator(true, false);
15
16
  return validator.validate(rules);
16
17
  }
17
18
  module.exports = {
@@ -0,0 +1,20 @@
1
+ const { Validator } = require('./validate');
2
+
3
+ /**
4
+ * Validates a list of rules and allows public suffix matching.
5
+ *
6
+ * This function creates an instance of the `Validator` class, enabling public
7
+ * suffix matching, and returns a filtered list of valid rules.
8
+ *
9
+ * @param {Array<string>} rules - The array of rules to validate.
10
+ * @returns {Array<string>} - The filtered array of valid rules.
11
+ */
12
+ function validateAllowPublicSuffix(rules) {
13
+ // ip is not allowed by default, but public suffix matching is allowed.
14
+ const validator = new Validator(false, true);
15
+ return validator.validate(rules);
16
+ }
17
+
18
+ module.exports = {
19
+ validateAllowPublicSuffix,
20
+ };