@adguard/hostlist-compiler 1.0.38 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +175 -0
- package/CHANGELOG.md +24 -0
- package/DEVELOPMENT.md +172 -0
- package/README.md +107 -94
- package/bamboo-specs/build.yaml +2 -10
- package/bamboo-specs/deploy.yaml +0 -7
- package/bamboo-specs/increment.yaml +1 -1
- package/bamboo-specs/test.yaml +0 -8
- package/package.json +2 -2
- package/src/compile-source.js +1 -1
- package/src/filter.js +2 -2
- package/src/index.d.ts +5 -2
- package/src/index.js +43 -1
- package/src/schemas/configuration.schema.json +102 -0
- package/src/transformations/transform.js +35 -1
- package/src/transformations/validate-allow-ip.js +3 -2
- package/src/transformations/validate-allow-public-suffix.js +20 -0
- package/src/transformations/validate.js +183 -19
- package/test/compile-source.test.js +44 -0
- package/test/configuration.test.js +64 -0
- package/test/index.test.js +64 -0
- package/test/resolve-include.test.js +98 -0
- package/test/transformations/transform.test.js +19 -0
- package/test/transformations/validate-allow-ip.test.js +44 -1
- package/test/transformations/validate-allow-public-suffix.test.js +264 -0
- package/test/transformations/validate.test.js +194 -0
package/bamboo-specs/test.yaml
CHANGED
|
@@ -51,14 +51,6 @@ branches:
|
|
|
51
51
|
after-inactive-days: '5'
|
|
52
52
|
link-to-jira: true
|
|
53
53
|
|
|
54
|
-
notifications:
|
|
55
|
-
- events:
|
|
56
|
-
- plan-status-changed
|
|
57
|
-
recipients:
|
|
58
|
-
- webhook:
|
|
59
|
-
name: Build webhook
|
|
60
|
-
url: http://prod.jirahub.service.eu.consul/v1/webhook/bamboo
|
|
61
|
-
|
|
62
54
|
labels: []
|
|
63
55
|
other:
|
|
64
56
|
concurrent-build-plugin: system-default
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adguard/hostlist-compiler",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.0.0",
|
|
4
4
|
"description": "A simple tool that compiles hosts blocklists from multiple sources",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"types": "src/index.d.ts",
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
"nock": "^12.0.2"
|
|
29
29
|
},
|
|
30
30
|
"dependencies": {
|
|
31
|
-
"@adguard/filters-downloader": "^2.3.
|
|
31
|
+
"@adguard/filters-downloader": "^2.3.1",
|
|
32
32
|
"ajv": "^6.12.0",
|
|
33
33
|
"ajv-errors": "^1.0.1",
|
|
34
34
|
"axios": "^0.19.2",
|
package/src/compile-source.js
CHANGED
|
@@ -21,7 +21,7 @@ const { transform } = require('./transformations/transform');
|
|
|
21
21
|
*/
|
|
22
22
|
async function compileSource(source) {
|
|
23
23
|
consola.info(`Start compiling ${source.source}`);
|
|
24
|
-
let rules = await FiltersDownloader.download(source.source);
|
|
24
|
+
let rules = await FiltersDownloader.download(source.source, {}, { allowEmptyResponse: true });
|
|
25
25
|
consola.info(`Original length is ${rules.length}`);
|
|
26
26
|
|
|
27
27
|
rules = await transform(rules, source, source.transformations);
|
package/src/filter.js
CHANGED
|
@@ -16,8 +16,8 @@ async function downloadAll(sources) {
|
|
|
16
16
|
return list;
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
await Promise.all(sources.map(async (
|
|
20
|
-
const rulesStr = await FiltersDownloader.download(
|
|
19
|
+
await Promise.all(sources.map(async (source) => {
|
|
20
|
+
const rulesStr = await FiltersDownloader.download(source, {}, { allowEmptyResponse: true });
|
|
21
21
|
const rules = rulesStr
|
|
22
22
|
.filter((el) => el.trim().length > 0 && !ruleUtils.isComment(el));
|
|
23
23
|
list = list.concat(rules);
|
package/src/index.d.ts
CHANGED
|
@@ -2,14 +2,17 @@ declare module '@adguard/hostlist-compiler' {
|
|
|
2
2
|
export type SourceType = 'adblock' | 'hosts';
|
|
3
3
|
|
|
4
4
|
export type Transformation =
|
|
5
|
+
'ConvertToAscii' |
|
|
6
|
+
'TrimLines' |
|
|
5
7
|
'RemoveComments' |
|
|
6
8
|
'Compress' |
|
|
7
9
|
'RemoveModifiers' |
|
|
10
|
+
'InvertAllow' |
|
|
8
11
|
'Validate' |
|
|
12
|
+
'ValidateAllowIp' |
|
|
13
|
+
'ValidateAllowPublicSuffix' |
|
|
9
14
|
'Deduplicate' |
|
|
10
|
-
'InvertAllow' |
|
|
11
15
|
'RemoveEmptyLines' |
|
|
12
|
-
'TrimLines' |
|
|
13
16
|
'InsertFinalNewLine';
|
|
14
17
|
|
|
15
18
|
/** A source for the filter list */
|
package/src/index.js
CHANGED
|
@@ -2,7 +2,7 @@ const _ = require('lodash');
|
|
|
2
2
|
const consola = require('consola');
|
|
3
3
|
const config = require('./configuration');
|
|
4
4
|
const compileSource = require('./compile-source');
|
|
5
|
-
const { transform } = require('./transformations/transform');
|
|
5
|
+
const { transform, TRANSFORMATIONS } = require('./transformations/transform');
|
|
6
6
|
const packageJson = require('../package.json');
|
|
7
7
|
|
|
8
8
|
/**
|
|
@@ -63,6 +63,46 @@ function prepareSourceHeader(source) {
|
|
|
63
63
|
return lines;
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
+
/**
|
|
67
|
+
* Checks for conflicting validation transformations across source-level and top-level.
|
|
68
|
+
* Throws if a validation transformation is used at both levels, as this causes silent data loss.
|
|
69
|
+
*
|
|
70
|
+
* @param {*} configuration - compilation configuration.
|
|
71
|
+
*/
|
|
72
|
+
function checkCrossLevelValidationConflicts(configuration) {
|
|
73
|
+
const VALIDATION_TRANSFORMATIONS = [
|
|
74
|
+
TRANSFORMATIONS.Validate,
|
|
75
|
+
TRANSFORMATIONS.ValidateAllowIp,
|
|
76
|
+
TRANSFORMATIONS.ValidateAllowPublicSuffix,
|
|
77
|
+
];
|
|
78
|
+
|
|
79
|
+
const topLevelTransformations = configuration.transformations || [];
|
|
80
|
+
const topLevelValidations = VALIDATION_TRANSFORMATIONS
|
|
81
|
+
.filter((t) => topLevelTransformations.indexOf(t) !== -1);
|
|
82
|
+
|
|
83
|
+
if (topLevelValidations.length === 0) {
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// Check each source for conflicting validations
|
|
88
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
89
|
+
for (const source of configuration.sources) {
|
|
90
|
+
const sourceTransformations = source.transformations || [];
|
|
91
|
+
const sourceValidations = VALIDATION_TRANSFORMATIONS
|
|
92
|
+
.filter((t) => sourceTransformations.indexOf(t) !== -1);
|
|
93
|
+
|
|
94
|
+
if (sourceValidations.length > 0) {
|
|
95
|
+
throw new Error(
|
|
96
|
+
'Validation transformations cannot be used at both source and top level. '
|
|
97
|
+
+ `Source "${source.source}" uses [${sourceValidations.join(', ')}], `
|
|
98
|
+
+ `but top level uses [${topLevelValidations.join(', ')}]. `
|
|
99
|
+
+ 'This causes the source-level validation to be overridden by the top-level one, '
|
|
100
|
+
+ 'making the source-level validation ineffective.',
|
|
101
|
+
);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
66
106
|
/**
|
|
67
107
|
* Compiles a filter list using the specified configuration.
|
|
68
108
|
*
|
|
@@ -78,6 +118,8 @@ async function compile(configuration) {
|
|
|
78
118
|
throw new Error('Failed to validate configuration');
|
|
79
119
|
}
|
|
80
120
|
|
|
121
|
+
checkCrossLevelValidationConflicts(configuration);
|
|
122
|
+
|
|
81
123
|
consola.info(`Configuration: ${JSON.stringify(configuration, 0, 4)}`);
|
|
82
124
|
|
|
83
125
|
// This will be the final list of rules
|
|
@@ -37,6 +37,56 @@
|
|
|
37
37
|
"transformations": {
|
|
38
38
|
"description": "A list of the transformations that will be applied",
|
|
39
39
|
"type": "array",
|
|
40
|
+
"allOf": [
|
|
41
|
+
{
|
|
42
|
+
"not": {
|
|
43
|
+
"allOf": [
|
|
44
|
+
{
|
|
45
|
+
"contains": {
|
|
46
|
+
"const": "Validate"
|
|
47
|
+
}
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
"contains": {
|
|
51
|
+
"const": "ValidateAllowIp"
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
]
|
|
55
|
+
}
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
"not": {
|
|
59
|
+
"allOf": [
|
|
60
|
+
{
|
|
61
|
+
"contains": {
|
|
62
|
+
"const": "Validate"
|
|
63
|
+
}
|
|
64
|
+
},
|
|
65
|
+
{
|
|
66
|
+
"contains": {
|
|
67
|
+
"const": "ValidateAllowPublicSuffix"
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
]
|
|
71
|
+
}
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
"not": {
|
|
75
|
+
"allOf": [
|
|
76
|
+
{
|
|
77
|
+
"contains": {
|
|
78
|
+
"const": "ValidateAllowIp"
|
|
79
|
+
}
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
"contains": {
|
|
83
|
+
"const": "ValidateAllowPublicSuffix"
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
]
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
],
|
|
40
90
|
"items": {
|
|
41
91
|
"type": "string",
|
|
42
92
|
"enum": [
|
|
@@ -45,6 +95,7 @@
|
|
|
45
95
|
"Compress",
|
|
46
96
|
"Validate",
|
|
47
97
|
"ValidateAllowIp",
|
|
98
|
+
"ValidateAllowPublicSuffix",
|
|
48
99
|
"Deduplicate",
|
|
49
100
|
"InvertAllow",
|
|
50
101
|
"RemoveEmptyLines",
|
|
@@ -114,6 +165,56 @@
|
|
|
114
165
|
"transformations": {
|
|
115
166
|
"description": "A list of the transformations that will be applied",
|
|
116
167
|
"type": "array",
|
|
168
|
+
"allOf": [
|
|
169
|
+
{
|
|
170
|
+
"not": {
|
|
171
|
+
"allOf": [
|
|
172
|
+
{
|
|
173
|
+
"contains": {
|
|
174
|
+
"const": "Validate"
|
|
175
|
+
}
|
|
176
|
+
},
|
|
177
|
+
{
|
|
178
|
+
"contains": {
|
|
179
|
+
"const": "ValidateAllowIp"
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
]
|
|
183
|
+
}
|
|
184
|
+
},
|
|
185
|
+
{
|
|
186
|
+
"not": {
|
|
187
|
+
"allOf": [
|
|
188
|
+
{
|
|
189
|
+
"contains": {
|
|
190
|
+
"const": "Validate"
|
|
191
|
+
}
|
|
192
|
+
},
|
|
193
|
+
{
|
|
194
|
+
"contains": {
|
|
195
|
+
"const": "ValidateAllowPublicSuffix"
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
]
|
|
199
|
+
}
|
|
200
|
+
},
|
|
201
|
+
{
|
|
202
|
+
"not": {
|
|
203
|
+
"allOf": [
|
|
204
|
+
{
|
|
205
|
+
"contains": {
|
|
206
|
+
"const": "ValidateAllowIp"
|
|
207
|
+
}
|
|
208
|
+
},
|
|
209
|
+
{
|
|
210
|
+
"contains": {
|
|
211
|
+
"const": "ValidateAllowPublicSuffix"
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
]
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
],
|
|
117
218
|
"items": {
|
|
118
219
|
"type": "string",
|
|
119
220
|
"enum": [
|
|
@@ -122,6 +223,7 @@
|
|
|
122
223
|
"Compress",
|
|
123
224
|
"Validate",
|
|
124
225
|
"ValidateAllowIp",
|
|
226
|
+
"ValidateAllowPublicSuffix",
|
|
125
227
|
"Deduplicate",
|
|
126
228
|
"InvertAllow",
|
|
127
229
|
"RemoveEmptyLines",
|
|
@@ -2,6 +2,7 @@ const removeComments = require('./remove-comments');
|
|
|
2
2
|
const removeModifiers = require('./remove-modifiers');
|
|
3
3
|
const { validate } = require('./validate');
|
|
4
4
|
const { validateAllowIp } = require('./validate-allow-ip');
|
|
5
|
+
const { validateAllowPublicSuffix } = require('./validate-allow-public-suffix');
|
|
5
6
|
const exclude = require('./exclude');
|
|
6
7
|
const include = require('./include');
|
|
7
8
|
const deduplicate = require('./deduplicate');
|
|
@@ -21,6 +22,7 @@ const TRANSFORMATIONS = Object.freeze({
|
|
|
21
22
|
RemoveModifiers: 'RemoveModifiers',
|
|
22
23
|
Validate: 'Validate',
|
|
23
24
|
ValidateAllowIp: 'ValidateAllowIp',
|
|
25
|
+
ValidateAllowPublicSuffix: 'ValidateAllowPublicSuffix',
|
|
24
26
|
Deduplicate: 'Deduplicate',
|
|
25
27
|
InvertAllow: 'InvertAllow',
|
|
26
28
|
RemoveEmptyLines: 'RemoveEmptyLines',
|
|
@@ -29,6 +31,32 @@ const TRANSFORMATIONS = Object.freeze({
|
|
|
29
31
|
ConvertToAscii: 'ConvertToAscii',
|
|
30
32
|
});
|
|
31
33
|
|
|
34
|
+
const VALIDATION_TRANSFORMATIONS = [
|
|
35
|
+
TRANSFORMATIONS.Validate,
|
|
36
|
+
TRANSFORMATIONS.ValidateAllowIp,
|
|
37
|
+
TRANSFORMATIONS.ValidateAllowPublicSuffix,
|
|
38
|
+
];
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Throws when multiple validation transformations are selected together.
|
|
42
|
+
* Combining them causes silent data loss: each validator runs on the already-filtered
|
|
43
|
+
* output of the previous one, so allow-modes (AllowIp, AllowPublicSuffix) become ineffective.
|
|
44
|
+
*
|
|
45
|
+
* TODO: When a combined mode for ValidateAllowIp + ValidateAllowPublicSuffix is implemented
|
|
46
|
+
* (see https://github.com/AdguardTeam/HostlistCompiler/issues/126),
|
|
47
|
+
* update this check to allow that specific pair and run Validator(true, true) instead.
|
|
48
|
+
*
|
|
49
|
+
* @param {Array<string>} transformations - configured transformations.
|
|
50
|
+
*/
|
|
51
|
+
function checkIncompatibleValidationTransformations(transformations) {
|
|
52
|
+
const selectedValidations = VALIDATION_TRANSFORMATIONS
|
|
53
|
+
.filter((transformation) => transformations.indexOf(transformation) !== -1);
|
|
54
|
+
|
|
55
|
+
if (selectedValidations.length > 1) {
|
|
56
|
+
throw new Error(`Validation transformations cannot be combined: ${selectedValidations.join(', ')}.`);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
32
60
|
/**
|
|
33
61
|
* Applies the specified transformations to the list of rules in the proper order.
|
|
34
62
|
*
|
|
@@ -38,12 +66,14 @@ const TRANSFORMATIONS = Object.freeze({
|
|
|
38
66
|
* @returns {Promise<Array<string>>} rules after applying all transformations.
|
|
39
67
|
*/
|
|
40
68
|
async function transform(rules, configuration, transformations) {
|
|
41
|
-
// If none specified -- apply all
|
|
69
|
+
// If none specified -- apply all transformations
|
|
42
70
|
if (!transformations) {
|
|
43
71
|
// eslint-disable-next-line no-param-reassign
|
|
44
72
|
transformations = [];
|
|
45
73
|
}
|
|
46
74
|
|
|
75
|
+
checkIncompatibleValidationTransformations(transformations);
|
|
76
|
+
|
|
47
77
|
let transformed = rules;
|
|
48
78
|
|
|
49
79
|
transformed = await exclude(
|
|
@@ -57,6 +87,7 @@ async function transform(rules, configuration, transformations) {
|
|
|
57
87
|
configuration.inclusions_sources,
|
|
58
88
|
);
|
|
59
89
|
|
|
90
|
+
// When updating the list of transformations, be sure to follow the same order in README.md
|
|
60
91
|
if (transformations.indexOf(TRANSFORMATIONS.ConvertToAscii) !== -1) {
|
|
61
92
|
transformed = convertToAscii(transformed);
|
|
62
93
|
}
|
|
@@ -81,6 +112,9 @@ async function transform(rules, configuration, transformations) {
|
|
|
81
112
|
if (transformations.indexOf(TRANSFORMATIONS.ValidateAllowIp) !== -1) {
|
|
82
113
|
transformed = validateAllowIp(transformed);
|
|
83
114
|
}
|
|
115
|
+
if (transformations.indexOf(TRANSFORMATIONS.ValidateAllowPublicSuffix) !== -1) {
|
|
116
|
+
transformed = validateAllowPublicSuffix(transformed);
|
|
117
|
+
}
|
|
84
118
|
if (transformations.indexOf(TRANSFORMATIONS.Deduplicate) !== -1) {
|
|
85
119
|
transformed = deduplicate(transformed);
|
|
86
120
|
}
|
|
@@ -10,8 +10,9 @@ const { Validator } = require('./validate');
|
|
|
10
10
|
* @returns {Array<string>} - The filtered array of valid rules, allowing IP addresses.
|
|
11
11
|
*/
|
|
12
12
|
function validateAllowIp(rules) {
|
|
13
|
-
// ip is
|
|
14
|
-
|
|
13
|
+
// ip is allowed for this transformation
|
|
14
|
+
// tld is not allowed by default here
|
|
15
|
+
const validator = new Validator(true, false);
|
|
15
16
|
return validator.validate(rules);
|
|
16
17
|
}
|
|
17
18
|
module.exports = {
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
const { Validator } = require('./validate');
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Validates a list of rules and allows public suffix matching.
|
|
5
|
+
*
|
|
6
|
+
* This function creates an instance of the `Validator` class, enabling public
|
|
7
|
+
* suffix matching, and returns a filtered list of valid rules.
|
|
8
|
+
*
|
|
9
|
+
* @param {Array<string>} rules - The array of rules to validate.
|
|
10
|
+
* @returns {Array<string>} - The filtered array of valid rules.
|
|
11
|
+
*/
|
|
12
|
+
function validateAllowPublicSuffix(rules) {
|
|
13
|
+
// ip is not allowed by default, but public suffix matching is allowed.
|
|
14
|
+
const validator = new Validator(false, true);
|
|
15
|
+
return validator.validate(rules);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
module.exports = {
|
|
19
|
+
validateAllowPublicSuffix,
|
|
20
|
+
};
|