@adguard/hostlist-compiler 2.1.1 → 3.0.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.markdownlint.json +29 -0
- package/.markdownlintignore +1 -0
- package/AGENTS.md +310 -98
- package/CHANGELOG.md +59 -8
- package/DEPLOYMENT.md +171 -0
- package/DEVELOPMENT.md +171 -72
- package/README.md +238 -143
- package/examples/china/README.md +2 -2
- package/examples/energized/README.md +4 -2
- package/examples/sdn/README.md +2 -2
- package/examples/whitelist/README.md +2 -2
- package/package.json +24 -18
- package/src/cli.js +53 -31
- package/src/compile-source.js +4 -3
- package/src/configuration.js +132 -2
- package/src/download.js +27 -0
- package/src/filter.js +2 -2
- package/src/index.d.ts +2 -0
- package/src/index.js +16 -50
- package/src/{transformations/ip-normalize.js → ip-normalize.js} +2 -2
- package/src/schemas/configuration.schema.js +137 -0
- package/src/source-types.js +17 -0
- package/src/transformations/enum.js +49 -0
- package/src/transformations/include.js +1 -1
- package/src/transformations/transform.js +40 -74
- package/src/transformations/validate-allow-ip-and-public-suffix.js +1 -1
- package/src/transformations/validate-allow-ip.js +1 -1
- package/src/utils.js +1284 -0
- package/src/validation-conflicts.js +106 -0
- package/test/cli-error-output.test.js +305 -0
- package/test/cli.test.js +47 -0
- package/test/compile-source.test.js +167 -7
- package/test/configuration.test.js +438 -0
- package/test/download.test.js +260 -0
- package/test/helpers/http-mock.js +94 -0
- package/test/index.test.js +497 -4
- package/test/{transformations/ip-normalize.test.js → ip-normalize.test.js} +1 -1
- package/test/resolve-include.test.js +8 -8
- package/test/transformations/covert-to-ascii.test.js +1 -1
- package/test/transformations/exclude.test.js +2 -2
- package/test/transformations/include.test.js +9 -3
- package/test/transformations/ip-alignment.test.js +1 -1
- package/test/transformations/transform.test.js +66 -1
- package/test/utils.test.js +1559 -0
- package/test/validation-conflicts.test.js +230 -0
- package/src/schemas/configuration.schema.json +0 -369
package/examples/china/README.md
CHANGED
|
@@ -2,9 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
Check [configuration.json](configuration.json) for more details.
|
|
4
4
|
|
|
5
|
-
> Note, that the resulting filter list is too strict for daily use. For
|
|
5
|
+
> Note, that the resulting filter list is too strict for daily use. For
|
|
6
|
+
> instance, it blocks `edgekey.net` which is a commonly used CDN. You need to
|
|
7
|
+
> exclude many rules to make this filter list useable.
|
|
6
8
|
|
|
7
|
-
```
|
|
9
|
+
```bash
|
|
8
10
|
npm i -g @adguard/hostlist-compiler
|
|
9
11
|
|
|
10
12
|
hostlist-compiler -c configuration.json -o filter.txt
|
package/examples/sdn/README.md
CHANGED
package/package.json
CHANGED
|
@@ -9,30 +9,36 @@
|
|
|
9
9
|
"bin": {
|
|
10
10
|
"hostlist-compiler": "src/cli.js"
|
|
11
11
|
},
|
|
12
|
+
"engines": {
|
|
13
|
+
"node": "^22.22.2 || >=24.0.0",
|
|
14
|
+
"pnpm": ">=10.33.4 <11"
|
|
15
|
+
},
|
|
12
16
|
"devDependencies": {
|
|
13
|
-
"@babel/core": "
|
|
14
|
-
"@babel/eslint-parser": "
|
|
15
|
-
"eslint": "
|
|
16
|
-
"eslint-config-airbnb-base": "
|
|
17
|
-
"eslint-plugin-import": "
|
|
18
|
-
"jest": "
|
|
19
|
-
"
|
|
20
|
-
"
|
|
17
|
+
"@babel/core": "7.26.9",
|
|
18
|
+
"@babel/eslint-parser": "7.19.1",
|
|
19
|
+
"eslint": "8.34.0",
|
|
20
|
+
"eslint-config-airbnb-base": "15.0.0",
|
|
21
|
+
"eslint-plugin-import": "2.27.5",
|
|
22
|
+
"jest": "29.7.0",
|
|
23
|
+
"markdownlint": "0.41.1",
|
|
24
|
+
"markdownlint-cli": "0.49.1",
|
|
25
|
+
"mock-fs": "4.11.0",
|
|
26
|
+
"undici": "7.30.1"
|
|
21
27
|
},
|
|
22
28
|
"dependencies": {
|
|
23
|
-
"@adguard/filters-downloader": "
|
|
24
|
-
"ajv": "
|
|
25
|
-
"ajv-errors": "
|
|
26
|
-
"
|
|
27
|
-
"
|
|
28
|
-
"consola": "^2.11.3",
|
|
29
|
-
"lodash": "^4.17.15",
|
|
29
|
+
"@adguard/filters-downloader": "3.1.0",
|
|
30
|
+
"ajv": "6.12.6",
|
|
31
|
+
"better-ajv-errors": "0.6.7",
|
|
32
|
+
"consola": "2.11.3",
|
|
33
|
+
"lodash": "4.17.21",
|
|
30
34
|
"tldts": "7.0.28",
|
|
31
|
-
"yargs": "
|
|
35
|
+
"yargs": "15.3.0"
|
|
32
36
|
},
|
|
33
|
-
"version": "
|
|
37
|
+
"version": "3.0.0-beta.0",
|
|
34
38
|
"scripts": {
|
|
35
|
-
"lint": "
|
|
39
|
+
"lint": "pnpm lint:code && pnpm lint:md",
|
|
40
|
+
"lint:code": "eslint . --cache",
|
|
41
|
+
"lint:md": "markdownlint .",
|
|
36
42
|
"test": "jest --runInBand --detectOpenHandles .",
|
|
37
43
|
"increment": "pnpm version patch --no-git-tag-version",
|
|
38
44
|
"compile": "node src/cli.js"
|
package/src/cli.js
CHANGED
|
@@ -3,6 +3,12 @@
|
|
|
3
3
|
const fs = require('fs').promises;
|
|
4
4
|
const consola = require('consola');
|
|
5
5
|
const compile = require('./index');
|
|
6
|
+
const { createConfiguration } = require('./configuration');
|
|
7
|
+
const {
|
|
8
|
+
sanitizeConfigurationForLog,
|
|
9
|
+
stripErrorCredentials,
|
|
10
|
+
stripUrlCredentials,
|
|
11
|
+
} = require('./utils');
|
|
6
12
|
const packageJson = require('../package.json');
|
|
7
13
|
|
|
8
14
|
// Version is not stored in the repo: CI stamps it into package.json at build
|
|
@@ -36,7 +42,7 @@ const { argv } = require('yargs')
|
|
|
36
42
|
.option('input-type', {
|
|
37
43
|
alias: 't',
|
|
38
44
|
type: 'string',
|
|
39
|
-
description: 'Type of the input file (/etc/hosts,
|
|
45
|
+
description: 'Type of the input file (/etc/hosts, adblock)',
|
|
40
46
|
nargs: 1,
|
|
41
47
|
})
|
|
42
48
|
.option('output', {
|
|
@@ -78,48 +84,64 @@ async function readConfig() {
|
|
|
78
84
|
}
|
|
79
85
|
|
|
80
86
|
/**
|
|
81
|
-
*
|
|
82
|
-
* We assume that the input is a /etc/hosts file.
|
|
87
|
+
* Logs an error and every error in its `cause` and `errors` chains.
|
|
83
88
|
*
|
|
84
|
-
*
|
|
89
|
+
* `fetch` wraps the original network error, so the useful message can be
|
|
90
|
+
* several levels deep. A connection failure to a host with both an IPv4 and an
|
|
91
|
+
* IPv6 address is an `AggregateError` with an empty message, and its useful
|
|
92
|
+
* messages are in the `errors` array. The causes are logged as strings because
|
|
93
|
+
* `consola` throttles identical log entries, and different error objects
|
|
94
|
+
* serialize to the same empty value.
|
|
95
|
+
*
|
|
96
|
+
* @param {Error} error - error to log.
|
|
85
97
|
*/
|
|
86
|
-
function
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
'InsertFinalNewLine',
|
|
101
|
-
],
|
|
102
|
-
};
|
|
98
|
+
function logError(error) {
|
|
99
|
+
stripErrorCredentials(error);
|
|
100
|
+
consola.error(error);
|
|
101
|
+
|
|
102
|
+
const logged = new Set();
|
|
103
|
+
|
|
104
|
+
const logCause = (cause) => {
|
|
105
|
+
if (cause === null || cause === undefined || logged.has(cause)) {
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
if (typeof cause === 'object') {
|
|
110
|
+
logged.add(cause);
|
|
111
|
+
}
|
|
103
112
|
|
|
104
|
-
|
|
105
|
-
config.sources.push({
|
|
106
|
-
source: input,
|
|
107
|
-
type: inputType,
|
|
108
|
-
});
|
|
109
|
-
});
|
|
113
|
+
const isAggregate = Array.isArray(cause.errors) && cause.errors.length > 0;
|
|
110
114
|
|
|
111
|
-
|
|
115
|
+
if (!isAggregate || cause.message) {
|
|
116
|
+
consola.error(`Caused by: ${stripUrlCredentials(cause.message || cause)}`);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
if (isAggregate) {
|
|
120
|
+
cause.errors.forEach(logCause);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
logCause(cause.cause);
|
|
124
|
+
};
|
|
125
|
+
|
|
126
|
+
logCause(error.cause);
|
|
112
127
|
}
|
|
113
128
|
|
|
129
|
+
/**
|
|
130
|
+
* Runs the compiler: builds the configuration from the CLI options (or reads
|
|
131
|
+
* it from the config file) and writes the compiled list to the output file.
|
|
132
|
+
*/
|
|
114
133
|
async function main() {
|
|
115
134
|
try {
|
|
116
135
|
if (!argv.input && !argv.config) {
|
|
117
136
|
throw new Error('Either --input or --config must be specified');
|
|
118
137
|
}
|
|
119
138
|
|
|
120
|
-
const config = argv.input
|
|
139
|
+
const config = argv.input
|
|
140
|
+
? createConfiguration(argv.input, argv['input-type'])
|
|
141
|
+
: await readConfig();
|
|
121
142
|
|
|
122
|
-
|
|
143
|
+
const sanitizedConfigDump = JSON.stringify(sanitizeConfigurationForLog(config), 0, 4);
|
|
144
|
+
consola.debug(`Configuration: ${sanitizedConfigDump}`);
|
|
123
145
|
|
|
124
146
|
const lines = await compile(config);
|
|
125
147
|
|
|
@@ -127,7 +149,7 @@ async function main() {
|
|
|
127
149
|
await fs.writeFile(argv.output, lines.join('\n'));
|
|
128
150
|
consola.info('Finished compiling');
|
|
129
151
|
} catch (ex) {
|
|
130
|
-
|
|
152
|
+
logError(ex);
|
|
131
153
|
process.exit(1);
|
|
132
154
|
}
|
|
133
155
|
}
|
package/src/compile-source.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
const consola = require('consola');
|
|
2
|
-
const {
|
|
2
|
+
const { download } = require('./download');
|
|
3
3
|
const { transform } = require('./transformations/transform');
|
|
4
|
+
const { stripKnownUrlCredentials } = require('./utils');
|
|
4
5
|
|
|
5
6
|
/**
|
|
6
7
|
* A source configuration. See a better description in README.md
|
|
@@ -20,8 +21,8 @@ const { transform } = require('./transformations/transform');
|
|
|
20
21
|
* @returns {Promise<Array<string>>} array with the source rules
|
|
21
22
|
*/
|
|
22
23
|
async function compileSource(source) {
|
|
23
|
-
consola.info(`Start compiling ${source.source}`);
|
|
24
|
-
let rules = await
|
|
24
|
+
consola.info(`Start compiling ${stripKnownUrlCredentials(source.source)}`);
|
|
25
|
+
let rules = await download(source.source);
|
|
25
26
|
consola.info(`Original length is ${rules.length}`);
|
|
26
27
|
|
|
27
28
|
rules = await transform(rules, source, source.transformations);
|
package/src/configuration.js
CHANGED
|
@@ -1,10 +1,138 @@
|
|
|
1
1
|
const Ajv = require('ajv');
|
|
2
2
|
const betterAjvErrors = require('better-ajv-errors');
|
|
3
|
+
const { SOURCE_TYPES } = require('./source-types');
|
|
4
|
+
const { DEFAULT_TRANSFORMATIONS } = require('./transformations/transform');
|
|
5
|
+
const {
|
|
6
|
+
sanitizeConfigurationForLog,
|
|
7
|
+
stripKnownUrlCredentials,
|
|
8
|
+
} = require('./utils');
|
|
3
9
|
|
|
4
10
|
const ajv = new Ajv({ allErrors: true, jsonPointers: true });
|
|
5
|
-
const schema = require('./schemas/configuration.schema
|
|
11
|
+
const schema = require('./schemas/configuration.schema');
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Escapes one segment of a JSON pointer. `better-ajv-errors` builds the
|
|
15
|
+
* pointer to an additional property as `${dataPath}/${additionalProperty}`
|
|
16
|
+
* without escaping, so a key with `/` or `~` would not resolve.
|
|
17
|
+
*
|
|
18
|
+
* @param {String} value - the property name.
|
|
19
|
+
* @returns {String} the escaped pointer segment.
|
|
20
|
+
*/
|
|
21
|
+
function escapeJsonPointerSegment(value) {
|
|
22
|
+
return value.replace(/~/g, '~0').replace(/\//g, '~1');
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Prepares `validate.errors` for `better-ajv-errors` against the sanitized
|
|
27
|
+
* configuration copy. An additional property is renamed in the sanitized copy
|
|
28
|
+
* (credentials are removed from keys), so the error must point at the
|
|
29
|
+
* sanitized name. The property name is also escaped as a JSON pointer segment,
|
|
30
|
+
* because `better-ajv-errors` does not escape it when it resolves the code
|
|
31
|
+
* frame. The returned replacements restore the readable property name in the
|
|
32
|
+
* rendered text.
|
|
33
|
+
*
|
|
34
|
+
* @param {Array<Object>} errors - the AJV errors.
|
|
35
|
+
* @returns {{ errors: Array<Object>, replacements: Array<Array<String>> }}
|
|
36
|
+
* the errors to render and the pointer-to-name replacements.
|
|
37
|
+
*/
|
|
38
|
+
function prepareValidationErrors(errors) {
|
|
39
|
+
const replacements = [];
|
|
40
|
+
const preparedErrors = errors.map((error) => {
|
|
41
|
+
const additionalProperty = error.params && error.params.additionalProperty;
|
|
42
|
+
|
|
43
|
+
if (typeof additionalProperty !== 'string') {
|
|
44
|
+
return error;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const sanitizedProperty = stripKnownUrlCredentials(additionalProperty);
|
|
48
|
+
const pointerSegment = escapeJsonPointerSegment(sanitizedProperty);
|
|
49
|
+
|
|
50
|
+
if (pointerSegment !== sanitizedProperty) {
|
|
51
|
+
replacements.push([pointerSegment, sanitizedProperty]);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
if (pointerSegment === additionalProperty) {
|
|
55
|
+
return error;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
return {
|
|
59
|
+
...error,
|
|
60
|
+
params: {
|
|
61
|
+
...error.params,
|
|
62
|
+
additionalProperty: pointerSegment,
|
|
63
|
+
},
|
|
64
|
+
};
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
// Replace a longer pointer first, so a pointer that is a prefix of another
|
|
68
|
+
// does not corrupt the longer one.
|
|
69
|
+
replacements.sort(([first], [second]) => second.length - first.length);
|
|
70
|
+
|
|
71
|
+
return { errors: preparedErrors, replacements };
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Restores the readable property names in the rendered validation text after
|
|
76
|
+
* `better-ajv-errors` used their escaped pointer segments.
|
|
77
|
+
*
|
|
78
|
+
* @param {String} errorsText - the rendered validation errors.
|
|
79
|
+
* @param {Array<Array<String>>} replacements - pairs of a pointer segment and
|
|
80
|
+
* the property name.
|
|
81
|
+
* @returns {String} the text with readable property names.
|
|
82
|
+
*/
|
|
83
|
+
function restorePropertyNames(errorsText, replacements) {
|
|
84
|
+
return replacements.reduce(
|
|
85
|
+
(text, [pointerSegment, property]) => text.split(pointerSegment).join(property),
|
|
86
|
+
errorsText,
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Renders the AJV errors against a sanitized configuration copy.
|
|
92
|
+
*
|
|
93
|
+
* `better-ajv-errors` embeds the configuration in the message, and it renders
|
|
94
|
+
* object keys and `toJSON` results, so the copy must be JSON-safe.
|
|
95
|
+
*
|
|
96
|
+
* @param {*} configuration - the configuration that failed validation.
|
|
97
|
+
* @param {Array<Object>} errors - the AJV errors.
|
|
98
|
+
* @returns {String} the rendered validation errors.
|
|
99
|
+
*/
|
|
100
|
+
function formatValidationErrors(configuration, errors) {
|
|
101
|
+
const prepared = prepareValidationErrors(errors);
|
|
102
|
+
const errorsText = betterAjvErrors(
|
|
103
|
+
schema,
|
|
104
|
+
sanitizeConfigurationForLog(configuration),
|
|
105
|
+
prepared.errors,
|
|
106
|
+
);
|
|
107
|
+
|
|
108
|
+
return restorePropertyNames(errorsText, prepared.replacements);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Creates a configuration object for the quick conversion mode (the CLI `-i`
|
|
113
|
+
* option): compiles the specified sources with the default transformation
|
|
114
|
+
* pipeline.
|
|
115
|
+
*
|
|
116
|
+
* @param {Array<string>} inputs - list of the source paths or URLs.
|
|
117
|
+
* @param {string} [type] - type of the input sources (`adblock` or `hosts`),
|
|
118
|
+
* defaults to `hosts`.
|
|
119
|
+
* @returns {Object} configuration object.
|
|
120
|
+
*/
|
|
121
|
+
function createConfiguration(inputs, type = SOURCE_TYPES.HOSTS) {
|
|
122
|
+
return {
|
|
123
|
+
name: 'Blocklist',
|
|
124
|
+
sources: inputs.map((input) => ({
|
|
125
|
+
source: input,
|
|
126
|
+
// An empty string (e.g. the CLI flag `-t ''`) must fall back to
|
|
127
|
+
// the default type, like the old `argv['input-type'] || 'hosts'`.
|
|
128
|
+
type: type || SOURCE_TYPES.HOSTS,
|
|
129
|
+
})),
|
|
130
|
+
transformations: [...DEFAULT_TRANSFORMATIONS],
|
|
131
|
+
};
|
|
132
|
+
}
|
|
6
133
|
|
|
7
134
|
module.exports = {
|
|
135
|
+
createConfiguration,
|
|
8
136
|
/**
|
|
9
137
|
* Validates the specified configuration object
|
|
10
138
|
*
|
|
@@ -16,7 +144,9 @@ module.exports = {
|
|
|
16
144
|
const valid = validate(configuration);
|
|
17
145
|
return {
|
|
18
146
|
valid,
|
|
19
|
-
|
|
147
|
+
// The validation itself runs against the original configuration,
|
|
148
|
+
// but the rendered text is built from a sanitized copy.
|
|
149
|
+
errorsText: valid ? null : formatValidationErrors(configuration, validate.errors),
|
|
20
150
|
};
|
|
21
151
|
},
|
|
22
152
|
};
|
package/src/download.js
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
const { FiltersDownloader } = require('@adguard/filters-downloader');
|
|
2
|
+
const { stripErrorCredentials, stripKnownUrlCredentials } = require('./utils');
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Downloads the specified source (a URL or a local file path) and
|
|
6
|
+
* resolves all `!#include` directives in it.
|
|
7
|
+
*
|
|
8
|
+
* @param {String} source - URL or local file path of the source.
|
|
9
|
+
* @returns {Promise<Array<String>>} array with the source rules.
|
|
10
|
+
*/
|
|
11
|
+
async function download(source) {
|
|
12
|
+
try {
|
|
13
|
+
return await FiltersDownloader.download(source, {}, { allowEmptyResponse: true });
|
|
14
|
+
} catch (error) {
|
|
15
|
+
// The downloader gets the original URL so it can send Basic auth, but
|
|
16
|
+
// its errors may contain that URL. Replace the exact source first: a
|
|
17
|
+
// password can contain a literal `@` followed by whitespace, and the
|
|
18
|
+
// generic scanner can leave the rest of such a password in the text.
|
|
19
|
+
// Then sanitize the whole chain before the error leaves the download
|
|
20
|
+
// layer.
|
|
21
|
+
throw stripErrorCredentials(error, [[source, stripKnownUrlCredentials(source)]]);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
module.exports = {
|
|
26
|
+
download,
|
|
27
|
+
};
|
package/src/filter.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
const _ = require('lodash');
|
|
2
|
-
const {
|
|
2
|
+
const { download } = require('./download');
|
|
3
3
|
const utils = require('./utils');
|
|
4
4
|
const ruleUtils = require('./rule');
|
|
5
5
|
|
|
@@ -17,7 +17,7 @@ async function downloadAll(sources) {
|
|
|
17
17
|
}
|
|
18
18
|
|
|
19
19
|
await Promise.all(sources.map(async (source) => {
|
|
20
|
-
const rulesStr = await
|
|
20
|
+
const rulesStr = await download(source);
|
|
21
21
|
const rules = rulesStr
|
|
22
22
|
.filter((el) => el.trim().length > 0 && !ruleUtils.isComment(el));
|
|
23
23
|
list = list.concat(rules);
|
package/src/index.d.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
declare module '@adguard/hostlist-compiler' {
|
|
2
|
+
// Keep in sync with SOURCE_TYPES in src/source-types.js (type declarations cannot consume runtime values).
|
|
2
3
|
export type SourceType = 'adblock' | 'hosts';
|
|
3
4
|
|
|
5
|
+
// Keep in sync with TRANSFORMATIONS in src/transformations/enum.js (type declarations cannot consume runtime values).
|
|
4
6
|
export type Transformation =
|
|
5
7
|
'ConvertToAscii' |
|
|
6
8
|
'TrimLines' |
|
package/src/index.js
CHANGED
|
@@ -2,7 +2,13 @@ const _ = require('lodash');
|
|
|
2
2
|
const consola = require('consola');
|
|
3
3
|
const config = require('./configuration');
|
|
4
4
|
const compileSource = require('./compile-source');
|
|
5
|
-
const { transform
|
|
5
|
+
const { transform } = require('./transformations/transform');
|
|
6
|
+
const { checkCrossLevelValidationConflicts } = require('./validation-conflicts');
|
|
7
|
+
const {
|
|
8
|
+
stripUrlValueCredentials,
|
|
9
|
+
stripKnownUrlCredentials,
|
|
10
|
+
sanitizeConfigurationForLog,
|
|
11
|
+
} = require('./utils');
|
|
6
12
|
const packageJson = require('../package.json');
|
|
7
13
|
|
|
8
14
|
// Version is not stored in the repo: CI stamps it into package.json at build
|
|
@@ -19,20 +25,20 @@ const packageVersion = packageJson.version || '0.0.0';
|
|
|
19
25
|
function prepareHeader(configuration) {
|
|
20
26
|
const lines = [
|
|
21
27
|
'!',
|
|
22
|
-
`! Title: ${configuration.name}`,
|
|
28
|
+
`! Title: ${stripUrlValueCredentials(configuration.name)}`,
|
|
23
29
|
];
|
|
24
30
|
|
|
25
31
|
if (!_.isEmpty(configuration.description)) {
|
|
26
|
-
lines.push(`! Description: ${configuration.description}`);
|
|
32
|
+
lines.push(`! Description: ${stripUrlValueCredentials(configuration.description)}`);
|
|
27
33
|
}
|
|
28
34
|
if (!_.isEmpty(configuration.version)) {
|
|
29
|
-
lines.push(`! Version: ${configuration.version}`);
|
|
35
|
+
lines.push(`! Version: ${stripUrlValueCredentials(configuration.version)}`);
|
|
30
36
|
}
|
|
31
37
|
if (!_.isEmpty(configuration.homepage)) {
|
|
32
|
-
lines.push(`! Homepage: ${configuration.homepage}`);
|
|
38
|
+
lines.push(`! Homepage: ${stripKnownUrlCredentials(configuration.homepage)}`);
|
|
33
39
|
}
|
|
34
40
|
if (!_.isEmpty(configuration.license)) {
|
|
35
|
-
lines.push(`! License: ${configuration.license}`);
|
|
41
|
+
lines.push(`! License: ${stripUrlValueCredentials(configuration.license)}`);
|
|
36
42
|
}
|
|
37
43
|
|
|
38
44
|
lines.push(`! Last modified: ${(new Date()).toISOString()}`);
|
|
@@ -60,54 +66,13 @@ function prepareSourceHeader(source) {
|
|
|
60
66
|
];
|
|
61
67
|
|
|
62
68
|
if (!_.isEmpty(source.name)) {
|
|
63
|
-
lines.push(`! Source name: ${source.name}`);
|
|
69
|
+
lines.push(`! Source name: ${stripUrlValueCredentials(source.name)}`);
|
|
64
70
|
}
|
|
65
|
-
lines.push(`! Source: ${source.source}`);
|
|
71
|
+
lines.push(`! Source: ${stripKnownUrlCredentials(source.source)}`);
|
|
66
72
|
lines.push('!');
|
|
67
73
|
return lines;
|
|
68
74
|
}
|
|
69
75
|
|
|
70
|
-
/**
|
|
71
|
-
* Checks for conflicting validation transformations across source-level and top-level.
|
|
72
|
-
* Throws if a validation transformation is used at both levels, as this causes silent data loss.
|
|
73
|
-
*
|
|
74
|
-
* @param {*} configuration - compilation configuration.
|
|
75
|
-
*/
|
|
76
|
-
function checkCrossLevelValidationConflicts(configuration) {
|
|
77
|
-
const VALIDATION_TRANSFORMATIONS = [
|
|
78
|
-
TRANSFORMATIONS.Validate,
|
|
79
|
-
TRANSFORMATIONS.ValidateAllowIp,
|
|
80
|
-
TRANSFORMATIONS.ValidateAllowPublicSuffix,
|
|
81
|
-
TRANSFORMATIONS.ValidateAllowIpAndPublicSuffix,
|
|
82
|
-
];
|
|
83
|
-
|
|
84
|
-
const topLevelTransformations = configuration.transformations || [];
|
|
85
|
-
const topLevelValidations = VALIDATION_TRANSFORMATIONS
|
|
86
|
-
.filter((t) => topLevelTransformations.indexOf(t) !== -1);
|
|
87
|
-
|
|
88
|
-
if (topLevelValidations.length === 0) {
|
|
89
|
-
return;
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
// Check each source for conflicting validations
|
|
93
|
-
// eslint-disable-next-line no-restricted-syntax
|
|
94
|
-
for (const source of configuration.sources) {
|
|
95
|
-
const sourceTransformations = source.transformations || [];
|
|
96
|
-
const sourceValidations = VALIDATION_TRANSFORMATIONS
|
|
97
|
-
.filter((t) => sourceTransformations.indexOf(t) !== -1);
|
|
98
|
-
|
|
99
|
-
if (sourceValidations.length > 0) {
|
|
100
|
-
throw new Error(
|
|
101
|
-
'Validation transformations cannot be used at both source and top level. '
|
|
102
|
-
+ `Source "${source.source}" uses [${sourceValidations.join(', ')}], `
|
|
103
|
-
+ `but top level uses [${topLevelValidations.join(', ')}]. `
|
|
104
|
-
+ 'This causes the source-level validation to be overridden by the top-level one, '
|
|
105
|
-
+ 'making the source-level validation ineffective.',
|
|
106
|
-
);
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
|
|
111
76
|
/**
|
|
112
77
|
* Compiles a filter list using the specified configuration.
|
|
113
78
|
*
|
|
@@ -125,7 +90,8 @@ async function compile(configuration) {
|
|
|
125
90
|
|
|
126
91
|
checkCrossLevelValidationConflicts(configuration);
|
|
127
92
|
|
|
128
|
-
|
|
93
|
+
const configurationDump = JSON.stringify(sanitizeConfigurationForLog(configuration), 0, 4);
|
|
94
|
+
consola.info(`Configuration: ${configurationDump}`);
|
|
129
95
|
|
|
130
96
|
// This will be the final list of rules
|
|
131
97
|
let finalList = [];
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
const consola = require('consola');
|
|
2
|
-
const ruleUtils = require('
|
|
3
|
-
const { classifyIpPattern, parseIpPattern } = require('
|
|
2
|
+
const ruleUtils = require('./rule');
|
|
3
|
+
const { classifyIpPattern, parseIpPattern } = require('./utils');
|
|
4
4
|
|
|
5
5
|
// Prefix for canonical IP/domain patterns
|
|
6
6
|
const IP_PREFIX = '||';
|