@adguard/hostlist-compiler 2.1.0 → 3.0.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dockerignore +29 -0
- package/.github/workflows/ci.yml +22 -0
- package/.github/workflows/mirror.yml +24 -0
- package/.github/workflows/prepare-release.yml +26 -0
- package/.github/workflows/publish-release.yml +44 -0
- package/.markdownlint.json +29 -0
- package/.markdownlintignore +1 -0
- package/AGENTS.md +314 -103
- package/CHANGELOG.md +76 -7
- package/DEPLOYMENT.md +171 -0
- package/DEVELOPMENT.md +174 -76
- package/README.md +240 -140
- package/examples/china/README.md +2 -2
- package/examples/energized/README.md +4 -2
- package/examples/sdn/README.md +2 -2
- package/examples/whitelist/README.md +2 -2
- package/package.json +29 -24
- package/src/cli.js +59 -33
- package/src/compile-source.js +4 -3
- package/src/configuration.js +132 -2
- package/src/download.js +27 -0
- package/src/filter.js +2 -2
- package/src/index.d.ts +2 -0
- package/src/index.js +21 -51
- package/src/{transformations/ip-normalize.js → ip-normalize.js} +2 -2
- package/src/schemas/configuration.schema.js +137 -0
- package/src/source-types.js +17 -0
- package/src/transformations/enum.js +49 -0
- package/src/transformations/include.js +1 -1
- package/src/transformations/transform.js +40 -74
- package/src/transformations/validate-allow-ip-and-public-suffix.js +1 -1
- package/src/transformations/validate-allow-ip.js +1 -1
- package/src/utils.js +1284 -0
- package/src/validation-conflicts.js +106 -0
- package/test/cli-error-output.test.js +305 -0
- package/test/cli.test.js +47 -0
- package/test/compile-source.test.js +167 -7
- package/test/configuration.test.js +438 -0
- package/test/download.test.js +260 -0
- package/test/helpers/http-mock.js +94 -0
- package/test/index.test.js +497 -4
- package/test/{transformations/ip-normalize.test.js → ip-normalize.test.js} +1 -1
- package/test/resolve-include.test.js +8 -8
- package/test/transformations/covert-to-ascii.test.js +1 -1
- package/test/transformations/exclude.test.js +2 -2
- package/test/transformations/include.test.js +9 -3
- package/test/transformations/ip-alignment.test.js +1 -1
- package/test/transformations/transform.test.js +66 -1
- package/test/utils.test.js +1559 -0
- package/test/validation-conflicts.test.js +230 -0
- package/bamboo-specs/bamboo.yaml +0 -14
- package/bamboo-specs/build.yaml +0 -92
- package/bamboo-specs/deploy.yaml +0 -38
- package/bamboo-specs/increment.yaml +0 -54
- package/bamboo-specs/permissions.yaml +0 -16
- package/bamboo-specs/test.yaml +0 -56
- package/src/schemas/configuration.schema.json +0 -369
- package/tools/build-txt.js +0 -19
package/examples/china/README.md
CHANGED
|
@@ -2,9 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
Check [configuration.json](configuration.json) for more details.
|
|
4
4
|
|
|
5
|
-
> Note, that the resulting filter list is too strict for daily use. For
|
|
5
|
+
> Note, that the resulting filter list is too strict for daily use. For
|
|
6
|
+
> instance, it blocks `edgekey.net` which is a commonly used CDN. You need to
|
|
7
|
+
> exclude many rules to make this filter list useable.
|
|
6
8
|
|
|
7
|
-
```
|
|
9
|
+
```bash
|
|
8
10
|
npm i -g @adguard/hostlist-compiler
|
|
9
11
|
|
|
10
12
|
hostlist-compiler -c configuration.json -o filter.txt
|
package/examples/sdn/README.md
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adguard/hostlist-compiler",
|
|
3
|
-
"version": "2.1.0",
|
|
4
3
|
"description": "A simple tool that compiles hosts blocklists from multiple sources",
|
|
5
4
|
"main": "src/index.js",
|
|
6
5
|
"types": "src/index.d.ts",
|
|
@@ -10,32 +9,38 @@
|
|
|
10
9
|
"bin": {
|
|
11
10
|
"hostlist-compiler": "src/cli.js"
|
|
12
11
|
},
|
|
13
|
-
"
|
|
14
|
-
"
|
|
15
|
-
"
|
|
16
|
-
"increment": "yarn version --patch --no-git-tag-version",
|
|
17
|
-
"build-txt": "node tools/build-txt.js",
|
|
18
|
-
"compile": "node src/cli.js"
|
|
12
|
+
"engines": {
|
|
13
|
+
"node": "^22.22.2 || >=24.0.0",
|
|
14
|
+
"pnpm": ">=10.33.4 <11"
|
|
19
15
|
},
|
|
20
16
|
"devDependencies": {
|
|
21
|
-
"@babel/core": "
|
|
22
|
-
"@babel/eslint-parser": "
|
|
23
|
-
"eslint": "
|
|
24
|
-
"eslint-config-airbnb-base": "
|
|
25
|
-
"eslint-plugin-import": "
|
|
26
|
-
"jest": "
|
|
27
|
-
"
|
|
28
|
-
"
|
|
17
|
+
"@babel/core": "7.26.9",
|
|
18
|
+
"@babel/eslint-parser": "7.19.1",
|
|
19
|
+
"eslint": "8.34.0",
|
|
20
|
+
"eslint-config-airbnb-base": "15.0.0",
|
|
21
|
+
"eslint-plugin-import": "2.27.5",
|
|
22
|
+
"jest": "29.7.0",
|
|
23
|
+
"markdownlint": "0.41.1",
|
|
24
|
+
"markdownlint-cli": "0.49.1",
|
|
25
|
+
"mock-fs": "4.11.0",
|
|
26
|
+
"undici": "7.30.1"
|
|
29
27
|
},
|
|
30
28
|
"dependencies": {
|
|
31
|
-
"@adguard/filters-downloader": "
|
|
32
|
-
"ajv": "
|
|
33
|
-
"ajv-errors": "
|
|
34
|
-
"
|
|
35
|
-
"
|
|
36
|
-
"consola": "^2.11.3",
|
|
37
|
-
"lodash": "^4.17.15",
|
|
29
|
+
"@adguard/filters-downloader": "3.1.0",
|
|
30
|
+
"ajv": "6.12.6",
|
|
31
|
+
"better-ajv-errors": "0.6.7",
|
|
32
|
+
"consola": "2.11.3",
|
|
33
|
+
"lodash": "4.17.21",
|
|
38
34
|
"tldts": "7.0.28",
|
|
39
|
-
"yargs": "
|
|
35
|
+
"yargs": "15.3.0"
|
|
36
|
+
},
|
|
37
|
+
"version": "3.0.0-beta.0",
|
|
38
|
+
"scripts": {
|
|
39
|
+
"lint": "pnpm lint:code && pnpm lint:md",
|
|
40
|
+
"lint:code": "eslint . --cache",
|
|
41
|
+
"lint:md": "markdownlint .",
|
|
42
|
+
"test": "jest --runInBand --detectOpenHandles .",
|
|
43
|
+
"increment": "pnpm version patch --no-git-tag-version",
|
|
44
|
+
"compile": "node src/cli.js"
|
|
40
45
|
}
|
|
41
|
-
}
|
|
46
|
+
}
|
package/src/cli.js
CHANGED
|
@@ -3,8 +3,18 @@
|
|
|
3
3
|
const fs = require('fs').promises;
|
|
4
4
|
const consola = require('consola');
|
|
5
5
|
const compile = require('./index');
|
|
6
|
+
const { createConfiguration } = require('./configuration');
|
|
7
|
+
const {
|
|
8
|
+
sanitizeConfigurationForLog,
|
|
9
|
+
stripErrorCredentials,
|
|
10
|
+
stripUrlCredentials,
|
|
11
|
+
} = require('./utils');
|
|
6
12
|
const packageJson = require('../package.json');
|
|
7
13
|
|
|
14
|
+
// Version is not stored in the repo: CI stamps it into package.json at build
|
|
15
|
+
// time (see publish-release.yml). Fall back to 0.0.0 in development.
|
|
16
|
+
const packageVersion = packageJson.version || '0.0.0';
|
|
17
|
+
|
|
8
18
|
// eslint-disable-next-line import/order
|
|
9
19
|
const { argv } = require('yargs')
|
|
10
20
|
.usage('Usage: $0 [options]')
|
|
@@ -32,7 +42,7 @@ const { argv } = require('yargs')
|
|
|
32
42
|
.option('input-type', {
|
|
33
43
|
alias: 't',
|
|
34
44
|
type: 'string',
|
|
35
|
-
description: 'Type of the input file (/etc/hosts,
|
|
45
|
+
description: 'Type of the input file (/etc/hosts, adblock)',
|
|
36
46
|
nargs: 1,
|
|
37
47
|
})
|
|
38
48
|
.option('output', {
|
|
@@ -47,7 +57,7 @@ const { argv } = require('yargs')
|
|
|
47
57
|
description: 'Run with verbose logging',
|
|
48
58
|
})
|
|
49
59
|
.demandOption(['o'])
|
|
50
|
-
.version()
|
|
60
|
+
.version(packageVersion)
|
|
51
61
|
.help('h')
|
|
52
62
|
.alias('h', 'help');
|
|
53
63
|
|
|
@@ -56,7 +66,7 @@ if (argv.verbose) {
|
|
|
56
66
|
consola.level = 5;
|
|
57
67
|
}
|
|
58
68
|
|
|
59
|
-
consola.info(`Starting ${packageJson.name} v${
|
|
69
|
+
consola.info(`Starting ${packageJson.name} v${packageVersion}`);
|
|
60
70
|
|
|
61
71
|
/**
|
|
62
72
|
* Reads the configuration file and returns the parsed JSON.
|
|
@@ -74,48 +84,64 @@ async function readConfig() {
|
|
|
74
84
|
}
|
|
75
85
|
|
|
76
86
|
/**
|
|
77
|
-
*
|
|
78
|
-
*
|
|
87
|
+
* Logs an error and every error in its `cause` and `errors` chains.
|
|
88
|
+
*
|
|
89
|
+
* `fetch` wraps the original network error, so the useful message can be
|
|
90
|
+
* several levels deep. A connection failure to a host with both an IPv4 and an
|
|
91
|
+
* IPv6 address is an `AggregateError` with an empty message, and its useful
|
|
92
|
+
* messages are in the `errors` array. The causes are logged as strings because
|
|
93
|
+
* `consola` throttles identical log entries, and different error objects
|
|
94
|
+
* serialize to the same empty value.
|
|
79
95
|
*
|
|
80
|
-
* @
|
|
96
|
+
* @param {Error} error - error to log.
|
|
81
97
|
*/
|
|
82
|
-
function
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
'InsertFinalNewLine',
|
|
97
|
-
],
|
|
98
|
-
};
|
|
98
|
+
function logError(error) {
|
|
99
|
+
stripErrorCredentials(error);
|
|
100
|
+
consola.error(error);
|
|
101
|
+
|
|
102
|
+
const logged = new Set();
|
|
103
|
+
|
|
104
|
+
const logCause = (cause) => {
|
|
105
|
+
if (cause === null || cause === undefined || logged.has(cause)) {
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
if (typeof cause === 'object') {
|
|
110
|
+
logged.add(cause);
|
|
111
|
+
}
|
|
99
112
|
|
|
100
|
-
|
|
101
|
-
config.sources.push({
|
|
102
|
-
source: input,
|
|
103
|
-
type: inputType,
|
|
104
|
-
});
|
|
105
|
-
});
|
|
113
|
+
const isAggregate = Array.isArray(cause.errors) && cause.errors.length > 0;
|
|
106
114
|
|
|
107
|
-
|
|
115
|
+
if (!isAggregate || cause.message) {
|
|
116
|
+
consola.error(`Caused by: ${stripUrlCredentials(cause.message || cause)}`);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
if (isAggregate) {
|
|
120
|
+
cause.errors.forEach(logCause);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
logCause(cause.cause);
|
|
124
|
+
};
|
|
125
|
+
|
|
126
|
+
logCause(error.cause);
|
|
108
127
|
}
|
|
109
128
|
|
|
129
|
+
/**
|
|
130
|
+
* Runs the compiler: builds the configuration from the CLI options (or reads
|
|
131
|
+
* it from the config file) and writes the compiled list to the output file.
|
|
132
|
+
*/
|
|
110
133
|
async function main() {
|
|
111
134
|
try {
|
|
112
135
|
if (!argv.input && !argv.config) {
|
|
113
136
|
throw new Error('Either --input or --config must be specified');
|
|
114
137
|
}
|
|
115
138
|
|
|
116
|
-
const config = argv.input
|
|
139
|
+
const config = argv.input
|
|
140
|
+
? createConfiguration(argv.input, argv['input-type'])
|
|
141
|
+
: await readConfig();
|
|
117
142
|
|
|
118
|
-
|
|
143
|
+
const sanitizedConfigDump = JSON.stringify(sanitizeConfigurationForLog(config), 0, 4);
|
|
144
|
+
consola.debug(`Configuration: ${sanitizedConfigDump}`);
|
|
119
145
|
|
|
120
146
|
const lines = await compile(config);
|
|
121
147
|
|
|
@@ -123,7 +149,7 @@ async function main() {
|
|
|
123
149
|
await fs.writeFile(argv.output, lines.join('\n'));
|
|
124
150
|
consola.info('Finished compiling');
|
|
125
151
|
} catch (ex) {
|
|
126
|
-
|
|
152
|
+
logError(ex);
|
|
127
153
|
process.exit(1);
|
|
128
154
|
}
|
|
129
155
|
}
|
package/src/compile-source.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
const consola = require('consola');
|
|
2
|
-
const {
|
|
2
|
+
const { download } = require('./download');
|
|
3
3
|
const { transform } = require('./transformations/transform');
|
|
4
|
+
const { stripKnownUrlCredentials } = require('./utils');
|
|
4
5
|
|
|
5
6
|
/**
|
|
6
7
|
* A source configuration. See a better description in README.md
|
|
@@ -20,8 +21,8 @@ const { transform } = require('./transformations/transform');
|
|
|
20
21
|
* @returns {Promise<Array<string>>} array with the source rules
|
|
21
22
|
*/
|
|
22
23
|
async function compileSource(source) {
|
|
23
|
-
consola.info(`Start compiling ${source.source}`);
|
|
24
|
-
let rules = await
|
|
24
|
+
consola.info(`Start compiling ${stripKnownUrlCredentials(source.source)}`);
|
|
25
|
+
let rules = await download(source.source);
|
|
25
26
|
consola.info(`Original length is ${rules.length}`);
|
|
26
27
|
|
|
27
28
|
rules = await transform(rules, source, source.transformations);
|
package/src/configuration.js
CHANGED
|
@@ -1,10 +1,138 @@
|
|
|
1
1
|
const Ajv = require('ajv');
|
|
2
2
|
const betterAjvErrors = require('better-ajv-errors');
|
|
3
|
+
const { SOURCE_TYPES } = require('./source-types');
|
|
4
|
+
const { DEFAULT_TRANSFORMATIONS } = require('./transformations/transform');
|
|
5
|
+
const {
|
|
6
|
+
sanitizeConfigurationForLog,
|
|
7
|
+
stripKnownUrlCredentials,
|
|
8
|
+
} = require('./utils');
|
|
3
9
|
|
|
4
10
|
const ajv = new Ajv({ allErrors: true, jsonPointers: true });
|
|
5
|
-
const schema = require('./schemas/configuration.schema
|
|
11
|
+
const schema = require('./schemas/configuration.schema');
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Escapes one segment of a JSON pointer. `better-ajv-errors` builds the
|
|
15
|
+
* pointer to an additional property as `${dataPath}/${additionalProperty}`
|
|
16
|
+
* without escaping, so a key with `/` or `~` would not resolve.
|
|
17
|
+
*
|
|
18
|
+
* @param {String} value - the property name.
|
|
19
|
+
* @returns {String} the escaped pointer segment.
|
|
20
|
+
*/
|
|
21
|
+
function escapeJsonPointerSegment(value) {
|
|
22
|
+
return value.replace(/~/g, '~0').replace(/\//g, '~1');
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Prepares `validate.errors` for `better-ajv-errors` against the sanitized
|
|
27
|
+
* configuration copy. An additional property is renamed in the sanitized copy
|
|
28
|
+
* (credentials are removed from keys), so the error must point at the
|
|
29
|
+
* sanitized name. The property name is also escaped as a JSON pointer segment,
|
|
30
|
+
* because `better-ajv-errors` does not escape it when it resolves the code
|
|
31
|
+
* frame. The returned replacements restore the readable property name in the
|
|
32
|
+
* rendered text.
|
|
33
|
+
*
|
|
34
|
+
* @param {Array<Object>} errors - the AJV errors.
|
|
35
|
+
* @returns {{ errors: Array<Object>, replacements: Array<Array<String>> }}
|
|
36
|
+
* the errors to render and the pointer-to-name replacements.
|
|
37
|
+
*/
|
|
38
|
+
function prepareValidationErrors(errors) {
|
|
39
|
+
const replacements = [];
|
|
40
|
+
const preparedErrors = errors.map((error) => {
|
|
41
|
+
const additionalProperty = error.params && error.params.additionalProperty;
|
|
42
|
+
|
|
43
|
+
if (typeof additionalProperty !== 'string') {
|
|
44
|
+
return error;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const sanitizedProperty = stripKnownUrlCredentials(additionalProperty);
|
|
48
|
+
const pointerSegment = escapeJsonPointerSegment(sanitizedProperty);
|
|
49
|
+
|
|
50
|
+
if (pointerSegment !== sanitizedProperty) {
|
|
51
|
+
replacements.push([pointerSegment, sanitizedProperty]);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
if (pointerSegment === additionalProperty) {
|
|
55
|
+
return error;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
return {
|
|
59
|
+
...error,
|
|
60
|
+
params: {
|
|
61
|
+
...error.params,
|
|
62
|
+
additionalProperty: pointerSegment,
|
|
63
|
+
},
|
|
64
|
+
};
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
// Replace a longer pointer first, so a pointer that is a prefix of another
|
|
68
|
+
// does not corrupt the longer one.
|
|
69
|
+
replacements.sort(([first], [second]) => second.length - first.length);
|
|
70
|
+
|
|
71
|
+
return { errors: preparedErrors, replacements };
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Restores the readable property names in the rendered validation text after
|
|
76
|
+
* `better-ajv-errors` used their escaped pointer segments.
|
|
77
|
+
*
|
|
78
|
+
* @param {String} errorsText - the rendered validation errors.
|
|
79
|
+
* @param {Array<Array<String>>} replacements - pairs of a pointer segment and
|
|
80
|
+
* the property name.
|
|
81
|
+
* @returns {String} the text with readable property names.
|
|
82
|
+
*/
|
|
83
|
+
function restorePropertyNames(errorsText, replacements) {
|
|
84
|
+
return replacements.reduce(
|
|
85
|
+
(text, [pointerSegment, property]) => text.split(pointerSegment).join(property),
|
|
86
|
+
errorsText,
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Renders the AJV errors against a sanitized configuration copy.
|
|
92
|
+
*
|
|
93
|
+
* `better-ajv-errors` embeds the configuration in the message, and it renders
|
|
94
|
+
* object keys and `toJSON` results, so the copy must be JSON-safe.
|
|
95
|
+
*
|
|
96
|
+
* @param {*} configuration - the configuration that failed validation.
|
|
97
|
+
* @param {Array<Object>} errors - the AJV errors.
|
|
98
|
+
* @returns {String} the rendered validation errors.
|
|
99
|
+
*/
|
|
100
|
+
function formatValidationErrors(configuration, errors) {
|
|
101
|
+
const prepared = prepareValidationErrors(errors);
|
|
102
|
+
const errorsText = betterAjvErrors(
|
|
103
|
+
schema,
|
|
104
|
+
sanitizeConfigurationForLog(configuration),
|
|
105
|
+
prepared.errors,
|
|
106
|
+
);
|
|
107
|
+
|
|
108
|
+
return restorePropertyNames(errorsText, prepared.replacements);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Creates a configuration object for the quick conversion mode (the CLI `-i`
|
|
113
|
+
* option): compiles the specified sources with the default transformation
|
|
114
|
+
* pipeline.
|
|
115
|
+
*
|
|
116
|
+
* @param {Array<string>} inputs - list of the source paths or URLs.
|
|
117
|
+
* @param {string} [type] - type of the input sources (`adblock` or `hosts`),
|
|
118
|
+
* defaults to `hosts`.
|
|
119
|
+
* @returns {Object} configuration object.
|
|
120
|
+
*/
|
|
121
|
+
function createConfiguration(inputs, type = SOURCE_TYPES.HOSTS) {
|
|
122
|
+
return {
|
|
123
|
+
name: 'Blocklist',
|
|
124
|
+
sources: inputs.map((input) => ({
|
|
125
|
+
source: input,
|
|
126
|
+
// An empty string (e.g. the CLI flag `-t ''`) must fall back to
|
|
127
|
+
// the default type, like the old `argv['input-type'] || 'hosts'`.
|
|
128
|
+
type: type || SOURCE_TYPES.HOSTS,
|
|
129
|
+
})),
|
|
130
|
+
transformations: [...DEFAULT_TRANSFORMATIONS],
|
|
131
|
+
};
|
|
132
|
+
}
|
|
6
133
|
|
|
7
134
|
module.exports = {
|
|
135
|
+
createConfiguration,
|
|
8
136
|
/**
|
|
9
137
|
* Validates the specified configuration object
|
|
10
138
|
*
|
|
@@ -16,7 +144,9 @@ module.exports = {
|
|
|
16
144
|
const valid = validate(configuration);
|
|
17
145
|
return {
|
|
18
146
|
valid,
|
|
19
|
-
|
|
147
|
+
// The validation itself runs against the original configuration,
|
|
148
|
+
// but the rendered text is built from a sanitized copy.
|
|
149
|
+
errorsText: valid ? null : formatValidationErrors(configuration, validate.errors),
|
|
20
150
|
};
|
|
21
151
|
},
|
|
22
152
|
};
|
package/src/download.js
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
const { FiltersDownloader } = require('@adguard/filters-downloader');
|
|
2
|
+
const { stripErrorCredentials, stripKnownUrlCredentials } = require('./utils');
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Downloads the specified source (a URL or a local file path) and
|
|
6
|
+
* resolves all `!#include` directives in it.
|
|
7
|
+
*
|
|
8
|
+
* @param {String} source - URL or local file path of the source.
|
|
9
|
+
* @returns {Promise<Array<String>>} array with the source rules.
|
|
10
|
+
*/
|
|
11
|
+
async function download(source) {
|
|
12
|
+
try {
|
|
13
|
+
return await FiltersDownloader.download(source, {}, { allowEmptyResponse: true });
|
|
14
|
+
} catch (error) {
|
|
15
|
+
// The downloader gets the original URL so it can send Basic auth, but
|
|
16
|
+
// its errors may contain that URL. Replace the exact source first: a
|
|
17
|
+
// password can contain a literal `@` followed by whitespace, and the
|
|
18
|
+
// generic scanner can leave the rest of such a password in the text.
|
|
19
|
+
// Then sanitize the whole chain before the error leaves the download
|
|
20
|
+
// layer.
|
|
21
|
+
throw stripErrorCredentials(error, [[source, stripKnownUrlCredentials(source)]]);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
module.exports = {
|
|
26
|
+
download,
|
|
27
|
+
};
|
package/src/filter.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
const _ = require('lodash');
|
|
2
|
-
const {
|
|
2
|
+
const { download } = require('./download');
|
|
3
3
|
const utils = require('./utils');
|
|
4
4
|
const ruleUtils = require('./rule');
|
|
5
5
|
|
|
@@ -17,7 +17,7 @@ async function downloadAll(sources) {
|
|
|
17
17
|
}
|
|
18
18
|
|
|
19
19
|
await Promise.all(sources.map(async (source) => {
|
|
20
|
-
const rulesStr = await
|
|
20
|
+
const rulesStr = await download(source);
|
|
21
21
|
const rules = rulesStr
|
|
22
22
|
.filter((el) => el.trim().length > 0 && !ruleUtils.isComment(el));
|
|
23
23
|
list = list.concat(rules);
|
package/src/index.d.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
declare module '@adguard/hostlist-compiler' {
|
|
2
|
+
// Keep in sync with SOURCE_TYPES in src/source-types.js (type declarations cannot consume runtime values).
|
|
2
3
|
export type SourceType = 'adblock' | 'hosts';
|
|
3
4
|
|
|
5
|
+
// Keep in sync with TRANSFORMATIONS in src/transformations/enum.js (type declarations cannot consume runtime values).
|
|
4
6
|
export type Transformation =
|
|
5
7
|
'ConvertToAscii' |
|
|
6
8
|
'TrimLines' |
|
package/src/index.js
CHANGED
|
@@ -2,9 +2,19 @@ const _ = require('lodash');
|
|
|
2
2
|
const consola = require('consola');
|
|
3
3
|
const config = require('./configuration');
|
|
4
4
|
const compileSource = require('./compile-source');
|
|
5
|
-
const { transform
|
|
5
|
+
const { transform } = require('./transformations/transform');
|
|
6
|
+
const { checkCrossLevelValidationConflicts } = require('./validation-conflicts');
|
|
7
|
+
const {
|
|
8
|
+
stripUrlValueCredentials,
|
|
9
|
+
stripKnownUrlCredentials,
|
|
10
|
+
sanitizeConfigurationForLog,
|
|
11
|
+
} = require('./utils');
|
|
6
12
|
const packageJson = require('../package.json');
|
|
7
13
|
|
|
14
|
+
// Version is not stored in the repo: CI stamps it into package.json at build
|
|
15
|
+
// time (see publish-release.yml). Fall back to 0.0.0 in development.
|
|
16
|
+
const packageVersion = packageJson.version || '0.0.0';
|
|
17
|
+
|
|
8
18
|
/**
|
|
9
19
|
* Prepares list header
|
|
10
20
|
*
|
|
@@ -15,27 +25,27 @@ const packageJson = require('../package.json');
|
|
|
15
25
|
function prepareHeader(configuration) {
|
|
16
26
|
const lines = [
|
|
17
27
|
'!',
|
|
18
|
-
`! Title: ${configuration.name}`,
|
|
28
|
+
`! Title: ${stripUrlValueCredentials(configuration.name)}`,
|
|
19
29
|
];
|
|
20
30
|
|
|
21
31
|
if (!_.isEmpty(configuration.description)) {
|
|
22
|
-
lines.push(`! Description: ${configuration.description}`);
|
|
32
|
+
lines.push(`! Description: ${stripUrlValueCredentials(configuration.description)}`);
|
|
23
33
|
}
|
|
24
34
|
if (!_.isEmpty(configuration.version)) {
|
|
25
|
-
lines.push(`! Version: ${configuration.version}`);
|
|
35
|
+
lines.push(`! Version: ${stripUrlValueCredentials(configuration.version)}`);
|
|
26
36
|
}
|
|
27
37
|
if (!_.isEmpty(configuration.homepage)) {
|
|
28
|
-
lines.push(`! Homepage: ${configuration.homepage}`);
|
|
38
|
+
lines.push(`! Homepage: ${stripKnownUrlCredentials(configuration.homepage)}`);
|
|
29
39
|
}
|
|
30
40
|
if (!_.isEmpty(configuration.license)) {
|
|
31
|
-
lines.push(`! License: ${configuration.license}`);
|
|
41
|
+
lines.push(`! License: ${stripUrlValueCredentials(configuration.license)}`);
|
|
32
42
|
}
|
|
33
43
|
|
|
34
44
|
lines.push(`! Last modified: ${(new Date()).toISOString()}`);
|
|
35
45
|
lines.push('!');
|
|
36
46
|
|
|
37
47
|
// Compiler info
|
|
38
|
-
lines.push(`! Compiled by ${packageJson.name} v${
|
|
48
|
+
lines.push(`! Compiled by ${packageJson.name} v${packageVersion}`);
|
|
39
49
|
lines.push('!');
|
|
40
50
|
return lines;
|
|
41
51
|
}
|
|
@@ -56,54 +66,13 @@ function prepareSourceHeader(source) {
|
|
|
56
66
|
];
|
|
57
67
|
|
|
58
68
|
if (!_.isEmpty(source.name)) {
|
|
59
|
-
lines.push(`! Source name: ${source.name}`);
|
|
69
|
+
lines.push(`! Source name: ${stripUrlValueCredentials(source.name)}`);
|
|
60
70
|
}
|
|
61
|
-
lines.push(`! Source: ${source.source}`);
|
|
71
|
+
lines.push(`! Source: ${stripKnownUrlCredentials(source.source)}`);
|
|
62
72
|
lines.push('!');
|
|
63
73
|
return lines;
|
|
64
74
|
}
|
|
65
75
|
|
|
66
|
-
/**
|
|
67
|
-
* Checks for conflicting validation transformations across source-level and top-level.
|
|
68
|
-
* Throws if a validation transformation is used at both levels, as this causes silent data loss.
|
|
69
|
-
*
|
|
70
|
-
* @param {*} configuration - compilation configuration.
|
|
71
|
-
*/
|
|
72
|
-
function checkCrossLevelValidationConflicts(configuration) {
|
|
73
|
-
const VALIDATION_TRANSFORMATIONS = [
|
|
74
|
-
TRANSFORMATIONS.Validate,
|
|
75
|
-
TRANSFORMATIONS.ValidateAllowIp,
|
|
76
|
-
TRANSFORMATIONS.ValidateAllowPublicSuffix,
|
|
77
|
-
TRANSFORMATIONS.ValidateAllowIpAndPublicSuffix,
|
|
78
|
-
];
|
|
79
|
-
|
|
80
|
-
const topLevelTransformations = configuration.transformations || [];
|
|
81
|
-
const topLevelValidations = VALIDATION_TRANSFORMATIONS
|
|
82
|
-
.filter((t) => topLevelTransformations.indexOf(t) !== -1);
|
|
83
|
-
|
|
84
|
-
if (topLevelValidations.length === 0) {
|
|
85
|
-
return;
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
// Check each source for conflicting validations
|
|
89
|
-
// eslint-disable-next-line no-restricted-syntax
|
|
90
|
-
for (const source of configuration.sources) {
|
|
91
|
-
const sourceTransformations = source.transformations || [];
|
|
92
|
-
const sourceValidations = VALIDATION_TRANSFORMATIONS
|
|
93
|
-
.filter((t) => sourceTransformations.indexOf(t) !== -1);
|
|
94
|
-
|
|
95
|
-
if (sourceValidations.length > 0) {
|
|
96
|
-
throw new Error(
|
|
97
|
-
'Validation transformations cannot be used at both source and top level. '
|
|
98
|
-
+ `Source "${source.source}" uses [${sourceValidations.join(', ')}], `
|
|
99
|
-
+ `but top level uses [${topLevelValidations.join(', ')}]. `
|
|
100
|
-
+ 'This causes the source-level validation to be overridden by the top-level one, '
|
|
101
|
-
+ 'making the source-level validation ineffective.',
|
|
102
|
-
);
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
|
|
107
76
|
/**
|
|
108
77
|
* Compiles a filter list using the specified configuration.
|
|
109
78
|
*
|
|
@@ -121,7 +90,8 @@ async function compile(configuration) {
|
|
|
121
90
|
|
|
122
91
|
checkCrossLevelValidationConflicts(configuration);
|
|
123
92
|
|
|
124
|
-
|
|
93
|
+
const configurationDump = JSON.stringify(sanitizeConfigurationForLog(configuration), 0, 4);
|
|
94
|
+
consola.info(`Configuration: ${configurationDump}`);
|
|
125
95
|
|
|
126
96
|
// This will be the final list of rules
|
|
127
97
|
let finalList = [];
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
const consola = require('consola');
|
|
2
|
-
const ruleUtils = require('
|
|
3
|
-
const { classifyIpPattern, parseIpPattern } = require('
|
|
2
|
+
const ruleUtils = require('./rule');
|
|
3
|
+
const { classifyIpPattern, parseIpPattern } = require('./utils');
|
|
4
4
|
|
|
5
5
|
// Prefix for canonical IP/domain patterns
|
|
6
6
|
const IP_PREFIX = '||';
|