@adguard/hostlist-compiler 2.1.0 → 3.0.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dockerignore +29 -0
- package/.github/workflows/ci.yml +22 -0
- package/.github/workflows/mirror.yml +24 -0
- package/.github/workflows/prepare-release.yml +26 -0
- package/.github/workflows/publish-release.yml +44 -0
- package/.markdownlint.json +29 -0
- package/.markdownlintignore +1 -0
- package/AGENTS.md +314 -103
- package/CHANGELOG.md +76 -7
- package/DEPLOYMENT.md +171 -0
- package/DEVELOPMENT.md +174 -76
- package/README.md +240 -140
- package/examples/china/README.md +2 -2
- package/examples/energized/README.md +4 -2
- package/examples/sdn/README.md +2 -2
- package/examples/whitelist/README.md +2 -2
- package/package.json +29 -24
- package/src/cli.js +59 -33
- package/src/compile-source.js +4 -3
- package/src/configuration.js +132 -2
- package/src/download.js +27 -0
- package/src/filter.js +2 -2
- package/src/index.d.ts +2 -0
- package/src/index.js +21 -51
- package/src/{transformations/ip-normalize.js → ip-normalize.js} +2 -2
- package/src/schemas/configuration.schema.js +137 -0
- package/src/source-types.js +17 -0
- package/src/transformations/enum.js +49 -0
- package/src/transformations/include.js +1 -1
- package/src/transformations/transform.js +40 -74
- package/src/transformations/validate-allow-ip-and-public-suffix.js +1 -1
- package/src/transformations/validate-allow-ip.js +1 -1
- package/src/utils.js +1284 -0
- package/src/validation-conflicts.js +106 -0
- package/test/cli-error-output.test.js +305 -0
- package/test/cli.test.js +47 -0
- package/test/compile-source.test.js +167 -7
- package/test/configuration.test.js +438 -0
- package/test/download.test.js +260 -0
- package/test/helpers/http-mock.js +94 -0
- package/test/index.test.js +497 -4
- package/test/{transformations/ip-normalize.test.js → ip-normalize.test.js} +1 -1
- package/test/resolve-include.test.js +8 -8
- package/test/transformations/covert-to-ascii.test.js +1 -1
- package/test/transformations/exclude.test.js +2 -2
- package/test/transformations/include.test.js +9 -3
- package/test/transformations/ip-alignment.test.js +1 -1
- package/test/transformations/transform.test.js +66 -1
- package/test/utils.test.js +1559 -0
- package/test/validation-conflicts.test.js +230 -0
- package/bamboo-specs/bamboo.yaml +0 -14
- package/bamboo-specs/build.yaml +0 -92
- package/bamboo-specs/deploy.yaml +0 -38
- package/bamboo-specs/increment.yaml +0 -54
- package/bamboo-specs/permissions.yaml +0 -16
- package/bamboo-specs/test.yaml +0 -56
- package/src/schemas/configuration.schema.json +0 -369
- package/tools/build-txt.js +0 -19
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
/* eslint-disable max-len -- schema descriptions are long human-readable strings */
|
|
2
|
+
|
|
3
|
+
const { TRANSFORMATIONS } = require('../transformations/enum');
|
|
4
|
+
const { SOURCE_TYPES } = require('../source-types');
|
|
5
|
+
const { buildSchemaConflictRules } = require('../validation-conflicts');
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* A list of the transformations that will be applied. The accepted names are
|
|
9
|
+
* derived from the `TRANSFORMATIONS` enum and the conflict rules are generated
|
|
10
|
+
* from src/validation-conflicts.js, so the schema cannot drift from the code.
|
|
11
|
+
* The enum's declaration order is preserved here — it is what the
|
|
12
|
+
* unknown-transformation error renders as the allowed-values list (pinned by a
|
|
13
|
+
* test in configuration.test.js).
|
|
14
|
+
*/
|
|
15
|
+
const transformations = {
|
|
16
|
+
description: 'A list of the transformations that will be applied',
|
|
17
|
+
type: 'array',
|
|
18
|
+
allOf: buildSchemaConflictRules(),
|
|
19
|
+
items: {
|
|
20
|
+
type: 'string',
|
|
21
|
+
enum: Object.values(TRANSFORMATIONS),
|
|
22
|
+
},
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* A JSON-schema fragment for a configuration property that is a list of strings.
|
|
27
|
+
*
|
|
28
|
+
* @param {string} description - human-readable description of the property.
|
|
29
|
+
* @param {string} [credentialUrl] - `x-credential-url` marker: `'list'` for a
|
|
30
|
+
* list of URLs that use the strict credential parser.
|
|
31
|
+
* @returns {Object} the JSON-schema fragment (type: 'array', items: string).
|
|
32
|
+
*/
|
|
33
|
+
function stringArray(description, credentialUrl) {
|
|
34
|
+
return {
|
|
35
|
+
description,
|
|
36
|
+
type: 'array',
|
|
37
|
+
items: {
|
|
38
|
+
type: 'string',
|
|
39
|
+
},
|
|
40
|
+
...(credentialUrl ? { 'x-credential-url': credentialUrl } : {}),
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// Shared between the top level and the source level, like `transformations`.
|
|
45
|
+
const exclusions = stringArray('A list of rules (or wildcards) to exclude from the source.');
|
|
46
|
+
const exclusionsSources = stringArray('An array of exclusions sources.', 'list');
|
|
47
|
+
const inclusions = stringArray("A list of wildcards to include from the source. All rules that don't match these wildcards won't be included.");
|
|
48
|
+
const inclusionsSources = stringArray('A list of files with inclusions.', 'list');
|
|
49
|
+
|
|
50
|
+
const schema = {
|
|
51
|
+
$id: 'https://adguard.com/hostlist-compiler.configuration.schema',
|
|
52
|
+
$schema: 'http://json-schema.org/draft-07/schema#',
|
|
53
|
+
title: 'Hostlist compiler configuration',
|
|
54
|
+
description: 'Configuration for the hostlist compiler',
|
|
55
|
+
type: 'object',
|
|
56
|
+
properties: {
|
|
57
|
+
name: {
|
|
58
|
+
description: 'Filter list name',
|
|
59
|
+
type: 'string',
|
|
60
|
+
minLength: 1,
|
|
61
|
+
},
|
|
62
|
+
description: {
|
|
63
|
+
description: 'Filter list description',
|
|
64
|
+
type: 'string',
|
|
65
|
+
},
|
|
66
|
+
homepage: {
|
|
67
|
+
description: 'Filter list homepage',
|
|
68
|
+
type: 'string',
|
|
69
|
+
// The `x-credential-url` marker pins this field to the strict
|
|
70
|
+
// credential parser in `src/utils.js`. A test derives the parser
|
|
71
|
+
// field sets from these markers.
|
|
72
|
+
'x-credential-url': 'value',
|
|
73
|
+
},
|
|
74
|
+
license: {
|
|
75
|
+
description: 'Filter list license',
|
|
76
|
+
type: 'string',
|
|
77
|
+
},
|
|
78
|
+
version: {
|
|
79
|
+
description: 'Filter version',
|
|
80
|
+
type: 'string',
|
|
81
|
+
},
|
|
82
|
+
sources: {
|
|
83
|
+
description: 'An array of the filter list sources',
|
|
84
|
+
type: 'array',
|
|
85
|
+
minItems: 1,
|
|
86
|
+
items: {
|
|
87
|
+
$ref: '#/definitions/source',
|
|
88
|
+
},
|
|
89
|
+
'x-credential-url': 'list',
|
|
90
|
+
},
|
|
91
|
+
transformations,
|
|
92
|
+
exclusions,
|
|
93
|
+
exclusions_sources: exclusionsSources,
|
|
94
|
+
inclusions,
|
|
95
|
+
inclusions_sources: inclusionsSources,
|
|
96
|
+
},
|
|
97
|
+
required: [
|
|
98
|
+
'name',
|
|
99
|
+
'sources',
|
|
100
|
+
],
|
|
101
|
+
additionalProperties: false,
|
|
102
|
+
definitions: {
|
|
103
|
+
source: {
|
|
104
|
+
description: 'A source for the filter list',
|
|
105
|
+
type: 'object',
|
|
106
|
+
properties: {
|
|
107
|
+
name: {
|
|
108
|
+
description: 'Name of the source',
|
|
109
|
+
type: 'string',
|
|
110
|
+
minLength: 1,
|
|
111
|
+
},
|
|
112
|
+
source: {
|
|
113
|
+
description: 'Path to a file or a URL',
|
|
114
|
+
type: 'string',
|
|
115
|
+
minLength: 1,
|
|
116
|
+
'x-credential-url': 'value',
|
|
117
|
+
},
|
|
118
|
+
type: {
|
|
119
|
+
description: 'Type of the source',
|
|
120
|
+
type: 'string',
|
|
121
|
+
enum: Object.values(SOURCE_TYPES),
|
|
122
|
+
},
|
|
123
|
+
transformations,
|
|
124
|
+
exclusions,
|
|
125
|
+
exclusions_sources: exclusionsSources,
|
|
126
|
+
inclusions,
|
|
127
|
+
inclusions_sources: inclusionsSources,
|
|
128
|
+
},
|
|
129
|
+
required: [
|
|
130
|
+
'source',
|
|
131
|
+
],
|
|
132
|
+
additionalProperties: false,
|
|
133
|
+
},
|
|
134
|
+
},
|
|
135
|
+
};
|
|
136
|
+
|
|
137
|
+
module.exports = schema;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Source-type vocabulary.
|
|
3
|
+
*
|
|
4
|
+
* The single source of truth for the source `type` configuration values
|
|
5
|
+
* ('adblock' | 'hosts'): the configuration schema and the CLI default are
|
|
6
|
+
* derived from `SOURCE_TYPES`, so they cannot drift apart. The `SourceType`
|
|
7
|
+
* union in `src/index.d.ts` is a type declaration and cannot consume runtime
|
|
8
|
+
* values — keep it in sync with this module.
|
|
9
|
+
*/
|
|
10
|
+
const SOURCE_TYPES = Object.freeze({
|
|
11
|
+
ADBLOCK: 'adblock',
|
|
12
|
+
HOSTS: 'hosts',
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
module.exports = {
|
|
16
|
+
SOURCE_TYPES,
|
|
17
|
+
};
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* All available transformations.
|
|
3
|
+
*
|
|
4
|
+
* This module is the single source of truth for transformation names: the
|
|
5
|
+
* configuration schema derives its accepted names from `TRANSFORMATIONS`, and
|
|
6
|
+
* the validation-conflict rules (`validation-conflicts.js`) derive their list
|
|
7
|
+
* from `VALIDATION_TRANSFORMATIONS` below, so they cannot drift apart.
|
|
8
|
+
*
|
|
9
|
+
* Declaration order is significant: the schema's `items.enum` is
|
|
10
|
+
* `Object.values(TRANSFORMATIONS)`, and that order is what the
|
|
11
|
+
* unknown-transformation error renders as the allowed-values list. Keep it
|
|
12
|
+
* matching the pre-AG-58267 schema order (RemoveComments, RemoveModifiers,
|
|
13
|
+
* Compress, ...) — a test pins it.
|
|
14
|
+
*/
|
|
15
|
+
const TRANSFORMATIONS = Object.freeze({
|
|
16
|
+
RemoveComments: 'RemoveComments',
|
|
17
|
+
RemoveModifiers: 'RemoveModifiers',
|
|
18
|
+
Compress: 'Compress',
|
|
19
|
+
Validate: 'Validate',
|
|
20
|
+
ValidateAllowIp: 'ValidateAllowIp',
|
|
21
|
+
ValidateAllowPublicSuffix: 'ValidateAllowPublicSuffix',
|
|
22
|
+
ValidateAllowIpAndPublicSuffix: 'ValidateAllowIpAndPublicSuffix',
|
|
23
|
+
Deduplicate: 'Deduplicate',
|
|
24
|
+
InvertAllow: 'InvertAllow',
|
|
25
|
+
RemoveEmptyLines: 'RemoveEmptyLines',
|
|
26
|
+
TrimLines: 'TrimLines',
|
|
27
|
+
InsertFinalNewLine: 'InsertFinalNewLine',
|
|
28
|
+
ConvertToAscii: 'ConvertToAscii',
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* The validation transformations — the subset of `TRANSFORMATIONS` that cannot
|
|
33
|
+
* be combined with each other.
|
|
34
|
+
*
|
|
35
|
+
* Defined right below the enum so that adding a validation transformation is a
|
|
36
|
+
* single-file change: the runtime conflict checks and the generated schema
|
|
37
|
+
* conflict rules (`validation-conflicts.js`) are derived from this list.
|
|
38
|
+
*/
|
|
39
|
+
const VALIDATION_TRANSFORMATIONS = Object.freeze([
|
|
40
|
+
TRANSFORMATIONS.Validate,
|
|
41
|
+
TRANSFORMATIONS.ValidateAllowIp,
|
|
42
|
+
TRANSFORMATIONS.ValidateAllowPublicSuffix,
|
|
43
|
+
TRANSFORMATIONS.ValidateAllowIpAndPublicSuffix,
|
|
44
|
+
]);
|
|
45
|
+
|
|
46
|
+
module.exports = {
|
|
47
|
+
TRANSFORMATIONS,
|
|
48
|
+
VALIDATION_TRANSFORMATIONS,
|
|
49
|
+
};
|
|
@@ -13,48 +13,47 @@ const removeEmptyLines = require('./remove-empty-lines');
|
|
|
13
13
|
const trimLines = require('./trim-lines');
|
|
14
14
|
const insertFinalNewLine = require('./insert-final-newline');
|
|
15
15
|
const convertToAscii = require('./covert-to-ascii');
|
|
16
|
+
const { TRANSFORMATIONS } = require('./enum');
|
|
17
|
+
const { checkIncompatibleValidationTransformations } = require('../validation-conflicts');
|
|
16
18
|
|
|
17
19
|
/**
|
|
18
|
-
*
|
|
20
|
+
* The default transformation pipeline for the quick conversion mode (the CLI
|
|
21
|
+
* `-i` option): converts /etc/hosts rules to AdGuard-syntax rules.
|
|
19
22
|
*/
|
|
20
|
-
const
|
|
21
|
-
RemoveComments
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
Validate: 'Validate',
|
|
25
|
-
ValidateAllowIp: 'ValidateAllowIp',
|
|
26
|
-
ValidateAllowPublicSuffix: 'ValidateAllowPublicSuffix',
|
|
27
|
-
ValidateAllowIpAndPublicSuffix: 'ValidateAllowIpAndPublicSuffix',
|
|
28
|
-
Deduplicate: 'Deduplicate',
|
|
29
|
-
InvertAllow: 'InvertAllow',
|
|
30
|
-
RemoveEmptyLines: 'RemoveEmptyLines',
|
|
31
|
-
TrimLines: 'TrimLines',
|
|
32
|
-
InsertFinalNewLine: 'InsertFinalNewLine',
|
|
33
|
-
ConvertToAscii: 'ConvertToAscii',
|
|
34
|
-
});
|
|
35
|
-
|
|
36
|
-
const VALIDATION_TRANSFORMATIONS = [
|
|
23
|
+
const DEFAULT_TRANSFORMATIONS = Object.freeze([
|
|
24
|
+
TRANSFORMATIONS.RemoveComments,
|
|
25
|
+
TRANSFORMATIONS.Deduplicate,
|
|
26
|
+
TRANSFORMATIONS.Compress,
|
|
37
27
|
TRANSFORMATIONS.Validate,
|
|
38
|
-
TRANSFORMATIONS.
|
|
39
|
-
TRANSFORMATIONS.
|
|
40
|
-
|
|
41
|
-
];
|
|
28
|
+
TRANSFORMATIONS.TrimLines,
|
|
29
|
+
TRANSFORMATIONS.InsertFinalNewLine,
|
|
30
|
+
]);
|
|
42
31
|
|
|
43
32
|
/**
|
|
44
|
-
*
|
|
45
|
-
* Combining them causes silent data loss: each validator runs on the already-filtered
|
|
46
|
-
* output of the previous one, so allow-modes (AllowIp, AllowPublicSuffix) become ineffective.
|
|
33
|
+
* The transformations in their fixed execution order, paired with their implementations.
|
|
47
34
|
*
|
|
48
|
-
*
|
|
35
|
+
* When adding a transformation: add it to the `TRANSFORMATIONS` enum in
|
|
36
|
+
* `enum.js`, then register it here (order matters — it is the execution order
|
|
37
|
+
* and MUST match the order list in README.md). A test in `transform.test.js`
|
|
38
|
+
* asserts this list covers every entry of the enum and pins its order, so a
|
|
39
|
+
* forgotten branch or an accidental reorder fails the suite instead of being
|
|
40
|
+
* silently skipped.
|
|
49
41
|
*/
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
42
|
+
const TRANSFORMATIONS_IN_ORDER = [
|
|
43
|
+
[TRANSFORMATIONS.ConvertToAscii, convertToAscii],
|
|
44
|
+
[TRANSFORMATIONS.TrimLines, trimLines],
|
|
45
|
+
[TRANSFORMATIONS.RemoveComments, removeComments],
|
|
46
|
+
[TRANSFORMATIONS.Compress, compress],
|
|
47
|
+
[TRANSFORMATIONS.RemoveModifiers, removeModifiers],
|
|
48
|
+
[TRANSFORMATIONS.InvertAllow, invertAllow],
|
|
49
|
+
[TRANSFORMATIONS.Validate, validate],
|
|
50
|
+
[TRANSFORMATIONS.ValidateAllowIp, validateAllowIp],
|
|
51
|
+
[TRANSFORMATIONS.ValidateAllowPublicSuffix, validateAllowPublicSuffix],
|
|
52
|
+
[TRANSFORMATIONS.ValidateAllowIpAndPublicSuffix, validateAllowIpAndPublicSuffix],
|
|
53
|
+
[TRANSFORMATIONS.Deduplicate, deduplicate],
|
|
54
|
+
[TRANSFORMATIONS.RemoveEmptyLines, removeEmptyLines],
|
|
55
|
+
[TRANSFORMATIONS.InsertFinalNewLine, insertFinalNewLine],
|
|
56
|
+
];
|
|
58
57
|
|
|
59
58
|
/**
|
|
60
59
|
* Applies the specified transformations to the list of rules in the proper order.
|
|
@@ -86,50 +85,17 @@ async function transform(rules, configuration, transformations) {
|
|
|
86
85
|
configuration.inclusions_sources,
|
|
87
86
|
);
|
|
88
87
|
|
|
89
|
-
//
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
transformed = trimLines(transformed);
|
|
95
|
-
}
|
|
96
|
-
if (transformations.indexOf(TRANSFORMATIONS.RemoveComments) !== -1) {
|
|
97
|
-
transformed = removeComments(transformed);
|
|
98
|
-
}
|
|
99
|
-
if (transformations.indexOf(TRANSFORMATIONS.Compress) !== -1) {
|
|
100
|
-
transformed = compress(transformed);
|
|
101
|
-
}
|
|
102
|
-
if (transformations.indexOf(TRANSFORMATIONS.RemoveModifiers) !== -1) {
|
|
103
|
-
transformed = removeModifiers(transformed);
|
|
104
|
-
}
|
|
105
|
-
if (transformations.indexOf(TRANSFORMATIONS.InvertAllow) !== -1) {
|
|
106
|
-
transformed = invertAllow(transformed);
|
|
107
|
-
}
|
|
108
|
-
if (transformations.indexOf(TRANSFORMATIONS.Validate) !== -1) {
|
|
109
|
-
transformed = validate(transformed);
|
|
110
|
-
}
|
|
111
|
-
if (transformations.indexOf(TRANSFORMATIONS.ValidateAllowIp) !== -1) {
|
|
112
|
-
transformed = validateAllowIp(transformed);
|
|
113
|
-
}
|
|
114
|
-
if (transformations.indexOf(TRANSFORMATIONS.ValidateAllowPublicSuffix) !== -1) {
|
|
115
|
-
transformed = validateAllowPublicSuffix(transformed);
|
|
116
|
-
}
|
|
117
|
-
if (transformations.indexOf(TRANSFORMATIONS.ValidateAllowIpAndPublicSuffix) !== -1) {
|
|
118
|
-
transformed = validateAllowIpAndPublicSuffix(transformed);
|
|
119
|
-
}
|
|
120
|
-
if (transformations.indexOf(TRANSFORMATIONS.Deduplicate) !== -1) {
|
|
121
|
-
transformed = deduplicate(transformed);
|
|
122
|
-
}
|
|
123
|
-
if (transformations.indexOf(TRANSFORMATIONS.RemoveEmptyLines) !== -1) {
|
|
124
|
-
transformed = removeEmptyLines(transformed);
|
|
125
|
-
}
|
|
126
|
-
if (transformations.indexOf(TRANSFORMATIONS.InsertFinalNewLine) !== -1) {
|
|
127
|
-
transformed = insertFinalNewLine(transformed);
|
|
88
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
89
|
+
for (const [name, apply] of TRANSFORMATIONS_IN_ORDER) {
|
|
90
|
+
if (transformations.indexOf(name) !== -1) {
|
|
91
|
+
transformed = apply(transformed);
|
|
92
|
+
}
|
|
128
93
|
}
|
|
129
94
|
return transformed;
|
|
130
95
|
}
|
|
131
96
|
|
|
132
97
|
module.exports = {
|
|
133
98
|
transform,
|
|
134
|
-
|
|
99
|
+
DEFAULT_TRANSFORMATIONS,
|
|
100
|
+
TRANSFORMATIONS_IN_ORDER,
|
|
135
101
|
};
|