@adguard/hostlist-compiler 2.1.0 → 3.0.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/.dockerignore +29 -0
  2. package/.github/workflows/ci.yml +22 -0
  3. package/.github/workflows/mirror.yml +24 -0
  4. package/.github/workflows/prepare-release.yml +26 -0
  5. package/.github/workflows/publish-release.yml +44 -0
  6. package/.markdownlint.json +29 -0
  7. package/.markdownlintignore +1 -0
  8. package/AGENTS.md +314 -103
  9. package/CHANGELOG.md +76 -7
  10. package/DEPLOYMENT.md +171 -0
  11. package/DEVELOPMENT.md +174 -76
  12. package/README.md +240 -140
  13. package/examples/china/README.md +2 -2
  14. package/examples/energized/README.md +4 -2
  15. package/examples/sdn/README.md +2 -2
  16. package/examples/whitelist/README.md +2 -2
  17. package/package.json +29 -24
  18. package/src/cli.js +59 -33
  19. package/src/compile-source.js +4 -3
  20. package/src/configuration.js +132 -2
  21. package/src/download.js +27 -0
  22. package/src/filter.js +2 -2
  23. package/src/index.d.ts +2 -0
  24. package/src/index.js +21 -51
  25. package/src/{transformations/ip-normalize.js → ip-normalize.js} +2 -2
  26. package/src/schemas/configuration.schema.js +137 -0
  27. package/src/source-types.js +17 -0
  28. package/src/transformations/enum.js +49 -0
  29. package/src/transformations/include.js +1 -1
  30. package/src/transformations/transform.js +40 -74
  31. package/src/transformations/validate-allow-ip-and-public-suffix.js +1 -1
  32. package/src/transformations/validate-allow-ip.js +1 -1
  33. package/src/utils.js +1284 -0
  34. package/src/validation-conflicts.js +106 -0
  35. package/test/cli-error-output.test.js +305 -0
  36. package/test/cli.test.js +47 -0
  37. package/test/compile-source.test.js +167 -7
  38. package/test/configuration.test.js +438 -0
  39. package/test/download.test.js +260 -0
  40. package/test/helpers/http-mock.js +94 -0
  41. package/test/index.test.js +497 -4
  42. package/test/{transformations/ip-normalize.test.js → ip-normalize.test.js} +1 -1
  43. package/test/resolve-include.test.js +8 -8
  44. package/test/transformations/covert-to-ascii.test.js +1 -1
  45. package/test/transformations/exclude.test.js +2 -2
  46. package/test/transformations/include.test.js +9 -3
  47. package/test/transformations/ip-alignment.test.js +1 -1
  48. package/test/transformations/transform.test.js +66 -1
  49. package/test/utils.test.js +1559 -0
  50. package/test/validation-conflicts.test.js +230 -0
  51. package/bamboo-specs/bamboo.yaml +0 -14
  52. package/bamboo-specs/build.yaml +0 -92
  53. package/bamboo-specs/deploy.yaml +0 -38
  54. package/bamboo-specs/increment.yaml +0 -54
  55. package/bamboo-specs/permissions.yaml +0 -16
  56. package/bamboo-specs/test.yaml +0 -56
  57. package/src/schemas/configuration.schema.json +0 -369
  58. package/tools/build-txt.js +0 -19
@@ -0,0 +1,137 @@
1
+ /* eslint-disable max-len -- schema descriptions are long human-readable strings */
2
+
3
+ const { TRANSFORMATIONS } = require('../transformations/enum');
4
+ const { SOURCE_TYPES } = require('../source-types');
5
+ const { buildSchemaConflictRules } = require('../validation-conflicts');
6
+
7
+ /**
8
+ * A list of the transformations that will be applied. The accepted names are
9
+ * derived from the `TRANSFORMATIONS` enum and the conflict rules are generated
10
+ * from src/validation-conflicts.js, so the schema cannot drift from the code.
11
+ * The enum's declaration order is preserved here — it is what the
12
+ * unknown-transformation error renders as the allowed-values list (pinned by a
13
+ * test in configuration.test.js).
14
+ */
15
+ const transformations = {
16
+ description: 'A list of the transformations that will be applied',
17
+ type: 'array',
18
+ allOf: buildSchemaConflictRules(),
19
+ items: {
20
+ type: 'string',
21
+ enum: Object.values(TRANSFORMATIONS),
22
+ },
23
+ };
24
+
25
+ /**
26
+ * A JSON-schema fragment for a configuration property that is a list of strings.
27
+ *
28
+ * @param {string} description - human-readable description of the property.
29
+ * @param {string} [credentialUrl] - `x-credential-url` marker: `'list'` for a
30
+ * list of URLs that use the strict credential parser.
31
+ * @returns {Object} the JSON-schema fragment (type: 'array', items: string).
32
+ */
33
+ function stringArray(description, credentialUrl) {
34
+ return {
35
+ description,
36
+ type: 'array',
37
+ items: {
38
+ type: 'string',
39
+ },
40
+ ...(credentialUrl ? { 'x-credential-url': credentialUrl } : {}),
41
+ };
42
+ }
43
+
44
+ // Shared between the top level and the source level, like `transformations`.
45
+ const exclusions = stringArray('A list of rules (or wildcards) to exclude from the source.');
46
+ const exclusionsSources = stringArray('An array of exclusions sources.', 'list');
47
+ const inclusions = stringArray("A list of wildcards to include from the source. All rules that don't match these wildcards won't be included.");
48
+ const inclusionsSources = stringArray('A list of files with inclusions.', 'list');
49
+
50
+ const schema = {
51
+ $id: 'https://adguard.com/hostlist-compiler.configuration.schema',
52
+ $schema: 'http://json-schema.org/draft-07/schema#',
53
+ title: 'Hostlist compiler configuration',
54
+ description: 'Configuration for the hostlist compiler',
55
+ type: 'object',
56
+ properties: {
57
+ name: {
58
+ description: 'Filter list name',
59
+ type: 'string',
60
+ minLength: 1,
61
+ },
62
+ description: {
63
+ description: 'Filter list description',
64
+ type: 'string',
65
+ },
66
+ homepage: {
67
+ description: 'Filter list homepage',
68
+ type: 'string',
69
+ // The `x-credential-url` marker pins this field to the strict
70
+ // credential parser in `src/utils.js`. A test derives the parser
71
+ // field sets from these markers.
72
+ 'x-credential-url': 'value',
73
+ },
74
+ license: {
75
+ description: 'Filter list license',
76
+ type: 'string',
77
+ },
78
+ version: {
79
+ description: 'Filter version',
80
+ type: 'string',
81
+ },
82
+ sources: {
83
+ description: 'An array of the filter list sources',
84
+ type: 'array',
85
+ minItems: 1,
86
+ items: {
87
+ $ref: '#/definitions/source',
88
+ },
89
+ 'x-credential-url': 'list',
90
+ },
91
+ transformations,
92
+ exclusions,
93
+ exclusions_sources: exclusionsSources,
94
+ inclusions,
95
+ inclusions_sources: inclusionsSources,
96
+ },
97
+ required: [
98
+ 'name',
99
+ 'sources',
100
+ ],
101
+ additionalProperties: false,
102
+ definitions: {
103
+ source: {
104
+ description: 'A source for the filter list',
105
+ type: 'object',
106
+ properties: {
107
+ name: {
108
+ description: 'Name of the source',
109
+ type: 'string',
110
+ minLength: 1,
111
+ },
112
+ source: {
113
+ description: 'Path to a file or a URL',
114
+ type: 'string',
115
+ minLength: 1,
116
+ 'x-credential-url': 'value',
117
+ },
118
+ type: {
119
+ description: 'Type of the source',
120
+ type: 'string',
121
+ enum: Object.values(SOURCE_TYPES),
122
+ },
123
+ transformations,
124
+ exclusions,
125
+ exclusions_sources: exclusionsSources,
126
+ inclusions,
127
+ inclusions_sources: inclusionsSources,
128
+ },
129
+ required: [
130
+ 'source',
131
+ ],
132
+ additionalProperties: false,
133
+ },
134
+ },
135
+ };
136
+
137
+ module.exports = schema;
@@ -0,0 +1,17 @@
1
+ /**
2
+ * Source-type vocabulary.
3
+ *
4
+ * The single source of truth for the source `type` configuration values
5
+ * ('adblock' | 'hosts'): the configuration schema and the CLI default are
6
+ * derived from `SOURCE_TYPES`, so they cannot drift apart. The `SourceType`
7
+ * union in `src/index.d.ts` is a type declaration and cannot consume runtime
8
+ * values — keep it in sync with this module.
9
+ */
10
+ const SOURCE_TYPES = Object.freeze({
11
+ ADBLOCK: 'adblock',
12
+ HOSTS: 'hosts',
13
+ });
14
+
15
+ module.exports = {
16
+ SOURCE_TYPES,
17
+ };
@@ -0,0 +1,49 @@
1
+ /**
2
+ * All available transformations.
3
+ *
4
+ * This module is the single source of truth for transformation names: the
5
+ * configuration schema derives its accepted names from `TRANSFORMATIONS`, and
6
+ * the validation-conflict rules (`validation-conflicts.js`) derive their list
7
+ * from `VALIDATION_TRANSFORMATIONS` below, so they cannot drift apart.
8
+ *
9
+ * Declaration order is significant: the schema's `items.enum` is
10
+ * `Object.values(TRANSFORMATIONS)`, and that order is what the
11
+ * unknown-transformation error renders as the allowed-values list. Keep it
12
+ * matching the pre-AG-58267 schema order (RemoveComments, RemoveModifiers,
13
+ * Compress, ...) — a test pins it.
14
+ */
15
+ const TRANSFORMATIONS = Object.freeze({
16
+ RemoveComments: 'RemoveComments',
17
+ RemoveModifiers: 'RemoveModifiers',
18
+ Compress: 'Compress',
19
+ Validate: 'Validate',
20
+ ValidateAllowIp: 'ValidateAllowIp',
21
+ ValidateAllowPublicSuffix: 'ValidateAllowPublicSuffix',
22
+ ValidateAllowIpAndPublicSuffix: 'ValidateAllowIpAndPublicSuffix',
23
+ Deduplicate: 'Deduplicate',
24
+ InvertAllow: 'InvertAllow',
25
+ RemoveEmptyLines: 'RemoveEmptyLines',
26
+ TrimLines: 'TrimLines',
27
+ InsertFinalNewLine: 'InsertFinalNewLine',
28
+ ConvertToAscii: 'ConvertToAscii',
29
+ });
30
+
31
+ /**
32
+ * The validation transformations — the subset of `TRANSFORMATIONS` that cannot
33
+ * be combined with each other.
34
+ *
35
+ * Defined right below the enum so that adding a validation transformation is a
36
+ * single-file change: the runtime conflict checks and the generated schema
37
+ * conflict rules (`validation-conflicts.js`) are derived from this list.
38
+ */
39
+ const VALIDATION_TRANSFORMATIONS = Object.freeze([
40
+ TRANSFORMATIONS.Validate,
41
+ TRANSFORMATIONS.ValidateAllowIp,
42
+ TRANSFORMATIONS.ValidateAllowPublicSuffix,
43
+ TRANSFORMATIONS.ValidateAllowIpAndPublicSuffix,
44
+ ]);
45
+
46
+ module.exports = {
47
+ TRANSFORMATIONS,
48
+ VALIDATION_TRANSFORMATIONS,
49
+ };
@@ -32,7 +32,7 @@ async function include(rules, inclusions, inclusionsSources) {
32
32
  return included;
33
33
  });
34
34
 
35
- consola.info(`Included ${rules.length} rules`);
35
+ consola.info(`Included ${filtered.length} rules`);
36
36
  return filtered;
37
37
  }
38
38
 
@@ -13,48 +13,47 @@ const removeEmptyLines = require('./remove-empty-lines');
13
13
  const trimLines = require('./trim-lines');
14
14
  const insertFinalNewLine = require('./insert-final-newline');
15
15
  const convertToAscii = require('./covert-to-ascii');
16
+ const { TRANSFORMATIONS } = require('./enum');
17
+ const { checkIncompatibleValidationTransformations } = require('../validation-conflicts');
16
18
 
17
19
  /**
18
- * Enum with all available transformations
20
+ * The default transformation pipeline for the quick conversion mode (the CLI
21
+ * `-i` option): converts /etc/hosts rules to AdGuard-syntax rules.
19
22
  */
20
- const TRANSFORMATIONS = Object.freeze({
21
- RemoveComments: 'RemoveComments',
22
- Compress: 'Compress',
23
- RemoveModifiers: 'RemoveModifiers',
24
- Validate: 'Validate',
25
- ValidateAllowIp: 'ValidateAllowIp',
26
- ValidateAllowPublicSuffix: 'ValidateAllowPublicSuffix',
27
- ValidateAllowIpAndPublicSuffix: 'ValidateAllowIpAndPublicSuffix',
28
- Deduplicate: 'Deduplicate',
29
- InvertAllow: 'InvertAllow',
30
- RemoveEmptyLines: 'RemoveEmptyLines',
31
- TrimLines: 'TrimLines',
32
- InsertFinalNewLine: 'InsertFinalNewLine',
33
- ConvertToAscii: 'ConvertToAscii',
34
- });
35
-
36
- const VALIDATION_TRANSFORMATIONS = [
23
+ const DEFAULT_TRANSFORMATIONS = Object.freeze([
24
+ TRANSFORMATIONS.RemoveComments,
25
+ TRANSFORMATIONS.Deduplicate,
26
+ TRANSFORMATIONS.Compress,
37
27
  TRANSFORMATIONS.Validate,
38
- TRANSFORMATIONS.ValidateAllowIp,
39
- TRANSFORMATIONS.ValidateAllowPublicSuffix,
40
- TRANSFORMATIONS.ValidateAllowIpAndPublicSuffix,
41
- ];
28
+ TRANSFORMATIONS.TrimLines,
29
+ TRANSFORMATIONS.InsertFinalNewLine,
30
+ ]);
42
31
 
43
32
  /**
44
- * Throws when multiple validation transformations are selected together.
45
- * Combining them causes silent data loss: each validator runs on the already-filtered
46
- * output of the previous one, so allow-modes (AllowIp, AllowPublicSuffix) become ineffective.
33
+ * The transformations in their fixed execution order, paired with their implementations.
47
34
  *
48
- * @param {Array<string>} transformations - configured transformations.
35
+ * When adding a transformation: add it to the `TRANSFORMATIONS` enum in
36
+ * `enum.js`, then register it here (order matters — it is the execution order
37
+ * and MUST match the order list in README.md). A test in `transform.test.js`
38
+ * asserts this list covers every entry of the enum and pins its order, so a
39
+ * forgotten branch or an accidental reorder fails the suite instead of being
40
+ * silently skipped.
49
41
  */
50
- function checkIncompatibleValidationTransformations(transformations) {
51
- const selectedValidations = VALIDATION_TRANSFORMATIONS
52
- .filter((transformation) => transformations.indexOf(transformation) !== -1);
53
-
54
- if (selectedValidations.length > 1) {
55
- throw new Error(`Validation transformations cannot be combined: ${selectedValidations.join(', ')}.`);
56
- }
57
- }
42
+ const TRANSFORMATIONS_IN_ORDER = [
43
+ [TRANSFORMATIONS.ConvertToAscii, convertToAscii],
44
+ [TRANSFORMATIONS.TrimLines, trimLines],
45
+ [TRANSFORMATIONS.RemoveComments, removeComments],
46
+ [TRANSFORMATIONS.Compress, compress],
47
+ [TRANSFORMATIONS.RemoveModifiers, removeModifiers],
48
+ [TRANSFORMATIONS.InvertAllow, invertAllow],
49
+ [TRANSFORMATIONS.Validate, validate],
50
+ [TRANSFORMATIONS.ValidateAllowIp, validateAllowIp],
51
+ [TRANSFORMATIONS.ValidateAllowPublicSuffix, validateAllowPublicSuffix],
52
+ [TRANSFORMATIONS.ValidateAllowIpAndPublicSuffix, validateAllowIpAndPublicSuffix],
53
+ [TRANSFORMATIONS.Deduplicate, deduplicate],
54
+ [TRANSFORMATIONS.RemoveEmptyLines, removeEmptyLines],
55
+ [TRANSFORMATIONS.InsertFinalNewLine, insertFinalNewLine],
56
+ ];
58
57
 
59
58
  /**
60
59
  * Applies the specified transformations to the list of rules in the proper order.
@@ -86,50 +85,17 @@ async function transform(rules, configuration, transformations) {
86
85
  configuration.inclusions_sources,
87
86
  );
88
87
 
89
- // When updating the list of transformations, be sure to follow the same order in README.md
90
- if (transformations.indexOf(TRANSFORMATIONS.ConvertToAscii) !== -1) {
91
- transformed = convertToAscii(transformed);
92
- }
93
- if (transformations.indexOf(TRANSFORMATIONS.TrimLines) !== -1) {
94
- transformed = trimLines(transformed);
95
- }
96
- if (transformations.indexOf(TRANSFORMATIONS.RemoveComments) !== -1) {
97
- transformed = removeComments(transformed);
98
- }
99
- if (transformations.indexOf(TRANSFORMATIONS.Compress) !== -1) {
100
- transformed = compress(transformed);
101
- }
102
- if (transformations.indexOf(TRANSFORMATIONS.RemoveModifiers) !== -1) {
103
- transformed = removeModifiers(transformed);
104
- }
105
- if (transformations.indexOf(TRANSFORMATIONS.InvertAllow) !== -1) {
106
- transformed = invertAllow(transformed);
107
- }
108
- if (transformations.indexOf(TRANSFORMATIONS.Validate) !== -1) {
109
- transformed = validate(transformed);
110
- }
111
- if (transformations.indexOf(TRANSFORMATIONS.ValidateAllowIp) !== -1) {
112
- transformed = validateAllowIp(transformed);
113
- }
114
- if (transformations.indexOf(TRANSFORMATIONS.ValidateAllowPublicSuffix) !== -1) {
115
- transformed = validateAllowPublicSuffix(transformed);
116
- }
117
- if (transformations.indexOf(TRANSFORMATIONS.ValidateAllowIpAndPublicSuffix) !== -1) {
118
- transformed = validateAllowIpAndPublicSuffix(transformed);
119
- }
120
- if (transformations.indexOf(TRANSFORMATIONS.Deduplicate) !== -1) {
121
- transformed = deduplicate(transformed);
122
- }
123
- if (transformations.indexOf(TRANSFORMATIONS.RemoveEmptyLines) !== -1) {
124
- transformed = removeEmptyLines(transformed);
125
- }
126
- if (transformations.indexOf(TRANSFORMATIONS.InsertFinalNewLine) !== -1) {
127
- transformed = insertFinalNewLine(transformed);
88
+ // eslint-disable-next-line no-restricted-syntax
89
+ for (const [name, apply] of TRANSFORMATIONS_IN_ORDER) {
90
+ if (transformations.indexOf(name) !== -1) {
91
+ transformed = apply(transformed);
92
+ }
128
93
  }
129
94
  return transformed;
130
95
  }
131
96
 
132
97
  module.exports = {
133
98
  transform,
134
- TRANSFORMATIONS,
99
+ DEFAULT_TRANSFORMATIONS,
100
+ TRANSFORMATIONS_IN_ORDER,
135
101
  };
@@ -1,5 +1,5 @@
1
1
  const { Validator } = require('./validate');
2
- const { normalizeIpRules } = require('./ip-normalize');
2
+ const { normalizeIpRules } = require('../ip-normalize');
3
3
 
4
4
  /**
5
5
  * Validates a list of rules, allowing both IP addresses and public suffix matching.
@@ -1,5 +1,5 @@
1
1
  const { Validator } = require('./validate');
2
- const { normalizeIpRules } = require('./ip-normalize');
2
+ const { normalizeIpRules } = require('../ip-normalize');
3
3
 
4
4
  /**
5
5
  * Validates a list of rules and allows IP validation.