@adguard/hostlist-compiler 1.0.29 → 1.0.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -5,6 +5,27 @@ All notable changes to this project will be documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
7
 
8
+ ## [1.0.32] - 2024-12-27
9
+
10
+ ### Fixed
11
+
12
+ - Bug with checksum calculation [#63]
13
+
14
+ [1.0.32]: https://github.com/AdguardTeam/HostlistCompiler/compare/v1.0.31...v1.0.32
15
+
16
+ ## [1.0.31] - 2024-12-23
17
+
18
+ ### Fixed
19
+
20
+ - Validation of TLD domains [#63]
21
+
22
+ ### Added
23
+
24
+ - Calculation of checksum for filters and `! Checksum` string to the filter list meta [#76]
25
+
26
+ [1.0.31]: https://github.com/AdguardTeam/HostlistCompiler/compare/v1.0.29...v1.0.31
27
+ [#63]: https://github.com/AdguardTeam/HostlistCompiler/issues/63
28
+ [#76]: https://github.com/AdguardTeam/FiltersCompiler/issues/76
8
29
 
9
30
  ## [1.0.29] - 2024-09-26
10
31
 
package/README.md CHANGED
@@ -310,6 +310,10 @@ So here's what it does:
310
310
  - Discards rules with unsupported modifiers. [Click here](https://github.com/AdguardTeam/AdGuardHome/wiki/Hosts-Blocklists#-adblock-style-syntax) to learn more about which modifiers are supported.
311
311
  - Discards rules that are too short.
312
312
  - Discards IP addresses. If you need to keep IP addresses, use [ValidateAllowIp](#validate-allow-ip) instead.
313
+ - Removes rules that block entire top-level domains (TLDs) like `||*.org^`, unless they have specific limiting modifiers such as `$denyallow`, `$badfilter`, or `$client`.
314
+ Examples:
315
+ - `||*.org^` - this rule will be removed
316
+ - `||*.org^$denyallow=example.com` - this rule will be kept because it has a limiting modifier
313
317
 
314
318
  If there are comments preceding the invalid rule, they will be removed as well.
315
319
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@adguard/hostlist-compiler",
3
- "version": "1.0.29",
3
+ "version": "1.0.32",
4
4
  "description": "A simple tool that compiles hosts blocklists from multiple sources",
5
5
  "main": "src/index.js",
6
6
  "types": "src/index.d.ts",
package/src/index.d.ts CHANGED
@@ -1,12 +1,12 @@
1
1
  declare module '@adguard/hostlist-compiler' {
2
2
  export type SourceType = 'adblock' | 'hosts';
3
-
4
- export type Transformation =
5
- 'RemoveComments' |
6
- 'Compress' |
7
- 'RemoveModifiers' |
8
- 'Validate' |
9
- 'Deduplicate' |
3
+
4
+ export type Transformation =
5
+ 'RemoveComments' |
6
+ 'Compress' |
7
+ 'RemoveModifiers' |
8
+ 'Validate' |
9
+ 'Deduplicate' |
10
10
  'InvertAllow' |
11
11
  'RemoveEmptyLines' |
12
12
  'TrimLines' |
@@ -62,7 +62,7 @@ declare module '@adguard/hostlist-compiler' {
62
62
  * Compiles a filter list using the specified configuration.
63
63
  *
64
64
  * @param {*} configuration - compilation configuration.
65
- See the repo README for the details on it.
65
+ * See the repo README for the details on it.
66
66
  * @returns {Promise<Array<string>>} the array of rules.
67
67
  */
68
68
  declare async function compile(configuration: IConfiguration): Promise<string[]>;
package/src/index.js CHANGED
@@ -4,17 +4,17 @@ const config = require('./configuration');
4
4
  const compileSource = require('./compile-source');
5
5
  const { transform } = require('./transformations/transform');
6
6
  const packageJson = require('../package.json');
7
+ const { calculateChecksum } = require('./utils');
7
8
 
8
9
  /**
9
10
  * Prepares list header
10
11
  *
11
12
  * @param {*} configuration - compilation configuration.
12
- See the repo README for the details on it.
13
+ * See the repo README for the details on it.
13
14
  * @returns {Array<string>} header lines
14
15
  */
15
16
  function prepareHeader(configuration) {
16
17
  const lines = [
17
- '!',
18
18
  `! Title: ${configuration.name}`,
19
19
  ];
20
20
 
@@ -67,7 +67,7 @@ function prepareSourceHeader(source) {
67
67
  * Compiles a filter list using the specified configuration.
68
68
  *
69
69
  * @param {*} configuration - compilation configuration.
70
- See the repo README for the details on it.
70
+ * See the repo README for the details on it.
71
71
  * @returns {Promise<Array<string>>} the array of rules.
72
72
  */
73
73
  async function compile(configuration) {
@@ -101,8 +101,12 @@ async function compile(configuration) {
101
101
 
102
102
  // Now prepend the list header and we're good to go
103
103
  const header = prepareHeader(configuration);
104
- consola.info(`Final length of the list is ${header.length + finalList.length}`);
105
- return header.concat(finalList);
104
+ // Calculate checksum
105
+ const checksum = calculateChecksum(header, finalList);
106
+ // Concat everything together
107
+ const data = [checksum, ...header, ...finalList];
108
+ consola.info(`Final length of the list is ${data.length}`);
109
+ return data;
106
110
  }
107
111
 
108
112
  module.exports = compile;
@@ -4,20 +4,36 @@ const tldts = require('tldts');
4
4
  const utils = require('../utils');
5
5
  const ruleUtils = require('../rule');
6
6
 
7
+ const DOMAIN_PREFIX = '||';
8
+ const DOMAIN_SEPARATOR = '^';
9
+ const WILDCARD = '*';
10
+ const WILDCARD_DOMAIN_PART = '*.';
11
+
12
+ // TODO: Remove lodash from the project if possible
13
+
14
+ /**
15
+ * The list of modifiers that limit the rule for specific domains.
16
+ */
17
+ const ANY_PATTERN_MODIFIER = [
18
+ 'denyallow',
19
+ 'badfilter',
20
+ // DNS-related modifiers.
21
+ 'client',
22
+ ];
23
+
7
24
  /**
8
25
  * The list of modifiers supported by hosts-level blockers.
9
26
  */
10
27
  const SUPPORTED_MODIFIERS = [
11
28
  'important',
12
29
  '~important',
13
- 'badfilter',
14
30
  'ctag',
15
- 'denyallow',
16
31
  // DNS-related modifiers.
17
- 'client',
18
32
  'dnstype',
19
33
  'dnsrewrite',
20
34
  'ctag',
35
+ // modifiers that limit the rule for specific domains
36
+ ...ANY_PATTERN_MODIFIER,
21
37
  ];
22
38
 
23
39
  /**
@@ -31,9 +47,10 @@ const MAX_PATTERN_LENGTH = 5;
31
47
  * @param {string} hostname - hostname to check
32
48
  * @param {string} ruleText - original rule text (for logging)
33
49
  * @param {boolean} allowedIP - flag to determine if IP validation is allowed
50
+ * @param {boolean} hasLimitModifier - flag to determine if the rule has a limit modifier, e.g. denyallow
34
51
  * @returns {boolean} true if the hostname is okay to be in the blocklist.
35
52
  */
36
- function validHostname(hostname, ruleText, allowedIP) {
53
+ function validHostname(hostname, ruleText, allowedIP, hasLimitModifier) {
37
54
  const result = tldts.parse(hostname);
38
55
 
39
56
  if (!result.hostname) {
@@ -46,7 +63,7 @@ function validHostname(hostname, ruleText, allowedIP) {
46
63
  return false;
47
64
  }
48
65
 
49
- if (result.hostname === result.publicSuffix) {
66
+ if (result.hostname === result.publicSuffix && !hasLimitModifier) {
50
67
  consola.debug(`matching the whole public suffix ${hostname} is not allowed: ${ruleText}`);
51
68
  return false;
52
69
  }
@@ -108,6 +125,9 @@ function validAdblockRule(ruleText, allowedIP) {
108
125
  return false;
109
126
  }
110
127
 
128
+ // need to check if rules with TLD has limit modifier
129
+ let hasLimitModifier = false;
130
+
111
131
  // 1. It checks if the rule contains only supported modifiers.
112
132
  if (props.options) {
113
133
  // eslint-disable-next-line no-restricted-syntax
@@ -116,6 +136,10 @@ function validAdblockRule(ruleText, allowedIP) {
116
136
  consola.debug(`Contains unsupported modifier ${option.name}: ${ruleText}`);
117
137
  return false;
118
138
  }
139
+ // if the rule has a limit modifier, TLD as a hostname is allowed
140
+ if (ANY_PATTERN_MODIFIER.includes(option.name)) {
141
+ hasLimitModifier = true;
142
+ }
119
143
  }
120
144
  }
121
145
 
@@ -151,30 +175,49 @@ function validAdblockRule(ruleText, allowedIP) {
151
175
 
152
176
  // 4. Validate domain name
153
177
  // Note that we don't check rules that contain wildcard characters
154
- const sepIdx = props.pattern.indexOf('^');
155
- const wildcardIdx = props.pattern.indexOf('*');
178
+ const sepIdx = props.pattern.indexOf(DOMAIN_SEPARATOR);
179
+ const wildcardIdx = props.pattern.indexOf(WILDCARD);
156
180
  if (sepIdx !== -1 && wildcardIdx !== -1 && wildcardIdx > sepIdx) {
157
181
  // Smth like ||example.org^test* -- invalid
158
182
  return false;
159
183
  }
160
184
 
161
- if (_.startsWith(props.pattern, '||')
162
- && sepIdx !== -1
163
- && wildcardIdx === -1) {
164
- const hostname = utils.substringBetween(ruleText, '||', '^');
165
- if (!validHostname(hostname, ruleText, allowedIP)) {
166
- return false;
167
- }
185
+ // Check if the pattern does not start with the domain prefix and does not contain a domain separator
186
+ if (!_.startsWith(props.pattern, DOMAIN_PREFIX) || sepIdx === -1) {
187
+ return true;
188
+ }
189
+
190
+ // Extract the domain to check from the rule text
191
+ const domainToCheck = utils.substringBetween(ruleText, DOMAIN_PREFIX, DOMAIN_SEPARATOR);
168
192
 
169
- // If there's something after ^ in the pattern - something went wrong
170
- // unless it's `^|` which is a rather often case
171
- if (props.pattern.length > (sepIdx + 1)
172
- && props.pattern[sepIdx + 1] !== '|') {
173
- return false;
193
+ // If there are wildcard characters in the pattern
194
+ if (wildcardIdx !== -1) {
195
+ // Check if the rule has wildcard characters but includes only TLD (e.g., ||*.org^ or ||*.co.uk^)
196
+ const startsWithWildcard = domainToCheck.startsWith(WILDCARD_DOMAIN_PART);
197
+ // Get the TLD pattern to check (e.g. ||*.org^ --> org)
198
+ const TLDPattern = domainToCheck.replace(WILDCARD_DOMAIN_PART, '');
199
+ // Compare the TLD pattern with the public suffix
200
+ const isOnlyTLD = tldts.getPublicSuffix(TLDPattern) === TLDPattern;
201
+ // If it's a wildcard with TLD, validate the cleaned TLD
202
+ if (startsWithWildcard && isOnlyTLD) {
203
+ const cleanedDomain = domainToCheck.replace(WILDCARD_DOMAIN_PART, '');
204
+ return validHostname(cleanedDomain, ruleText, allowedIP, hasLimitModifier);
174
205
  }
206
+ // If the rule has wildcard characters but is not a TLD (e.g., ||*.example.org^)
207
+ return true;
175
208
  }
176
209
 
177
- return true;
210
+ // Validate the domain
211
+ if (!validHostname(domainToCheck, ruleText, allowedIP, hasLimitModifier)) {
212
+ return false;
213
+ }
214
+
215
+ // Ensure there's nothing after the domain separator unless it's `^|`
216
+ // If there's something after ^ in the pattern - something went wrong
217
+ // unless it's `^|` which is a rather often case
218
+ // ||example.org^| -- valid
219
+ // @@||example.org^|$important -- invalid
220
+ return !(props.pattern.length > (sepIdx + 1) && props.pattern[sepIdx + 1] !== '|');
178
221
  }
179
222
 
180
223
  /**
@@ -197,6 +240,7 @@ function valid(ruleText, allowedIP) {
197
240
  if (ruleUtils.isEtcHostsRule(ruleText)) {
198
241
  return validEtcHostsRule(ruleText, allowedIP);
199
242
  }
243
+
200
244
  return validAdblockRule(ruleText, allowedIP);
201
245
  }
202
246
 
package/src/utils.js CHANGED
@@ -1,7 +1,49 @@
1
1
  const _ = require('lodash');
2
2
  const fs = require('fs').promises;
3
+ const crypto = require('crypto');
3
4
  const axios = require('axios');
4
5
 
6
+ /**
7
+ * Strips specified character from the end of a string.
8
+ *
9
+ * @param {string} string - The string to strip.
10
+ * @param {string} char - The character to strip.
11
+ * @returns {string} - The stripped string.
12
+ */
13
+ const stripEnd = (str, char) => {
14
+ let result = str;
15
+ while (result.endsWith(char)) {
16
+ result = result.slice(0, -1);
17
+ }
18
+ return result;
19
+ };
20
+
21
+ /**
22
+ * Normalizes data by replacing newlines.
23
+ *
24
+ * @param {string} message - The message to normalize.
25
+ * @returns {string} - The normalized message.
26
+ */
27
+ const normalizeData = (message) => {
28
+ return message.replace(/\r/g, '').replace(/\n+/g, '\n');
29
+ };
30
+
31
+ /**
32
+ * Calculates checksum for the given header and rules.
33
+ * See:
34
+ * https://adblockplus.org/en/filters#special-comments
35
+ * https://hg.adblockplus.org/adblockplus/file/tip/addChecksum.py
36
+ *
37
+ * @param {Array<string>} header - The header lines.
38
+ * @param {Array<string>} rules - The rules lines.
39
+ * @returns {string} - The calculated checksum.
40
+ */
41
+ const calculateChecksum = (header, rules) => {
42
+ const content = normalizeData(header.concat(rules).join('\n'));
43
+ const checksum = crypto.createHash('md5').update(content).digest('base64');
44
+ return `! Checksum: ${stripEnd(checksum.trim(), '=')}`;
45
+ };
46
+
5
47
  function isURL(str) {
6
48
  try {
7
49
  // eslint-disable-next-line no-unused-vars
@@ -169,6 +211,7 @@ class Wildcard {
169
211
  }
170
212
 
171
213
  module.exports = {
214
+ calculateChecksum,
172
215
  download,
173
216
  Wildcard,
174
217
  splitByDelimiterWithEscapeCharacter,
@@ -1,6 +1,7 @@
1
1
  const nock = require('nock');
2
2
  const consola = require('consola');
3
3
  const compile = require('../src/index');
4
+ const { calculateChecksum } = require('../src/utils');
4
5
 
5
6
  describe('Hostlist compiler', () => {
6
7
  it('compile from multiple sources', async () => {
@@ -32,6 +33,7 @@ describe('Hostlist compiler', () => {
32
33
  const list = await compile(configuration);
33
34
 
34
35
  // assert
36
+ expect(list[0].startsWith('! Checksum:')).toBe(true);
35
37
  expect(list).toContain('||example.org');
36
38
  expect(list).toContain('||example.com');
37
39
  expect(list).toContain('! Version: 1.0.0.9');
@@ -41,4 +43,39 @@ describe('Hostlist compiler', () => {
41
43
 
42
44
  scope.done();
43
45
  });
46
+
47
+ it('calculates checksum correctly', async () => {
48
+ // Prepare source
49
+ const scope = nock('https://example.org')
50
+ .get('/source1.txt')
51
+ .reply(200, '||example.org');
52
+
53
+ // compiler configuration
54
+ const configuration = {
55
+ name: 'Test filter',
56
+ description: 'Checksum test filter',
57
+ version: '1.0.0.1',
58
+ sources: [
59
+ {
60
+ name: 'source 1',
61
+ source: 'https://example.org/source1.txt',
62
+ },
63
+ ],
64
+ };
65
+
66
+ // compile the final list
67
+ const list = await compile(configuration);
68
+
69
+ // assert
70
+ const checksumLine = list[0];
71
+ expect(checksumLine.startsWith('! Checksum:')).toBe(true);
72
+
73
+ const header = list.slice(1, list.indexOf('||example.org'));
74
+ const finalList = list.slice(list.indexOf('||example.org'));
75
+
76
+ const expectedChecksum = calculateChecksum(header, finalList);
77
+ expect(checksumLine).toBe(expectedChecksum);
78
+
79
+ scope.done();
80
+ });
44
81
  });
@@ -67,4 +67,70 @@ describe('Validate', () => {
67
67
  '@@||example.org^|$important',
68
68
  ]);
69
69
  });
70
+
71
+ it('adblock-style rules with wildcard and denyallow modifier', () => {
72
+ const rules = `||*.org^$denyallow=example.com
73
+ ||*.asia^
74
+ ||*.example.org^
75
+ ||*.asia^$denyallow=fap.bar
76
+ ||xyz^$denyallow=example.com
77
+ ||xyz^`.split(/\r?\n/);
78
+ const filtered = validate(rules);
79
+
80
+ expect(filtered).toEqual([
81
+ '||*.org^$denyallow=example.com',
82
+ '||*.example.org^',
83
+ '||*.asia^$denyallow=fap.bar',
84
+ '||xyz^$denyallow=example.com',
85
+ ]);
86
+ });
87
+
88
+ it('adblock-style rules with wildcard and badfilter modifier', () => {
89
+ const rules = `||*.org^$badfilter
90
+ ||*.asia^
91
+ ||*.example.org^
92
+ ||*.asia^$badfilter
93
+ ||xyz^$badfilter
94
+ ||xyz^`.split(/\r?\n/);
95
+ const filtered = validate(rules);
96
+
97
+ expect(filtered).toEqual([
98
+ '||*.org^$badfilter',
99
+ '||*.example.org^',
100
+ '||*.asia^$badfilter',
101
+ '||xyz^$badfilter',
102
+ ]);
103
+ });
104
+
105
+ it('adblock-style rules with wildcard and client modifier', () => {
106
+ const rules = `@@||*.org^$client=127.0.0.1
107
+ ||*.asia^
108
+ ||*.example.org^
109
+ ||*.asia^$client=192.168.0.0/24
110
+ ||xyz^$client=192.168.0.0/24
111
+ ||xyz^`.split(/\r?\n/);
112
+ const filtered = validate(rules);
113
+
114
+ expect(filtered).toEqual([
115
+ '@@||*.org^$client=127.0.0.1',
116
+ '||*.example.org^',
117
+ '||*.asia^$client=192.168.0.0/24',
118
+ '||xyz^$client=192.168.0.0/24',
119
+ ]);
120
+ });
121
+
122
+ it('check for composite TLDs', () => {
123
+ const rules = `||*.com.tr^$denyallow=example.com
124
+ ||*.com.tr^
125
+ ||*.co.uk^$client=127.0.0.1
126
+ ||*.co.uk^
127
+ ||*.example.org^`.split(/\r?\n/);
128
+ const filtered = validate(rules);
129
+
130
+ expect(filtered).toEqual([
131
+ '||*.com.tr^$denyallow=example.com',
132
+ '||*.co.uk^$client=127.0.0.1',
133
+ '||*.example.org^',
134
+ ]);
135
+ });
70
136
  });
@@ -34,3 +34,32 @@ describe('substringBetween', () => {
34
34
  expect(substr).toBe(null);
35
35
  });
36
36
  });
37
+
38
+ describe('calculateChecksum', () => {
39
+ it('calculates checksum with multiple headers and rules', () => {
40
+ const header = ['! Title: AdGuard DNS filter', '! Title: Example filter list'];
41
+ const rules = ['||example.com^', '||test.com^'];
42
+ const expectedChecksum = '! Checksum: VSbHtv4WyU405Fks327CsA';
43
+
44
+ const checksum = utils.calculateChecksum(header, rules);
45
+ expect(checksum).toBe(expectedChecksum);
46
+ });
47
+
48
+ it('calculates checksum with empty rules', () => {
49
+ const header = ['[Adblock Plus 2.0]'];
50
+ const rules = [];
51
+ const expectedChecksum = '! Checksum: 87GLA4VsMnV9PiusIkPkEg';
52
+
53
+ const checksum = utils.calculateChecksum(header, rules);
54
+ expect(checksum).toBe(expectedChecksum);
55
+ });
56
+
57
+ it('calculates checksum with empty header and rules', () => {
58
+ const header = [];
59
+ const rules = [];
60
+ const expectedChecksum = '! Checksum: 1B2M2Y8AsgTpgAmY7PhCfg';
61
+
62
+ const checksum = utils.calculateChecksum(header, rules);
63
+ expect(checksum).toBe(expectedChecksum);
64
+ });
65
+ });