@adguard/hostlist-compiler 1.0.35 → 1.0.38

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -5,6 +5,28 @@ All notable changes to this project will be documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
7
 
8
+ ## [1.0.38] - 2025-03-06
9
+
10
+ ### Changed
11
+
12
+ - Updated [@adguard/filters-downloader] to 2.3.0.
13
+
14
+ ### Fixed
15
+
16
+ - Path determination issue. [#82]
17
+
18
+ [1.0.38]: https://github.com/AdguardTeam/HostlistCompiler/compare/v1.0.37...v1.0.38
19
+ [#82]: https://github.com/AdguardTeam/HostlistCompiler/issues/82
20
+
21
+ ## [1.0.37] - 2025-02-28
22
+
23
+ ### Added
24
+
25
+ - Resolving the `!#include` directive by using `@adguard/filters-downloader`. [#78]
26
+
27
+ [1.0.37]: https://github.com/AdguardTeam/HostlistCompiler/compare/v1.0.35...v1.0.37
28
+ [#78]: https://github.com/AdguardTeam/HostlistCompiler/issues/78
29
+
8
30
  ## [1.0.35] - 2025-02-07
9
31
 
10
32
  ### Added
@@ -79,3 +101,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
79
101
 
80
102
  [1.0.27]: https://github.com/AdguardTeam/HostlistCompiler/compare/v1.0.26...v1.0.27
81
103
  [#69]: https://github.com/AdguardTeam/FiltersCompiler/issues/69
104
+
105
+ [@adguard/filters-downloader]: https://github.com/AdguardTeam/FiltersDownloader/blob/master/CHANGELOG.md
package/README.md CHANGED
@@ -152,6 +152,55 @@ Please note, that exclusion or inclusion rules may be a plain string, wildcard,
152
152
  - `/regex/` - every rule that matches this regular expression, will match the rule. By default, regular expressions are case-insensitive.
153
153
  - `! comment` - comments will be ignored.
154
154
 
155
+ > [!IMPORTANT]
156
+ > Ensure that rules in the exclusion list match the format of the rules in the filter list.
157
+ > To maintain a consistent format, add the `Compress` transformation to convert `/etc/hosts` rules to adblock syntax.
158
+ > This is especially useful if you have multiple lists in different formats.
159
+
160
+ Here is an example:
161
+
162
+ Rules in HOSTS syntax: `/hosts.txt`
163
+
164
+ ```txt
165
+ 0.0.0.0 ads.example.com
166
+ 0.0.0.0 tracking.example1.com
167
+ 0.0.0.0 example.com
168
+ ```
169
+
170
+ Exclusion rules in adblock syntax: `/exclusions.txt`
171
+
172
+ ```txt
173
+ ||example.com^
174
+ ```
175
+
176
+ Configuration of the final list:
177
+
178
+ ```json
179
+ {
180
+ "name": "List name",
181
+ "description": "List description",
182
+ "sources": [
183
+ {
184
+ "name": "HOSTS rules",
185
+ "source": "hosts.txt",
186
+ "type": "hosts",
187
+ "transformations": ["Compress"]
188
+ }
189
+ ],
190
+ "transformations": ["Deduplicate", "Compress"],
191
+ "exclusions_sources": ["exclusions.txt"]
192
+ }
193
+ ```
194
+
195
+ Final filter output of `/hosts.txt` after applying the `Compress` transformation and exclusions:
196
+
197
+ ```txt
198
+ ||ads.example.com^
199
+ ||tracking.example1.com^
200
+ ```
201
+
202
+ The last rule now `||example.com^` will correctly match the rule from the exclusion list and will be excluded.
203
+
155
204
  ### <a name="command-line"></a> Command-line
156
205
 
157
206
  Command-line arguments.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@adguard/hostlist-compiler",
3
- "version": "1.0.35",
3
+ "version": "1.0.38",
4
4
  "description": "A simple tool that compiles hosts blocklists from multiple sources",
5
5
  "main": "src/index.js",
6
6
  "types": "src/index.d.ts",
@@ -23,11 +23,12 @@
23
23
  "eslint": "^8.34.0",
24
24
  "eslint-config-airbnb-base": "^15.0.0",
25
25
  "eslint-plugin-import": "^2.27.5",
26
- "jest": "^25.1.0",
26
+ "jest": "^29.7.0",
27
27
  "mock-fs": "^4.11.0",
28
28
  "nock": "^12.0.2"
29
29
  },
30
30
  "dependencies": {
31
+ "@adguard/filters-downloader": "^2.3.0",
31
32
  "ajv": "^6.12.0",
32
33
  "ajv-errors": "^1.0.1",
33
34
  "axios": "^0.19.2",
@@ -1,5 +1,5 @@
1
1
  const consola = require('consola');
2
- const utils = require('./utils');
2
+ const { FiltersDownloader } = require('@adguard/filters-downloader');
3
3
  const { transform } = require('./transformations/transform');
4
4
 
5
5
  /**
@@ -21,8 +21,7 @@ const { transform } = require('./transformations/transform');
21
21
  */
22
22
  async function compileSource(source) {
23
23
  consola.info(`Start compiling ${source.source}`);
24
- const str = await utils.download(source.source);
25
- let rules = str.split(/\r?\n/);
24
+ let rules = await FiltersDownloader.download(source.source);
26
25
  consola.info(`Original length is ${rules.length}`);
27
26
 
28
27
  rules = await transform(rules, source, source.transformations);
package/src/filter.js CHANGED
@@ -1,4 +1,5 @@
1
1
  const _ = require('lodash');
2
+ const { FiltersDownloader } = require('@adguard/filters-downloader');
2
3
  const utils = require('./utils');
3
4
  const ruleUtils = require('./rule');
4
5
 
@@ -16,9 +17,8 @@ async function downloadAll(sources) {
16
17
  }
17
18
 
18
19
  await Promise.all(sources.map(async (s) => {
19
- const rulesStr = await utils.download(s);
20
+ const rulesStr = await FiltersDownloader.download(s);
20
21
  const rules = rulesStr
21
- .split(/\r?\n/)
22
22
  .filter((el) => el.trim().length > 0 && !ruleUtils.isComment(el));
23
23
  list = list.concat(rules);
24
24
  }));
package/src/utils.js CHANGED
@@ -1,36 +1,4 @@
1
1
  const _ = require('lodash');
2
- const fs = require('fs').promises;
3
- const axios = require('axios');
4
-
5
- function isURL(str) {
6
- try {
7
- // eslint-disable-next-line no-unused-vars
8
- const u = new URL(str);
9
- return true;
10
- } catch {
11
- return false;
12
- }
13
- }
14
-
15
- /**
16
- * Downloads (or reads from the disk) the specified source
17
- *
18
- * @param {*} urlOrPath url or path to a file
19
- * @returns {Promise<String>} contents of the files
20
- */
21
- async function download(urlOrPath) {
22
- let str = '';
23
-
24
- if (isURL(urlOrPath)) {
25
- const url = new URL(urlOrPath);
26
- const response = await axios.get(url.toString(), { responseType: 'text' });
27
- str = response.data;
28
- } else {
29
- str = (await fs.readFile(urlOrPath)).toString();
30
- }
31
-
32
- return str;
33
- }
34
2
 
35
3
  /**
36
4
  * Extracts a substring between two tags.
@@ -169,7 +137,6 @@ class Wildcard {
169
137
  }
170
138
 
171
139
  module.exports = {
172
- download,
173
140
  Wildcard,
174
141
  splitByDelimiterWithEscapeCharacter,
175
142
  substringBetween,
@@ -1,8 +1,12 @@
1
1
  const nock = require('nock');
2
2
  const mock = require('mock-fs');
3
+ const path = require('path');
3
4
  const compileSource = require('../src/compile-source');
4
5
  const { TRANSFORMATIONS } = require('../src/transformations/transform');
5
6
 
7
+ const testDirPath = path.resolve(__dirname, 'test/dir');
8
+ const exclusionsFilePath = path.resolve(testDirPath, 'exclusions.txt');
9
+
6
10
  describe('Source compiler', () => {
7
11
  afterEach(() => {
8
12
  // make sure FS is restored after running tests
@@ -12,7 +16,9 @@ describe('Source compiler', () => {
12
16
  it('compile a simple URL source', async () => {
13
17
  const scope = nock('https://example.org')
14
18
  .get('/filter.txt')
15
- .reply(200, 'testrule');
19
+ .reply(200, 'testrule', {
20
+ 'Content-Type': 'text/plain',
21
+ });
16
22
 
17
23
  const source = {
18
24
  name: 'test source',
@@ -33,6 +39,7 @@ describe('Source compiler', () => {
33
39
  'rules.txt': 'testrule',
34
40
  },
35
41
  });
42
+
36
43
  const source = {
37
44
  name: 'test source',
38
45
  source: 'test/dir/rules.txt',
@@ -53,13 +60,15 @@ describe('Source compiler', () => {
53
60
  ||rule4`;
54
61
  const scope = nock('https://example.org')
55
62
  .get('/filter.txt')
56
- .reply(200, rules);
63
+ .reply(200, rules, {
64
+ 'Content-Type': 'text/plain',
65
+ });
57
66
 
58
67
  // STEP 2: MOCK EXCLUSIONS
59
68
  const exclusions = `||rule1
60
69
  ||rule3`;
61
70
  mock({
62
- 'test/dir': {
71
+ [testDirPath]: {
63
72
  'exclusions.txt': exclusions,
64
73
  },
65
74
  });
@@ -68,7 +77,7 @@ describe('Source compiler', () => {
68
77
  name: 'test source',
69
78
  source: 'https://example.org/filter.txt',
70
79
  exclusions: ['rule4'],
71
- exclusions_sources: ['test/dir/exclusions.txt'],
80
+ exclusions_sources: [exclusionsFilePath],
72
81
  transformations: [
73
82
  TRANSFORMATIONS.Deduplicate,
74
83
  TRANSFORMATIONS.Validate,
@@ -99,7 +108,9 @@ describe('Source compiler', () => {
99
108
  test2.com`;
100
109
  const scope = nock('https://example.org')
101
110
  .get('/test-filter.txt')
102
- .reply(200, testList);
111
+ .reply(200, testList, {
112
+ 'Content-Type': 'text/plain',
113
+ });
103
114
 
104
115
  const source = {
105
116
  name: 'test source',
@@ -134,7 +145,9 @@ describe('Source compiler', () => {
134
145
  ||*.հայ^`;
135
146
  const scope = nock('https://example.org')
136
147
  .get('/test-filter.txt')
137
- .reply(200, testList);
148
+ .reply(200, testList, {
149
+ 'Content-Type': 'text/plain',
150
+ });
138
151
 
139
152
  const source = {
140
153
  name: 'test source',
@@ -7,9 +7,13 @@ describe('Hostlist compiler', () => {
7
7
  // Prepare source 1
8
8
  const scope = nock('https://example.org')
9
9
  .get('/source1.txt')
10
- .reply(200, '||example.org')
10
+ .reply(200, '||example.org', {
11
+ 'Content-Type': 'text/plain',
12
+ })
11
13
  .get('/source2.txt')
12
- .reply(200, '||example.com');
14
+ .reply(200, '||example.com', {
15
+ 'Content-Type': 'text/plain',
16
+ });
13
17
 
14
18
  // compiler configuration
15
19
  const configuration = {
@@ -0,0 +1,296 @@
1
+ const nock = require('nock');
2
+ const consola = require('consola');
3
+ const compile = require('../src/index');
4
+
5
+ describe('Hostlist compiler', () => {
6
+ it('compile from one source with nested includes', async () => {
7
+ // Prepare filters content
8
+ const filterContent1 = `! this is a source
9
+ ||example.org
10
+ ||example.com
11
+ !#include https://example.org/source2.txt`;
12
+
13
+ const filterContent2 = `
14
+ ||example.net
15
+ ||example.io
16
+ !#include https://example.org/source3.txt`;
17
+
18
+ const filterContent3 = `
19
+ last.include.com
20
+ non/valid_rule`;
21
+
22
+ // Prepare source
23
+ const scope = nock('https://example.org')
24
+ .get('/source1.txt')
25
+ .reply(200, filterContent1, {
26
+ 'Content-Type': 'text/plain',
27
+ })
28
+ .get('/source2.txt')
29
+ .reply(200, filterContent2, {
30
+ 'Content-Type': 'text/plain',
31
+ })
32
+ .get('/source3.txt')
33
+ .reply(200, filterContent3, {
34
+ 'Content-Type': 'text/plain',
35
+ });
36
+
37
+ // compiler configuration
38
+ const configuration = {
39
+ name: 'Test filter',
40
+ description: 'Our test filter',
41
+ version: '1.0.0.9',
42
+ sources: [
43
+ {
44
+ name: 'source 1',
45
+ source: 'https://example.org/source1.txt',
46
+ },
47
+ ],
48
+ transformations: [
49
+ 'RemoveComments',
50
+ 'Compress',
51
+ 'InsertFinalNewLine',
52
+ 'Validate',
53
+ ],
54
+ };
55
+
56
+ // compile the final list
57
+ const list = await compile(configuration);
58
+
59
+ // assert
60
+ const expectedRules = [
61
+ '||example.org',
62
+ '||example.com',
63
+ '||example.net',
64
+ '||example.io',
65
+ '||last.include.com^',
66
+ ];
67
+
68
+ expectedRules.forEach((rule) => {
69
+ expect(list).toContain(rule);
70
+ });
71
+
72
+ expect(list).not.toContain('non/valid_rule');
73
+
74
+ expect(list).toContain('! Version: 1.0.0.9');
75
+
76
+ const str = list.join('\n');
77
+ consola.info(str);
78
+
79
+ scope.done();
80
+ });
81
+
82
+ it('compile from one source with include and exclude file', async () => {
83
+ // Prepare filters content
84
+ const filterContent1 = `! this is a source
85
+ ads.example.com
86
+ trackers.exclude.com
87
+ exclude.com
88
+ !#include source2.txt`;
89
+
90
+ const filterContent2 = `
91
+ popups.com
92
+ event.com`;
93
+
94
+ const exclusionsContent = `
95
+ exclude.com
96
+ non/valid_rule`;
97
+
98
+ // Prepare source
99
+ const scope = nock('https://example.org')
100
+ .get('/source1.txt')
101
+ .reply(200, filterContent1, {
102
+ 'Content-Type': 'text/plain',
103
+ })
104
+ .get('/source2.txt')
105
+ .reply(200, filterContent2, {
106
+ 'Content-Type': 'text/plain',
107
+ })
108
+ .get('/exclusions.txt')
109
+ .reply(200, exclusionsContent, {
110
+ 'Content-Type': 'text/plain',
111
+ });
112
+
113
+ // compiler configuration
114
+ const configuration = {
115
+ name: 'Test filter',
116
+ description: 'Our test filter',
117
+ version: '1.0.0.9',
118
+ sources: [
119
+ {
120
+ name: 'source 1',
121
+ source: 'https://example.org/source1.txt',
122
+ },
123
+ ],
124
+ transformations: [
125
+ 'RemoveComments',
126
+ 'Compress',
127
+ 'InsertFinalNewLine',
128
+ 'Validate',
129
+ ],
130
+ exclusions_sources: [
131
+ 'https://example.org/exclusions.txt',
132
+ ],
133
+ };
134
+
135
+ // compile the final list
136
+ const list = await compile(configuration);
137
+
138
+ // assert
139
+ const expectedRules = [
140
+ '||ads.example.com^',
141
+ '||popups.com^',
142
+ '||event.com^',
143
+ ];
144
+
145
+ expectedRules.forEach((rule) => {
146
+ expect(list).toContain(rule);
147
+ });
148
+
149
+ expect(list).not.toContain('||exclude.com^', 'non/valid_rule');
150
+
151
+ expect(list).toContain('! Version: 1.0.0.9');
152
+
153
+ const str = list.join('\n');
154
+ consola.info(str);
155
+
156
+ scope.done();
157
+ });
158
+
159
+ it('compile from local source with local include', async () => {
160
+ // compiler configuration
161
+ const configuration = {
162
+ name: 'Test filter',
163
+ description: 'Our test filter',
164
+ version: '1.0.0.9',
165
+ sources: [
166
+ {
167
+ name: 'filter',
168
+ source: 'test/resources/rules.txt',
169
+ },
170
+ ],
171
+ transformations: [
172
+ 'RemoveComments',
173
+ 'Compress',
174
+ 'InsertFinalNewLine',
175
+ 'Validate',
176
+ ],
177
+ };
178
+
179
+ // compile the final list
180
+ const list = await compile(configuration);
181
+
182
+ const str = list.join('\n');
183
+ consola.info(str);
184
+
185
+ const expectedRules = [
186
+ '||sat.terithrow2.net^',
187
+ '||sat.fevilsor5.net^',
188
+ '||leegreemula.net^',
189
+ '||should.be.included.com^',
190
+ '||last.rule^',
191
+ ];
192
+
193
+ expectedRules.forEach((rule) => {
194
+ expect(list).toContain(rule);
195
+ });
196
+ });
197
+
198
+ it('compile from external source with nested includes', async () => {
199
+ // Prepare filters content
200
+ const filterContent1 = `! this is a source
201
+ ||sub_test_main
202
+ !#include source2.txt`;
203
+
204
+ const filterContent2 = `
205
+ ||sub_test
206
+ !#include subdir/source3.txt`;
207
+
208
+ const filterContent3 = `
209
+ ||sub_test_final`;
210
+
211
+ // Prepare source
212
+ const scope = nock('https://example.org')
213
+ .get('/testdir/source1.txt')
214
+ .reply(200, filterContent1, {
215
+ 'Content-Type': 'text/plain',
216
+ })
217
+ .get('/testdir/source2.txt')
218
+ .reply(200, filterContent2, {
219
+ 'Content-Type': 'text/plain',
220
+ })
221
+ .get('/testdir/subdir/source3.txt')
222
+ .reply(200, filterContent3, {
223
+ 'Content-Type': 'text/plain',
224
+ });
225
+
226
+ // compiler configuration
227
+ const configuration = {
228
+ name: 'Test filter',
229
+ description: 'Our test filter',
230
+ version: '1.0.0.9',
231
+ sources: [
232
+ {
233
+ name: 'filter',
234
+ source: 'https://example.org/testdir/source1.txt',
235
+ },
236
+ ],
237
+ transformations: [
238
+ 'RemoveComments',
239
+ ],
240
+ exclusions: ['test_parent'],
241
+ };
242
+
243
+ // compile the final list
244
+ const list = await compile(configuration);
245
+
246
+ const str = list.join('\n');
247
+ consola.info(str);
248
+
249
+ const expectedRules = [
250
+ '||sub_test_main',
251
+ '||sub_test',
252
+ '||sub_test_final',
253
+ ];
254
+
255
+ expectedRules.forEach((rule) => {
256
+ expect(list).toContain(rule);
257
+ });
258
+
259
+ scope.done();
260
+ });
261
+
262
+ it('should not compile source from different origin', async () => {
263
+ // Prepare filters content
264
+ const filterContent1 = `! this is a source
265
+ ads.example.com
266
+ trackers.exclude.com
267
+ exclude.com
268
+ !#include https://example1.org/source.txt`;
269
+
270
+ // Prepare source
271
+ const scope = nock('https://example.org')
272
+ .get('/source1.txt')
273
+ .reply(200, filterContent1, {
274
+ 'Content-Type': 'text/plain',
275
+ });
276
+
277
+ // compiler configuration
278
+ const configuration = {
279
+ name: 'Test filter',
280
+ description: 'Our test filter',
281
+ version: '1.0.0.9',
282
+ sources: [
283
+ {
284
+ name: 'source 1',
285
+ source: 'https://example.org/source1.txt',
286
+ },
287
+ ],
288
+ };
289
+
290
+ await expect(async () => {
291
+ await compile(configuration);
292
+ }).rejects.toThrow(Error);
293
+
294
+ scope.done();
295
+ });
296
+ });
@@ -0,0 +1,2 @@
1
+ should.be.included.com
2
+ !#include subinclude.txt
@@ -0,0 +1,4 @@
1
+ ||sat.terithrow2.net^
2
+ ||sat.fevilsor5.net^
3
+ ||leegreemula.net^
4
+ !#include includedfile.txt
@@ -0,0 +1 @@
1
+ last.rule
@@ -1,7 +1,11 @@
1
1
  const nock = require('nock');
2
2
  const mock = require('mock-fs');
3
+ const path = require('path');
3
4
  const exclude = require('../../src/transformations/exclude');
4
5
 
6
+ const testDirPath = path.resolve(__dirname, 'test/dir');
7
+ const exclusionsFilePath = path.resolve(testDirPath, 'exclusions.txt');
8
+
5
9
  describe('Exclusions', () => {
6
10
  afterEach(() => {
7
11
  // make sure FS is restored after running tests
@@ -20,11 +24,15 @@ describe('Exclusions', () => {
20
24
  // Mock exclusions
21
25
  const scope = nock('https://example.org')
22
26
  .get('/exclusions.txt')
23
- .reply(200, 'rule1')
27
+ .reply(200, 'rule1', {
28
+ 'Content-Type': 'text/plain',
29
+ })
24
30
  .get('/exclusions2.txt')
25
- .reply(200, 'rule2');
31
+ .reply(200, 'rule2', {
32
+ 'Content-Type': 'text/plain',
33
+ });
26
34
  mock({
27
- 'test/dir': {
35
+ [testDirPath]: {
28
36
  'exclusions.txt': 'rule3',
29
37
  },
30
38
  });
@@ -36,7 +44,7 @@ describe('Exclusions', () => {
36
44
  const filtered = await exclude(
37
45
  rules,
38
46
  ['rule4'],
39
- ['https://example.org/exclusions.txt', 'https://example.org/exclusions2.txt', 'test/dir/exclusions.txt'],
47
+ ['https://example.org/exclusions.txt', 'https://example.org/exclusions2.txt', exclusionsFilePath],
40
48
  );
41
49
 
42
50
  // Assert
@@ -1,7 +1,11 @@
1
1
  const nock = require('nock');
2
2
  const mock = require('mock-fs');
3
+ const path = require('path');
3
4
  const include = require('../../src/transformations/include');
4
5
 
6
+ const testDirPath = path.resolve(__dirname, 'test/dir');
7
+ const inclusionsFilePath = path.resolve(testDirPath, 'inclusions.txt');
8
+
5
9
  describe('Exclusions', () => {
6
10
  afterEach(() => {
7
11
  // make sure FS is restored after running tests
@@ -39,11 +43,15 @@ describe('Exclusions', () => {
39
43
  // Mock inclusions sources
40
44
  const scope = nock('https://example.org')
41
45
  .get('/inclusions.txt')
42
- .reply(200, 'rule1')
46
+ .reply(200, 'rule1', {
47
+ 'Content-Type': 'text/plain',
48
+ })
43
49
  .get('/inclusions2.txt')
44
- .reply(200, 'rule2');
50
+ .reply(200, 'rule2', {
51
+ 'Content-Type': 'text/plain',
52
+ });
45
53
  mock({
46
- 'test/dir': {
54
+ [testDirPath]: {
47
55
  'inclusions.txt': 'rule3',
48
56
  },
49
57
  });
@@ -56,7 +64,7 @@ describe('Exclusions', () => {
56
64
  const inclusionsSources = [
57
65
  'https://example.org/inclusions.txt',
58
66
  'https://example.org/inclusions2.txt',
59
- 'test/dir/inclusions.txt',
67
+ inclusionsFilePath,
60
68
  ];
61
69
 
62
70
  // Include!