@adguard/hostlist-compiler 1.0.37 → 1.0.39

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -5,11 +5,33 @@ All notable changes to this project will be documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
7
 
8
+ ## [1.0.39] - 2025-03-13
9
+
10
+ ### Changed
11
+
12
+ - Updated [@adguard/filters-downloader] to 2.3.1. This version handles empty sources correctly. [#85]
13
+
14
+ [1.0.39]: https://github.com/AdguardTeam/HostlistCompiler/compare/v1.0.38...v1.0.39
15
+ [#85]: https://github.com/AdguardTeam/HostlistCompiler/issues/85
16
+
17
+ ## [1.0.38] - 2025-03-06
18
+
19
+ ### Changed
20
+
21
+ - Updated [@adguard/filters-downloader] to 2.3.0.
22
+
23
+ ### Fixed
24
+
25
+ - Path determination issue. [#82]
26
+
27
+ [1.0.38]: https://github.com/AdguardTeam/HostlistCompiler/compare/v1.0.37...v1.0.38
28
+ [#82]: https://github.com/AdguardTeam/HostlistCompiler/issues/82
29
+
8
30
  ## [1.0.37] - 2025-02-28
9
31
 
10
32
  ### Added
11
33
 
12
- - Resolving the `!#include `directive by using `@adguard/filters-downloader`. [#78]
34
+ - Resolving the `!#include` directive by using `@adguard/filters-downloader`. [#78]
13
35
 
14
36
  [1.0.37]: https://github.com/AdguardTeam/HostlistCompiler/compare/v1.0.35...v1.0.37
15
37
  [#78]: https://github.com/AdguardTeam/HostlistCompiler/issues/78
@@ -88,3 +110,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
88
110
 
89
111
  [1.0.27]: https://github.com/AdguardTeam/HostlistCompiler/compare/v1.0.26...v1.0.27
90
112
  [#69]: https://github.com/AdguardTeam/FiltersCompiler/issues/69
113
+
114
+ [@adguard/filters-downloader]: https://github.com/AdguardTeam/FiltersDownloader/blob/master/CHANGELOG.md
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@adguard/hostlist-compiler",
3
- "version": "1.0.37",
3
+ "version": "1.0.39",
4
4
  "description": "A simple tool that compiles hosts blocklists from multiple sources",
5
5
  "main": "src/index.js",
6
6
  "types": "src/index.d.ts",
@@ -28,7 +28,7 @@
28
28
  "nock": "^12.0.2"
29
29
  },
30
30
  "dependencies": {
31
- "@adguard/filters-downloader": "^2.2.6",
31
+ "@adguard/filters-downloader": "^2.3.1",
32
32
  "ajv": "^6.12.0",
33
33
  "ajv-errors": "^1.0.1",
34
34
  "axios": "^0.19.2",
@@ -21,7 +21,7 @@ const { transform } = require('./transformations/transform');
21
21
  */
22
22
  async function compileSource(source) {
23
23
  consola.info(`Start compiling ${source.source}`);
24
- let rules = await FiltersDownloader.download(source.source);
24
+ let rules = await FiltersDownloader.download(source.source, {}, { allowEmptyResponse: true });
25
25
  consola.info(`Original length is ${rules.length}`);
26
26
 
27
27
  rules = await transform(rules, source, source.transformations);
package/src/filter.js CHANGED
@@ -16,8 +16,8 @@ async function downloadAll(sources) {
16
16
  return list;
17
17
  }
18
18
 
19
- await Promise.all(sources.map(async (s) => {
20
- const rulesStr = await FiltersDownloader.download(s);
19
+ await Promise.all(sources.map(async (source) => {
20
+ const rulesStr = await FiltersDownloader.download(source, {}, { allowEmptyResponse: true });
21
21
  const rules = rulesStr
22
22
  .filter((el) => el.trim().length > 0 && !ruleUtils.isComment(el));
23
23
  list = list.concat(rules);
@@ -5,7 +5,6 @@ const compileSource = require('../src/compile-source');
5
5
  const { TRANSFORMATIONS } = require('../src/transformations/transform');
6
6
 
7
7
  const testDirPath = path.resolve(__dirname, 'test/dir');
8
- const rulesFilePath = path.resolve(testDirPath, 'rules.txt');
9
8
  const exclusionsFilePath = path.resolve(testDirPath, 'exclusions.txt');
10
9
 
11
10
  describe('Source compiler', () => {
@@ -36,13 +35,14 @@ describe('Source compiler', () => {
36
35
 
37
36
  it('compile a simple file source', async () => {
38
37
  mock({
39
- [testDirPath]: {
38
+ 'test/dir': {
40
39
  'rules.txt': 'testrule',
41
40
  },
42
41
  });
42
+
43
43
  const source = {
44
44
  name: 'test source',
45
- source: rulesFilePath,
45
+ source: 'test/dir/rules.txt',
46
46
  };
47
47
 
48
48
  const rules = await compileSource(source);
@@ -170,4 +170,48 @@ describe('Source compiler', () => {
170
170
  ]);
171
171
  scope.done();
172
172
  });
173
+
174
+ it('compile the empty local source', async () => {
175
+ mock({
176
+ 'test/dir': {
177
+ 'empty.txt': '',
178
+ },
179
+ });
180
+
181
+ const source = {
182
+ name: 'test source',
183
+ source: 'test/dir/empty.txt',
184
+ transformations: [
185
+ TRANSFORMATIONS.ConvertToAscii,
186
+ TRANSFORMATIONS.Deduplicate,
187
+ TRANSFORMATIONS.RemoveComments,
188
+ TRANSFORMATIONS.TrimLines,
189
+ ],
190
+ };
191
+ const compiled = await compileSource(source);
192
+ expect(compiled).toEqual(['']);
193
+ });
194
+
195
+ it('compile the empty external source', async () => {
196
+ const scope = nock('https://example.org')
197
+ .get('/empty.txt')
198
+ .reply(200, '', {
199
+ 'Content-Type': 'text/plain',
200
+ });
201
+
202
+ const source = {
203
+ name: 'test source',
204
+ source: 'https://example.org/empty.txt',
205
+ transformations: [
206
+ TRANSFORMATIONS.ConvertToAscii,
207
+ TRANSFORMATIONS.Deduplicate,
208
+ TRANSFORMATIONS.RemoveComments,
209
+ TRANSFORMATIONS.TrimLines,
210
+ TRANSFORMATIONS.Compress,
211
+ ],
212
+ };
213
+ const compiled = await compileSource(source);
214
+ expect(compiled).toEqual(['']);
215
+ scope.done();
216
+ });
173
217
  });
@@ -1,11 +1,7 @@
1
1
  const nock = require('nock');
2
2
  const consola = require('consola');
3
- const path = require('path');
4
3
  const compile = require('../src/index');
5
4
 
6
- const testDirPath = path.resolve(__dirname, 'resources');
7
- const filterFilePath = path.resolve(testDirPath, 'rules.txt');
8
-
9
5
  describe('Hostlist compiler', () => {
10
6
  it('compile from one source with nested includes', async () => {
11
7
  // Prepare filters content
@@ -89,7 +85,7 @@ non/valid_rule`;
89
85
  ads.example.com
90
86
  trackers.exclude.com
91
87
  exclude.com
92
- !#include https://example.org/source2.txt`;
88
+ !#include source2.txt`;
93
89
 
94
90
  const filterContent2 = `
95
91
  popups.com
@@ -169,7 +165,7 @@ non/valid_rule`;
169
165
  sources: [
170
166
  {
171
167
  name: 'filter',
172
- source: filterFilePath,
168
+ source: 'test/resources/rules.txt',
173
169
  },
174
170
  ],
175
171
  transformations: [
@@ -198,4 +194,201 @@ non/valid_rule`;
198
194
  expect(list).toContain(rule);
199
195
  });
200
196
  });
197
+
198
+ it('compile from external source with nested includes', async () => {
199
+ // Prepare filters content
200
+ const filterContent1 = `! this is a source
201
+ ||sub_test_main
202
+ !#include source2.txt`;
203
+
204
+ const filterContent2 = `
205
+ ||sub_test
206
+ !#include subdir/source3.txt`;
207
+
208
+ const filterContent3 = `
209
+ ||sub_test_final`;
210
+
211
+ // Prepare source
212
+ const scope = nock('https://example.org')
213
+ .get('/testdir/source1.txt')
214
+ .reply(200, filterContent1, {
215
+ 'Content-Type': 'text/plain',
216
+ })
217
+ .get('/testdir/source2.txt')
218
+ .reply(200, filterContent2, {
219
+ 'Content-Type': 'text/plain',
220
+ })
221
+ .get('/testdir/subdir/source3.txt')
222
+ .reply(200, filterContent3, {
223
+ 'Content-Type': 'text/plain',
224
+ });
225
+
226
+ // compiler configuration
227
+ const configuration = {
228
+ name: 'Test filter',
229
+ description: 'Our test filter',
230
+ version: '1.0.0.9',
231
+ sources: [
232
+ {
233
+ name: 'filter',
234
+ source: 'https://example.org/testdir/source1.txt',
235
+ },
236
+ ],
237
+ transformations: [
238
+ 'RemoveComments',
239
+ ],
240
+ exclusions: ['test_parent'],
241
+ };
242
+
243
+ // compile the final list
244
+ const list = await compile(configuration);
245
+
246
+ const str = list.join('\n');
247
+ consola.info(str);
248
+
249
+ const expectedRules = [
250
+ '||sub_test_main',
251
+ '||sub_test',
252
+ '||sub_test_final',
253
+ ];
254
+
255
+ expectedRules.forEach((rule) => {
256
+ expect(list).toContain(rule);
257
+ });
258
+
259
+ scope.done();
260
+ });
261
+
262
+ it('should not compile source from different origin', async () => {
263
+ // Prepare filters content
264
+ const filterContent1 = `! this is a source
265
+ ads.example.com
266
+ trackers.exclude.com
267
+ exclude.com
268
+ !#include https://example1.org/source.txt`;
269
+
270
+ // Prepare source
271
+ const scope = nock('https://example.org')
272
+ .get('/source1.txt')
273
+ .reply(200, filterContent1, {
274
+ 'Content-Type': 'text/plain',
275
+ });
276
+
277
+ // compiler configuration
278
+ const configuration = {
279
+ name: 'Test filter',
280
+ description: 'Our test filter',
281
+ version: '1.0.0.9',
282
+ sources: [
283
+ {
284
+ name: 'source 1',
285
+ source: 'https://example.org/source1.txt',
286
+ },
287
+ ],
288
+ };
289
+
290
+ await expect(async () => {
291
+ await compile(configuration);
292
+ }).rejects.toThrow(Error);
293
+
294
+ scope.done();
295
+ });
296
+
297
+ it('should compile source with empty !#include and empty source', async () => {
298
+ // Prepare filters content
299
+ const filterContent = `! this is a source
300
+ ||*.ком^
301
+ ||trackers.exclude.com^
302
+ ||exclude.com^
303
+ !#include https://example.org/source.txt`;
304
+
305
+ // Prepare source
306
+ const scope = nock('https://example.org')
307
+ .get('/source1.txt')
308
+ .reply(200, filterContent, {
309
+ 'Content-Type': 'text/plain',
310
+ })
311
+ .get('/source.txt')
312
+ .reply(200, '', {
313
+ 'Content-Type': 'text/plain',
314
+ })
315
+ .get('/empty.txt')
316
+ .reply(200, '', {
317
+ 'Content-Type': 'text/plain',
318
+ });
319
+
320
+ // compiler configuration
321
+ const configuration = {
322
+ name: 'Test filter',
323
+ description: 'Our test filter',
324
+ version: '1.0.0.9',
325
+ sources: [
326
+ {
327
+ name: 'source 1',
328
+ source: 'https://example.org/source1.txt',
329
+ },
330
+ ],
331
+ transformations: [
332
+ 'RemoveComments',
333
+ 'ConvertToAscii',
334
+ ],
335
+ inclusions_sources: ['https://example.org/empty.txt'],
336
+ };
337
+
338
+ const expectedRules = [
339
+ '||*.xn--j1aef^',
340
+ '||trackers.exclude.com^',
341
+ '||exclude.com^',
342
+ ];
343
+
344
+ // compile the final list
345
+ const list = await compile(configuration);
346
+
347
+ const str = list.join('\n');
348
+ consola.info(str);
349
+
350
+ expectedRules.forEach((rule) => {
351
+ expect(list).toContain(rule);
352
+ });
353
+
354
+ scope.done();
355
+ });
356
+
357
+ it('should compile source with empty source', async () => {
358
+ // Prepare source
359
+ const scope = nock('https://example.org')
360
+ .get('/source.txt')
361
+ .reply(200, '', {
362
+ 'Content-Type': 'text/plain',
363
+ })
364
+ .get('/empty.txt')
365
+ .reply(200, '||example.com^', {
366
+ 'Content-Type': 'text/plain',
367
+ });
368
+
369
+ // compiler configuration
370
+ const configuration = {
371
+ name: 'Test filter',
372
+ description: 'Our test filter',
373
+ version: '1.0.0.9',
374
+ sources: [
375
+ {
376
+ name: 'source 1',
377
+ source: 'https://example.org/empty.txt',
378
+ },
379
+ ],
380
+ transformations: [
381
+ 'RemoveComments',
382
+ 'ConvertToAscii',
383
+ ],
384
+ inclusions_sources: ['https://example.org/source.txt'],
385
+ };
386
+
387
+ const list = await compile(configuration);
388
+ consola.info(list.join('\n'));
389
+
390
+ expect(list).toContain('||example.com^');
391
+
392
+ scope.done();
393
+ });
201
394
  });
@@ -1,2 +1,2 @@
1
1
  should.be.included.com
2
- last.rule
2
+ !#include subinclude.txt
@@ -1,4 +1,4 @@
1
1
  ||sat.terithrow2.net^
2
2
  ||sat.fevilsor5.net^
3
3
  ||leegreemula.net^
4
- !#include /includedfile.txt
4
+ !#include includedfile.txt
@@ -0,0 +1 @@
1
+ last.rule