@adguard/hostlist-compiler 1.0.37 → 1.0.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -1
- package/package.json +2 -2
- package/src/compile-source.js +1 -1
- package/src/filter.js +2 -2
- package/test/compile-source.test.js +47 -3
- package/test/resolve-include.test.js +199 -6
- package/test/resources/includedfile.txt +1 -1
- package/test/resources/rules.txt +1 -1
- package/test/resources/subinclude.txt +1 -0
package/CHANGELOG.md
CHANGED
|
@@ -5,11 +5,33 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [1.0.39] - 2025-03-13
|
|
9
|
+
|
|
10
|
+
### Changed
|
|
11
|
+
|
|
12
|
+
- Updated [@adguard/filters-downloader] to 2.3.1. This version handles empty sources correctly. [#85]
|
|
13
|
+
|
|
14
|
+
[1.0.39]: https://github.com/AdguardTeam/HostlistCompiler/compare/v1.0.38...v1.0.39
|
|
15
|
+
[#85]: https://github.com/AdguardTeam/HostlistCompiler/issues/85
|
|
16
|
+
|
|
17
|
+
## [1.0.38] - 2025-03-06
|
|
18
|
+
|
|
19
|
+
### Changed
|
|
20
|
+
|
|
21
|
+
- Updated [@adguard/filters-downloader] to 2.3.0.
|
|
22
|
+
|
|
23
|
+
### Fixed
|
|
24
|
+
|
|
25
|
+
- Path determination issue. [#82]
|
|
26
|
+
|
|
27
|
+
[1.0.38]: https://github.com/AdguardTeam/HostlistCompiler/compare/v1.0.37...v1.0.38
|
|
28
|
+
[#82]: https://github.com/AdguardTeam/HostlistCompiler/issues/82
|
|
29
|
+
|
|
8
30
|
## [1.0.37] - 2025-02-28
|
|
9
31
|
|
|
10
32
|
### Added
|
|
11
33
|
|
|
12
|
-
- Resolving the `!#include
|
|
34
|
+
- Resolving the `!#include` directive by using `@adguard/filters-downloader`. [#78]
|
|
13
35
|
|
|
14
36
|
[1.0.37]: https://github.com/AdguardTeam/HostlistCompiler/compare/v1.0.35...v1.0.37
|
|
15
37
|
[#78]: https://github.com/AdguardTeam/HostlistCompiler/issues/78
|
|
@@ -88,3 +110,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
88
110
|
|
|
89
111
|
[1.0.27]: https://github.com/AdguardTeam/HostlistCompiler/compare/v1.0.26...v1.0.27
|
|
90
112
|
[#69]: https://github.com/AdguardTeam/FiltersCompiler/issues/69
|
|
113
|
+
|
|
114
|
+
[@adguard/filters-downloader]: https://github.com/AdguardTeam/FiltersDownloader/blob/master/CHANGELOG.md
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adguard/hostlist-compiler",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.39",
|
|
4
4
|
"description": "A simple tool that compiles hosts blocklists from multiple sources",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"types": "src/index.d.ts",
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
"nock": "^12.0.2"
|
|
29
29
|
},
|
|
30
30
|
"dependencies": {
|
|
31
|
-
"@adguard/filters-downloader": "^2.
|
|
31
|
+
"@adguard/filters-downloader": "^2.3.1",
|
|
32
32
|
"ajv": "^6.12.0",
|
|
33
33
|
"ajv-errors": "^1.0.1",
|
|
34
34
|
"axios": "^0.19.2",
|
package/src/compile-source.js
CHANGED
|
@@ -21,7 +21,7 @@ const { transform } = require('./transformations/transform');
|
|
|
21
21
|
*/
|
|
22
22
|
async function compileSource(source) {
|
|
23
23
|
consola.info(`Start compiling ${source.source}`);
|
|
24
|
-
let rules = await FiltersDownloader.download(source.source);
|
|
24
|
+
let rules = await FiltersDownloader.download(source.source, {}, { allowEmptyResponse: true });
|
|
25
25
|
consola.info(`Original length is ${rules.length}`);
|
|
26
26
|
|
|
27
27
|
rules = await transform(rules, source, source.transformations);
|
package/src/filter.js
CHANGED
|
@@ -16,8 +16,8 @@ async function downloadAll(sources) {
|
|
|
16
16
|
return list;
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
await Promise.all(sources.map(async (
|
|
20
|
-
const rulesStr = await FiltersDownloader.download(
|
|
19
|
+
await Promise.all(sources.map(async (source) => {
|
|
20
|
+
const rulesStr = await FiltersDownloader.download(source, {}, { allowEmptyResponse: true });
|
|
21
21
|
const rules = rulesStr
|
|
22
22
|
.filter((el) => el.trim().length > 0 && !ruleUtils.isComment(el));
|
|
23
23
|
list = list.concat(rules);
|
|
@@ -5,7 +5,6 @@ const compileSource = require('../src/compile-source');
|
|
|
5
5
|
const { TRANSFORMATIONS } = require('../src/transformations/transform');
|
|
6
6
|
|
|
7
7
|
const testDirPath = path.resolve(__dirname, 'test/dir');
|
|
8
|
-
const rulesFilePath = path.resolve(testDirPath, 'rules.txt');
|
|
9
8
|
const exclusionsFilePath = path.resolve(testDirPath, 'exclusions.txt');
|
|
10
9
|
|
|
11
10
|
describe('Source compiler', () => {
|
|
@@ -36,13 +35,14 @@ describe('Source compiler', () => {
|
|
|
36
35
|
|
|
37
36
|
it('compile a simple file source', async () => {
|
|
38
37
|
mock({
|
|
39
|
-
|
|
38
|
+
'test/dir': {
|
|
40
39
|
'rules.txt': 'testrule',
|
|
41
40
|
},
|
|
42
41
|
});
|
|
42
|
+
|
|
43
43
|
const source = {
|
|
44
44
|
name: 'test source',
|
|
45
|
-
source:
|
|
45
|
+
source: 'test/dir/rules.txt',
|
|
46
46
|
};
|
|
47
47
|
|
|
48
48
|
const rules = await compileSource(source);
|
|
@@ -170,4 +170,48 @@ describe('Source compiler', () => {
|
|
|
170
170
|
]);
|
|
171
171
|
scope.done();
|
|
172
172
|
});
|
|
173
|
+
|
|
174
|
+
it('compile the empty local source', async () => {
|
|
175
|
+
mock({
|
|
176
|
+
'test/dir': {
|
|
177
|
+
'empty.txt': '',
|
|
178
|
+
},
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
const source = {
|
|
182
|
+
name: 'test source',
|
|
183
|
+
source: 'test/dir/empty.txt',
|
|
184
|
+
transformations: [
|
|
185
|
+
TRANSFORMATIONS.ConvertToAscii,
|
|
186
|
+
TRANSFORMATIONS.Deduplicate,
|
|
187
|
+
TRANSFORMATIONS.RemoveComments,
|
|
188
|
+
TRANSFORMATIONS.TrimLines,
|
|
189
|
+
],
|
|
190
|
+
};
|
|
191
|
+
const compiled = await compileSource(source);
|
|
192
|
+
expect(compiled).toEqual(['']);
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
it('compile the empty external source', async () => {
|
|
196
|
+
const scope = nock('https://example.org')
|
|
197
|
+
.get('/empty.txt')
|
|
198
|
+
.reply(200, '', {
|
|
199
|
+
'Content-Type': 'text/plain',
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
const source = {
|
|
203
|
+
name: 'test source',
|
|
204
|
+
source: 'https://example.org/empty.txt',
|
|
205
|
+
transformations: [
|
|
206
|
+
TRANSFORMATIONS.ConvertToAscii,
|
|
207
|
+
TRANSFORMATIONS.Deduplicate,
|
|
208
|
+
TRANSFORMATIONS.RemoveComments,
|
|
209
|
+
TRANSFORMATIONS.TrimLines,
|
|
210
|
+
TRANSFORMATIONS.Compress,
|
|
211
|
+
],
|
|
212
|
+
};
|
|
213
|
+
const compiled = await compileSource(source);
|
|
214
|
+
expect(compiled).toEqual(['']);
|
|
215
|
+
scope.done();
|
|
216
|
+
});
|
|
173
217
|
});
|
|
@@ -1,11 +1,7 @@
|
|
|
1
1
|
const nock = require('nock');
|
|
2
2
|
const consola = require('consola');
|
|
3
|
-
const path = require('path');
|
|
4
3
|
const compile = require('../src/index');
|
|
5
4
|
|
|
6
|
-
const testDirPath = path.resolve(__dirname, 'resources');
|
|
7
|
-
const filterFilePath = path.resolve(testDirPath, 'rules.txt');
|
|
8
|
-
|
|
9
5
|
describe('Hostlist compiler', () => {
|
|
10
6
|
it('compile from one source with nested includes', async () => {
|
|
11
7
|
// Prepare filters content
|
|
@@ -89,7 +85,7 @@ non/valid_rule`;
|
|
|
89
85
|
ads.example.com
|
|
90
86
|
trackers.exclude.com
|
|
91
87
|
exclude.com
|
|
92
|
-
!#include
|
|
88
|
+
!#include source2.txt`;
|
|
93
89
|
|
|
94
90
|
const filterContent2 = `
|
|
95
91
|
popups.com
|
|
@@ -169,7 +165,7 @@ non/valid_rule`;
|
|
|
169
165
|
sources: [
|
|
170
166
|
{
|
|
171
167
|
name: 'filter',
|
|
172
|
-
source:
|
|
168
|
+
source: 'test/resources/rules.txt',
|
|
173
169
|
},
|
|
174
170
|
],
|
|
175
171
|
transformations: [
|
|
@@ -198,4 +194,201 @@ non/valid_rule`;
|
|
|
198
194
|
expect(list).toContain(rule);
|
|
199
195
|
});
|
|
200
196
|
});
|
|
197
|
+
|
|
198
|
+
it('compile from external source with nested includes', async () => {
|
|
199
|
+
// Prepare filters content
|
|
200
|
+
const filterContent1 = `! this is a source
|
|
201
|
+
||sub_test_main
|
|
202
|
+
!#include source2.txt`;
|
|
203
|
+
|
|
204
|
+
const filterContent2 = `
|
|
205
|
+
||sub_test
|
|
206
|
+
!#include subdir/source3.txt`;
|
|
207
|
+
|
|
208
|
+
const filterContent3 = `
|
|
209
|
+
||sub_test_final`;
|
|
210
|
+
|
|
211
|
+
// Prepare source
|
|
212
|
+
const scope = nock('https://example.org')
|
|
213
|
+
.get('/testdir/source1.txt')
|
|
214
|
+
.reply(200, filterContent1, {
|
|
215
|
+
'Content-Type': 'text/plain',
|
|
216
|
+
})
|
|
217
|
+
.get('/testdir/source2.txt')
|
|
218
|
+
.reply(200, filterContent2, {
|
|
219
|
+
'Content-Type': 'text/plain',
|
|
220
|
+
})
|
|
221
|
+
.get('/testdir/subdir/source3.txt')
|
|
222
|
+
.reply(200, filterContent3, {
|
|
223
|
+
'Content-Type': 'text/plain',
|
|
224
|
+
});
|
|
225
|
+
|
|
226
|
+
// compiler configuration
|
|
227
|
+
const configuration = {
|
|
228
|
+
name: 'Test filter',
|
|
229
|
+
description: 'Our test filter',
|
|
230
|
+
version: '1.0.0.9',
|
|
231
|
+
sources: [
|
|
232
|
+
{
|
|
233
|
+
name: 'filter',
|
|
234
|
+
source: 'https://example.org/testdir/source1.txt',
|
|
235
|
+
},
|
|
236
|
+
],
|
|
237
|
+
transformations: [
|
|
238
|
+
'RemoveComments',
|
|
239
|
+
],
|
|
240
|
+
exclusions: ['test_parent'],
|
|
241
|
+
};
|
|
242
|
+
|
|
243
|
+
// compile the final list
|
|
244
|
+
const list = await compile(configuration);
|
|
245
|
+
|
|
246
|
+
const str = list.join('\n');
|
|
247
|
+
consola.info(str);
|
|
248
|
+
|
|
249
|
+
const expectedRules = [
|
|
250
|
+
'||sub_test_main',
|
|
251
|
+
'||sub_test',
|
|
252
|
+
'||sub_test_final',
|
|
253
|
+
];
|
|
254
|
+
|
|
255
|
+
expectedRules.forEach((rule) => {
|
|
256
|
+
expect(list).toContain(rule);
|
|
257
|
+
});
|
|
258
|
+
|
|
259
|
+
scope.done();
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
it('should not compile source from different origin', async () => {
|
|
263
|
+
// Prepare filters content
|
|
264
|
+
const filterContent1 = `! this is a source
|
|
265
|
+
ads.example.com
|
|
266
|
+
trackers.exclude.com
|
|
267
|
+
exclude.com
|
|
268
|
+
!#include https://example1.org/source.txt`;
|
|
269
|
+
|
|
270
|
+
// Prepare source
|
|
271
|
+
const scope = nock('https://example.org')
|
|
272
|
+
.get('/source1.txt')
|
|
273
|
+
.reply(200, filterContent1, {
|
|
274
|
+
'Content-Type': 'text/plain',
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
// compiler configuration
|
|
278
|
+
const configuration = {
|
|
279
|
+
name: 'Test filter',
|
|
280
|
+
description: 'Our test filter',
|
|
281
|
+
version: '1.0.0.9',
|
|
282
|
+
sources: [
|
|
283
|
+
{
|
|
284
|
+
name: 'source 1',
|
|
285
|
+
source: 'https://example.org/source1.txt',
|
|
286
|
+
},
|
|
287
|
+
],
|
|
288
|
+
};
|
|
289
|
+
|
|
290
|
+
await expect(async () => {
|
|
291
|
+
await compile(configuration);
|
|
292
|
+
}).rejects.toThrow(Error);
|
|
293
|
+
|
|
294
|
+
scope.done();
|
|
295
|
+
});
|
|
296
|
+
|
|
297
|
+
it('should compile source with empty !#include and empty source', async () => {
|
|
298
|
+
// Prepare filters content
|
|
299
|
+
const filterContent = `! this is a source
|
|
300
|
+
||*.ком^
|
|
301
|
+
||trackers.exclude.com^
|
|
302
|
+
||exclude.com^
|
|
303
|
+
!#include https://example.org/source.txt`;
|
|
304
|
+
|
|
305
|
+
// Prepare source
|
|
306
|
+
const scope = nock('https://example.org')
|
|
307
|
+
.get('/source1.txt')
|
|
308
|
+
.reply(200, filterContent, {
|
|
309
|
+
'Content-Type': 'text/plain',
|
|
310
|
+
})
|
|
311
|
+
.get('/source.txt')
|
|
312
|
+
.reply(200, '', {
|
|
313
|
+
'Content-Type': 'text/plain',
|
|
314
|
+
})
|
|
315
|
+
.get('/empty.txt')
|
|
316
|
+
.reply(200, '', {
|
|
317
|
+
'Content-Type': 'text/plain',
|
|
318
|
+
});
|
|
319
|
+
|
|
320
|
+
// compiler configuration
|
|
321
|
+
const configuration = {
|
|
322
|
+
name: 'Test filter',
|
|
323
|
+
description: 'Our test filter',
|
|
324
|
+
version: '1.0.0.9',
|
|
325
|
+
sources: [
|
|
326
|
+
{
|
|
327
|
+
name: 'source 1',
|
|
328
|
+
source: 'https://example.org/source1.txt',
|
|
329
|
+
},
|
|
330
|
+
],
|
|
331
|
+
transformations: [
|
|
332
|
+
'RemoveComments',
|
|
333
|
+
'ConvertToAscii',
|
|
334
|
+
],
|
|
335
|
+
inclusions_sources: ['https://example.org/empty.txt'],
|
|
336
|
+
};
|
|
337
|
+
|
|
338
|
+
const expectedRules = [
|
|
339
|
+
'||*.xn--j1aef^',
|
|
340
|
+
'||trackers.exclude.com^',
|
|
341
|
+
'||exclude.com^',
|
|
342
|
+
];
|
|
343
|
+
|
|
344
|
+
// compile the final list
|
|
345
|
+
const list = await compile(configuration);
|
|
346
|
+
|
|
347
|
+
const str = list.join('\n');
|
|
348
|
+
consola.info(str);
|
|
349
|
+
|
|
350
|
+
expectedRules.forEach((rule) => {
|
|
351
|
+
expect(list).toContain(rule);
|
|
352
|
+
});
|
|
353
|
+
|
|
354
|
+
scope.done();
|
|
355
|
+
});
|
|
356
|
+
|
|
357
|
+
it('should compile source with empty source', async () => {
|
|
358
|
+
// Prepare source
|
|
359
|
+
const scope = nock('https://example.org')
|
|
360
|
+
.get('/source.txt')
|
|
361
|
+
.reply(200, '', {
|
|
362
|
+
'Content-Type': 'text/plain',
|
|
363
|
+
})
|
|
364
|
+
.get('/empty.txt')
|
|
365
|
+
.reply(200, '||example.com^', {
|
|
366
|
+
'Content-Type': 'text/plain',
|
|
367
|
+
});
|
|
368
|
+
|
|
369
|
+
// compiler configuration
|
|
370
|
+
const configuration = {
|
|
371
|
+
name: 'Test filter',
|
|
372
|
+
description: 'Our test filter',
|
|
373
|
+
version: '1.0.0.9',
|
|
374
|
+
sources: [
|
|
375
|
+
{
|
|
376
|
+
name: 'source 1',
|
|
377
|
+
source: 'https://example.org/empty.txt',
|
|
378
|
+
},
|
|
379
|
+
],
|
|
380
|
+
transformations: [
|
|
381
|
+
'RemoveComments',
|
|
382
|
+
'ConvertToAscii',
|
|
383
|
+
],
|
|
384
|
+
inclusions_sources: ['https://example.org/source.txt'],
|
|
385
|
+
};
|
|
386
|
+
|
|
387
|
+
const list = await compile(configuration);
|
|
388
|
+
consola.info(list.join('\n'));
|
|
389
|
+
|
|
390
|
+
expect(list).toContain('||example.com^');
|
|
391
|
+
|
|
392
|
+
scope.done();
|
|
393
|
+
});
|
|
201
394
|
});
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
should.be.included.com
|
|
2
|
-
|
|
2
|
+
!#include subinclude.txt
|
package/test/resources/rules.txt
CHANGED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
last.rule
|