@adguard/hostlist-compiler 1.0.38 → 1.0.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/package.json +2 -2
- package/src/compile-source.js +1 -1
- package/src/filter.js +2 -2
- package/test/compile-source.test.js +44 -0
- package/test/resolve-include.test.js +98 -0
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [1.0.39] - 2025-03-13
|
|
9
|
+
|
|
10
|
+
### Changed
|
|
11
|
+
|
|
12
|
+
- Updated [@adguard/filters-downloader] to 2.3.1. This version handles empty sources correctly. [#85]
|
|
13
|
+
|
|
14
|
+
[1.0.39]: https://github.com/AdguardTeam/HostlistCompiler/compare/v1.0.38...v1.0.39
|
|
15
|
+
[#85]: https://github.com/AdguardTeam/HostlistCompiler/issues/85
|
|
16
|
+
|
|
8
17
|
## [1.0.38] - 2025-03-06
|
|
9
18
|
|
|
10
19
|
### Changed
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adguard/hostlist-compiler",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.39",
|
|
4
4
|
"description": "A simple tool that compiles hosts blocklists from multiple sources",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"types": "src/index.d.ts",
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
"nock": "^12.0.2"
|
|
29
29
|
},
|
|
30
30
|
"dependencies": {
|
|
31
|
-
"@adguard/filters-downloader": "^2.3.
|
|
31
|
+
"@adguard/filters-downloader": "^2.3.1",
|
|
32
32
|
"ajv": "^6.12.0",
|
|
33
33
|
"ajv-errors": "^1.0.1",
|
|
34
34
|
"axios": "^0.19.2",
|
package/src/compile-source.js
CHANGED
|
@@ -21,7 +21,7 @@ const { transform } = require('./transformations/transform');
|
|
|
21
21
|
*/
|
|
22
22
|
async function compileSource(source) {
|
|
23
23
|
consola.info(`Start compiling ${source.source}`);
|
|
24
|
-
let rules = await FiltersDownloader.download(source.source);
|
|
24
|
+
let rules = await FiltersDownloader.download(source.source, {}, { allowEmptyResponse: true });
|
|
25
25
|
consola.info(`Original length is ${rules.length}`);
|
|
26
26
|
|
|
27
27
|
rules = await transform(rules, source, source.transformations);
|
package/src/filter.js
CHANGED
|
@@ -16,8 +16,8 @@ async function downloadAll(sources) {
|
|
|
16
16
|
return list;
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
await Promise.all(sources.map(async (
|
|
20
|
-
const rulesStr = await FiltersDownloader.download(
|
|
19
|
+
await Promise.all(sources.map(async (source) => {
|
|
20
|
+
const rulesStr = await FiltersDownloader.download(source, {}, { allowEmptyResponse: true });
|
|
21
21
|
const rules = rulesStr
|
|
22
22
|
.filter((el) => el.trim().length > 0 && !ruleUtils.isComment(el));
|
|
23
23
|
list = list.concat(rules);
|
|
@@ -170,4 +170,48 @@ describe('Source compiler', () => {
|
|
|
170
170
|
]);
|
|
171
171
|
scope.done();
|
|
172
172
|
});
|
|
173
|
+
|
|
174
|
+
it('compile the empty local source', async () => {
|
|
175
|
+
mock({
|
|
176
|
+
'test/dir': {
|
|
177
|
+
'empty.txt': '',
|
|
178
|
+
},
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
const source = {
|
|
182
|
+
name: 'test source',
|
|
183
|
+
source: 'test/dir/empty.txt',
|
|
184
|
+
transformations: [
|
|
185
|
+
TRANSFORMATIONS.ConvertToAscii,
|
|
186
|
+
TRANSFORMATIONS.Deduplicate,
|
|
187
|
+
TRANSFORMATIONS.RemoveComments,
|
|
188
|
+
TRANSFORMATIONS.TrimLines,
|
|
189
|
+
],
|
|
190
|
+
};
|
|
191
|
+
const compiled = await compileSource(source);
|
|
192
|
+
expect(compiled).toEqual(['']);
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
it('compile the empty external source', async () => {
|
|
196
|
+
const scope = nock('https://example.org')
|
|
197
|
+
.get('/empty.txt')
|
|
198
|
+
.reply(200, '', {
|
|
199
|
+
'Content-Type': 'text/plain',
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
const source = {
|
|
203
|
+
name: 'test source',
|
|
204
|
+
source: 'https://example.org/empty.txt',
|
|
205
|
+
transformations: [
|
|
206
|
+
TRANSFORMATIONS.ConvertToAscii,
|
|
207
|
+
TRANSFORMATIONS.Deduplicate,
|
|
208
|
+
TRANSFORMATIONS.RemoveComments,
|
|
209
|
+
TRANSFORMATIONS.TrimLines,
|
|
210
|
+
TRANSFORMATIONS.Compress,
|
|
211
|
+
],
|
|
212
|
+
};
|
|
213
|
+
const compiled = await compileSource(source);
|
|
214
|
+
expect(compiled).toEqual(['']);
|
|
215
|
+
scope.done();
|
|
216
|
+
});
|
|
173
217
|
});
|
|
@@ -293,4 +293,102 @@ exclude.com
|
|
|
293
293
|
|
|
294
294
|
scope.done();
|
|
295
295
|
});
|
|
296
|
+
|
|
297
|
+
it('should compile source with empty !#include and empty source', async () => {
|
|
298
|
+
// Prepare filters content
|
|
299
|
+
const filterContent = `! this is a source
|
|
300
|
+
||*.ком^
|
|
301
|
+
||trackers.exclude.com^
|
|
302
|
+
||exclude.com^
|
|
303
|
+
!#include https://example.org/source.txt`;
|
|
304
|
+
|
|
305
|
+
// Prepare source
|
|
306
|
+
const scope = nock('https://example.org')
|
|
307
|
+
.get('/source1.txt')
|
|
308
|
+
.reply(200, filterContent, {
|
|
309
|
+
'Content-Type': 'text/plain',
|
|
310
|
+
})
|
|
311
|
+
.get('/source.txt')
|
|
312
|
+
.reply(200, '', {
|
|
313
|
+
'Content-Type': 'text/plain',
|
|
314
|
+
})
|
|
315
|
+
.get('/empty.txt')
|
|
316
|
+
.reply(200, '', {
|
|
317
|
+
'Content-Type': 'text/plain',
|
|
318
|
+
});
|
|
319
|
+
|
|
320
|
+
// compiler configuration
|
|
321
|
+
const configuration = {
|
|
322
|
+
name: 'Test filter',
|
|
323
|
+
description: 'Our test filter',
|
|
324
|
+
version: '1.0.0.9',
|
|
325
|
+
sources: [
|
|
326
|
+
{
|
|
327
|
+
name: 'source 1',
|
|
328
|
+
source: 'https://example.org/source1.txt',
|
|
329
|
+
},
|
|
330
|
+
],
|
|
331
|
+
transformations: [
|
|
332
|
+
'RemoveComments',
|
|
333
|
+
'ConvertToAscii',
|
|
334
|
+
],
|
|
335
|
+
inclusions_sources: ['https://example.org/empty.txt'],
|
|
336
|
+
};
|
|
337
|
+
|
|
338
|
+
const expectedRules = [
|
|
339
|
+
'||*.xn--j1aef^',
|
|
340
|
+
'||trackers.exclude.com^',
|
|
341
|
+
'||exclude.com^',
|
|
342
|
+
];
|
|
343
|
+
|
|
344
|
+
// compile the final list
|
|
345
|
+
const list = await compile(configuration);
|
|
346
|
+
|
|
347
|
+
const str = list.join('\n');
|
|
348
|
+
consola.info(str);
|
|
349
|
+
|
|
350
|
+
expectedRules.forEach((rule) => {
|
|
351
|
+
expect(list).toContain(rule);
|
|
352
|
+
});
|
|
353
|
+
|
|
354
|
+
scope.done();
|
|
355
|
+
});
|
|
356
|
+
|
|
357
|
+
it('should compile source with empty source', async () => {
|
|
358
|
+
// Prepare source
|
|
359
|
+
const scope = nock('https://example.org')
|
|
360
|
+
.get('/source.txt')
|
|
361
|
+
.reply(200, '', {
|
|
362
|
+
'Content-Type': 'text/plain',
|
|
363
|
+
})
|
|
364
|
+
.get('/empty.txt')
|
|
365
|
+
.reply(200, '||example.com^', {
|
|
366
|
+
'Content-Type': 'text/plain',
|
|
367
|
+
});
|
|
368
|
+
|
|
369
|
+
// compiler configuration
|
|
370
|
+
const configuration = {
|
|
371
|
+
name: 'Test filter',
|
|
372
|
+
description: 'Our test filter',
|
|
373
|
+
version: '1.0.0.9',
|
|
374
|
+
sources: [
|
|
375
|
+
{
|
|
376
|
+
name: 'source 1',
|
|
377
|
+
source: 'https://example.org/empty.txt',
|
|
378
|
+
},
|
|
379
|
+
],
|
|
380
|
+
transformations: [
|
|
381
|
+
'RemoveComments',
|
|
382
|
+
'ConvertToAscii',
|
|
383
|
+
],
|
|
384
|
+
inclusions_sources: ['https://example.org/source.txt'],
|
|
385
|
+
};
|
|
386
|
+
|
|
387
|
+
const list = await compile(configuration);
|
|
388
|
+
consola.info(list.join('\n'));
|
|
389
|
+
|
|
390
|
+
expect(list).toContain('||example.com^');
|
|
391
|
+
|
|
392
|
+
scope.done();
|
|
393
|
+
});
|
|
296
394
|
});
|