@adguard/hostlist-compiler 1.0.12 → 1.0.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +165 -18
- package/bamboo-specs/bamboo.yaml +1 -1
- package/package.json +2 -1
- package/src/cli.js +78 -7
- package/src/compile-source.js +1 -1
- package/src/filter.js +2 -2
- package/src/index.d.ts +69 -0
- package/src/index.js +1 -1
- package/src/schemas/configuration.schema.json +8 -2
- package/src/transformations/exclude.js +1 -1
- package/src/transformations/include.js +1 -1
- package/src/transformations/insert-final-newline.js +17 -0
- package/src/transformations/remove-empty-lines.js +15 -0
- package/src/transformations/remove-modifiers.js +2 -1
- package/src/transformations/transform.js +16 -1
- package/src/transformations/trim-lines.js +16 -0
- package/src/utils.js +1 -1
- package/test/compile-source.test.js +36 -0
- package/test/transformations/insert-final-newline.test.js +48 -0
- package/test/transformations/remove-empty-lines.test.js +77 -0
- package/test/transformations/remove-modifiers.test.js +4 -2
- package/test/transformations/trim-lines.test.js +66 -0
package/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
[](https://www.npmjs.com/package/@adguard/hostlist-compiler/)
|
|
4
4
|
|
|
5
|
-
This is a simple tool that makes it easier to compile a [hosts blocklist](https://
|
|
5
|
+
This is a simple tool that makes it easier to compile a [hosts blocklist](https://adguard-dns.io/kb/general/dns-filtering-syntax/) compatible with AdGuard Home or any other AdGuard product with **DNS filtering**.
|
|
6
6
|
|
|
7
7
|
- [Usage](#usage)
|
|
8
8
|
- [Configuration](#configuration)
|
|
@@ -15,6 +15,9 @@ This is a simple tool that makes it easier to compile a [hosts blocklist](https:
|
|
|
15
15
|
- [Validate](#validate)
|
|
16
16
|
- [Deduplicate](#deduplicate)
|
|
17
17
|
- [InvertAllow](#invertallow)
|
|
18
|
+
- [RemoveEmptyLines](#removeemptylines)
|
|
19
|
+
- [TrimLines](#trimlines)
|
|
20
|
+
- [InsertFinalNewLine](#insertfinalnewline)
|
|
18
21
|
- [How to build](#how-to-build)
|
|
19
22
|
|
|
20
23
|
## <a id="usage"></a> Usage
|
|
@@ -25,6 +28,18 @@ First of all, install the `hostlist-compiler`:
|
|
|
25
28
|
npm i -g @adguard/hostlist-compiler
|
|
26
29
|
```
|
|
27
30
|
|
|
31
|
+
After that you have two options.
|
|
32
|
+
|
|
33
|
+
**Quick hosts conversion**
|
|
34
|
+
|
|
35
|
+
Convert and compress a `/etc/hosts`-syntax blocklist to [AdGuard syntax](https://adguard-dns.io/kb/general/dns-filtering-syntax/).
|
|
36
|
+
|
|
37
|
+
```
|
|
38
|
+
hostlist-compiler -i hosts.txt -i hosts2.txt -o output.txt
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
**Build a configurable blocklist from multiple sources**
|
|
42
|
+
|
|
28
43
|
Prepare the list configuration (read more about that below) and run the compiler:
|
|
29
44
|
|
|
30
45
|
```bash
|
|
@@ -131,28 +146,83 @@ Examples:
|
|
|
131
146
|
|
|
132
147
|
### <a id="api"></a> API
|
|
133
148
|
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
149
|
+
Install: `npm i @adguard/hostlist-compiler` or `yarn add @adguard/hostlist-compiler`
|
|
150
|
+
|
|
151
|
+
#### JavaScript example:
|
|
137
152
|
|
|
138
153
|
```javascript
|
|
139
154
|
const compile = require("@adguard/hostlist-compiler");
|
|
140
155
|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
156
|
+
;(async () => {
|
|
157
|
+
// Compile filters
|
|
158
|
+
const result = await compile({
|
|
159
|
+
name: 'Your Hostlist',
|
|
160
|
+
sources: [
|
|
161
|
+
{
|
|
162
|
+
type: 'adblock',
|
|
163
|
+
source: 'https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt', // or local file
|
|
164
|
+
transformations: ['RemoveComments', 'Validate'],
|
|
165
|
+
},
|
|
166
|
+
],
|
|
167
|
+
transformations: ['Deduplicate'],
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
// Write to file
|
|
171
|
+
writeFileSync('your-hostlist.txt', result.join('\n'));
|
|
172
|
+
})();
|
|
173
|
+
```
|
|
150
174
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
175
|
+
#### TypeScript example:
|
|
176
|
+
|
|
177
|
+
```typescript
|
|
178
|
+
import compile from '@adguard/hostlist-compiler';
|
|
179
|
+
import { writeFileSync } from 'fs';
|
|
180
|
+
|
|
181
|
+
;(async () => {
|
|
182
|
+
// Compile filters
|
|
183
|
+
const result = await compile({
|
|
184
|
+
name: 'Your Hostlist',
|
|
185
|
+
sources: [
|
|
186
|
+
{
|
|
187
|
+
type: 'adblock',
|
|
188
|
+
source: 'https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt',
|
|
189
|
+
transformations: ['RemoveComments', 'Validate'],
|
|
190
|
+
},
|
|
191
|
+
],
|
|
192
|
+
transformations: ['Deduplicate'],
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
// Write to file
|
|
196
|
+
writeFileSync('your-hostlist.txt', result.join('\n'));
|
|
197
|
+
})();
|
|
198
|
+
```
|
|
154
199
|
|
|
155
|
-
|
|
200
|
+
or:
|
|
201
|
+
|
|
202
|
+
```typescript
|
|
203
|
+
import HostlistCompiler, { IConfiguration as HostlistCompilerConfiguration } from '@adguard/hostlist-compiler';
|
|
204
|
+
import { writeFileSync } from 'fs';
|
|
205
|
+
|
|
206
|
+
;(async () => {
|
|
207
|
+
// Configuration
|
|
208
|
+
const config: HostlistCompilerConfiguration = {
|
|
209
|
+
name: 'Your Hostlist',
|
|
210
|
+
sources: [
|
|
211
|
+
{
|
|
212
|
+
type: 'adblock',
|
|
213
|
+
source: 'https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt',
|
|
214
|
+
transformations: ['RemoveComments', 'Validate'],
|
|
215
|
+
},
|
|
216
|
+
],
|
|
217
|
+
transformations: ['Deduplicate'],
|
|
218
|
+
};
|
|
219
|
+
|
|
220
|
+
// Compile filters
|
|
221
|
+
const result = await HostlistCompiler(config);
|
|
222
|
+
|
|
223
|
+
// Write to file
|
|
224
|
+
writeFileSync('your-hostlist.txt', result.join('\n'));
|
|
225
|
+
})();
|
|
156
226
|
```
|
|
157
227
|
|
|
158
228
|
## <a id="transformations"></a> Transformations
|
|
@@ -165,6 +235,9 @@ Here is the full list of transformations that are available:
|
|
|
165
235
|
4. `Validate`
|
|
166
236
|
5. `Deduplicate`
|
|
167
237
|
6. `InvertAllow`
|
|
238
|
+
7. `RemoveEmptyLines`
|
|
239
|
+
8. `TrimLines`
|
|
240
|
+
9. `InsertFinalNewLine`
|
|
168
241
|
|
|
169
242
|
Please note that these transformations are are always applied in the order specified here.
|
|
170
243
|
|
|
@@ -188,7 +261,7 @@ By default, [AdGuard Home](https://github.com/AdguardTeam/AdGuardHome) will igno
|
|
|
188
261
|
Here is the list of modifiers that will be removed:
|
|
189
262
|
|
|
190
263
|
- `$third-party` and `$3p` modifiers
|
|
191
|
-
- `$document`
|
|
264
|
+
- `$document` and `$doc` modifiers
|
|
192
265
|
- `$all` modifier
|
|
193
266
|
- `$popup` modifier
|
|
194
267
|
|
|
@@ -266,6 +339,80 @@ Here's what we will have after applying this transformation:
|
|
|
266
339
|
@@rule2
|
|
267
340
|
```
|
|
268
341
|
|
|
342
|
+
### <a id="removeemptylines"></a> RemoveEmptyLines
|
|
343
|
+
|
|
344
|
+
This is a very simple transformation that removes empty lines.
|
|
345
|
+
|
|
346
|
+
**Example:**
|
|
347
|
+
|
|
348
|
+
Original list:
|
|
349
|
+
|
|
350
|
+
```
|
|
351
|
+
rule1
|
|
352
|
+
|
|
353
|
+
rule2
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
rule3
|
|
357
|
+
```
|
|
358
|
+
|
|
359
|
+
Here's what we will have after applying this transformation:
|
|
360
|
+
|
|
361
|
+
```
|
|
362
|
+
rule1
|
|
363
|
+
rule2
|
|
364
|
+
rule3
|
|
365
|
+
```
|
|
366
|
+
|
|
367
|
+
### <a id="trimlines"></a> TrimLines
|
|
368
|
+
|
|
369
|
+
This is a very simple transformation that removes leading and trailing spaces/tabs.
|
|
370
|
+
|
|
371
|
+
**Example:**
|
|
372
|
+
|
|
373
|
+
Original list:
|
|
374
|
+
|
|
375
|
+
```
|
|
376
|
+
rule1
|
|
377
|
+
rule2
|
|
378
|
+
rule3
|
|
379
|
+
rule4
|
|
380
|
+
```
|
|
381
|
+
|
|
382
|
+
Here's what we will have after applying this transformation:
|
|
383
|
+
|
|
384
|
+
```
|
|
385
|
+
rule1
|
|
386
|
+
rule2
|
|
387
|
+
rule3
|
|
388
|
+
rule4
|
|
389
|
+
```
|
|
390
|
+
|
|
391
|
+
### <a id="insertfinalnewline"></a> InsertFinalNewLine
|
|
392
|
+
|
|
393
|
+
This is a very simple transformation that inserts a final newline.
|
|
394
|
+
|
|
395
|
+
**Example:**
|
|
396
|
+
|
|
397
|
+
Original list:
|
|
398
|
+
|
|
399
|
+
```
|
|
400
|
+
rule1
|
|
401
|
+
rule2
|
|
402
|
+
rule3
|
|
403
|
+
```
|
|
404
|
+
|
|
405
|
+
Here's what we will have after applying this transformation:
|
|
406
|
+
|
|
407
|
+
```
|
|
408
|
+
rule1
|
|
409
|
+
rule2
|
|
410
|
+
rule3
|
|
411
|
+
|
|
412
|
+
```
|
|
413
|
+
|
|
414
|
+
`RemoveEmptyLines` doesn't delete this empty row due to the execution order.
|
|
415
|
+
|
|
269
416
|
## <a id="how-to-build"></a> How to build
|
|
270
417
|
|
|
271
418
|
- `yarn install` - installs dependencies
|
package/bamboo-specs/bamboo.yaml
CHANGED
package/package.json
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adguard/hostlist-compiler",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.16",
|
|
4
4
|
"description": "A simple tool that compiles hosts blocklists from multiple sources",
|
|
5
5
|
"main": "src/index.js",
|
|
6
|
+
"types": "src/index.d.ts",
|
|
6
7
|
"repository": "https://github.com/AdguardTeam/HostlistCompiler",
|
|
7
8
|
"author": "AdGuard",
|
|
8
9
|
"license": "GPL-3.0",
|
package/src/cli.js
CHANGED
|
@@ -8,13 +8,32 @@ const packageJson = require('../package.json');
|
|
|
8
8
|
// eslint-disable-next-line import/order
|
|
9
9
|
const { argv } = require('yargs')
|
|
10
10
|
.usage('Usage: $0 [options]')
|
|
11
|
-
.example(
|
|
11
|
+
.example(
|
|
12
|
+
'$0 -c config.json -o output.txt',
|
|
13
|
+
'compile a blocklist and write the output to output.txt',
|
|
14
|
+
)
|
|
15
|
+
.example(
|
|
16
|
+
'$0 -i https://example.org/hosts.txt -o output.txt',
|
|
17
|
+
'compile a blocklist from the URL and write the output to output.txt',
|
|
18
|
+
)
|
|
12
19
|
.option('config', {
|
|
13
20
|
alias: 'c',
|
|
14
21
|
type: 'string',
|
|
15
22
|
description: 'Path to the compiler configuration file',
|
|
16
23
|
nargs: 1,
|
|
17
24
|
})
|
|
25
|
+
.option('input', {
|
|
26
|
+
alias: 'i',
|
|
27
|
+
array: true,
|
|
28
|
+
type: 'string',
|
|
29
|
+
description: 'URL (or path to a file) to convert to an AdGuard-syntax blocklist. Can be specified multiple times.',
|
|
30
|
+
})
|
|
31
|
+
.option('input-type', {
|
|
32
|
+
alias: 't',
|
|
33
|
+
type: 'string',
|
|
34
|
+
description: 'Type of the input file (/etc/hosts, adguard)',
|
|
35
|
+
nargs: 1,
|
|
36
|
+
})
|
|
18
37
|
.option('output', {
|
|
19
38
|
alias: 'o',
|
|
20
39
|
type: 'string',
|
|
@@ -26,25 +45,77 @@ const { argv } = require('yargs')
|
|
|
26
45
|
type: 'boolean',
|
|
27
46
|
description: 'Run with verbose logging',
|
|
28
47
|
})
|
|
29
|
-
.demandOption(['
|
|
48
|
+
.demandOption(['o'])
|
|
30
49
|
.version()
|
|
31
50
|
.help('h')
|
|
32
51
|
.alias('h', 'help');
|
|
33
52
|
|
|
34
53
|
if (argv.verbose) {
|
|
35
|
-
// trace level
|
|
54
|
+
// trace level.
|
|
36
55
|
consola.level = 5;
|
|
37
56
|
}
|
|
38
57
|
|
|
39
58
|
consola.info(`Starting ${packageJson.name} v${packageJson.version}`);
|
|
40
59
|
|
|
60
|
+
/**
|
|
61
|
+
* Reads the configuration file and returns the parsed JSON.
|
|
62
|
+
*
|
|
63
|
+
* @returns {Promise<Object> | Promise<Error>} configuration file as JSON.
|
|
64
|
+
*/
|
|
65
|
+
async function readConfig() {
|
|
66
|
+
consola.debug(`Reading configuration from ${argv.config}`);
|
|
67
|
+
|
|
68
|
+
// Check if the configuration file exists.
|
|
69
|
+
await fs.access(argv.config);
|
|
70
|
+
const configStr = (await fs.readFile(argv.config)).toString();
|
|
71
|
+
|
|
72
|
+
return JSON.parse(configStr);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Creates a configuration object for the specified blocklist source.
|
|
77
|
+
* We assume that the input is a /etc/hosts file.
|
|
78
|
+
*
|
|
79
|
+
* @returns {Object} configuration object.
|
|
80
|
+
*/
|
|
81
|
+
function createConfig() {
|
|
82
|
+
const inputType = argv['input-type'] || 'hosts';
|
|
83
|
+
|
|
84
|
+
consola.debug(`Creating configuration for input ${argv.input} of type ${inputType}`);
|
|
85
|
+
|
|
86
|
+
const config = {
|
|
87
|
+
name: 'Blocklist',
|
|
88
|
+
sources: [],
|
|
89
|
+
transformations: [
|
|
90
|
+
'RemoveComments',
|
|
91
|
+
'Deduplicate',
|
|
92
|
+
'Compress',
|
|
93
|
+
'Validate',
|
|
94
|
+
'TrimLines',
|
|
95
|
+
'InsertFinalNewLine',
|
|
96
|
+
],
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
argv.input.forEach((input) => {
|
|
100
|
+
config.sources.push({
|
|
101
|
+
source: input,
|
|
102
|
+
type: inputType,
|
|
103
|
+
});
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
return config;
|
|
107
|
+
}
|
|
108
|
+
|
|
41
109
|
async function main() {
|
|
42
110
|
try {
|
|
43
|
-
|
|
44
|
-
|
|
111
|
+
if (!argv.input && !argv.config) {
|
|
112
|
+
throw new Error('Either --input or --config must be specified');
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
const config = argv.input ? createConfig() : await readConfig();
|
|
116
|
+
|
|
117
|
+
consola.debug(`Configuration: ${JSON.stringify(config, 0, 4)}`);
|
|
45
118
|
|
|
46
|
-
const configStr = (await fs.readFile(argv.config)).toString();
|
|
47
|
-
const config = JSON.parse(configStr);
|
|
48
119
|
const lines = await compile(config);
|
|
49
120
|
|
|
50
121
|
consola.info(`Writing output to ${argv.output}`);
|
package/src/compile-source.js
CHANGED
|
@@ -17,7 +17,7 @@ const { transform } = require('./transformations/transform');
|
|
|
17
17
|
* Compiles an individual source according to it's configuration.
|
|
18
18
|
*
|
|
19
19
|
* @param {ListSource} source - source configuration.
|
|
20
|
-
* @returns {Array<string
|
|
20
|
+
* @returns {Promise<Array<string>>} array with the source rules
|
|
21
21
|
*/
|
|
22
22
|
async function compileSource(source) {
|
|
23
23
|
consola.info(`Start compiling ${source.source}`);
|
package/src/filter.js
CHANGED
|
@@ -7,7 +7,7 @@ const ruleUtils = require('./rule');
|
|
|
7
7
|
* and return a final array with all the lines from all files.
|
|
8
8
|
*
|
|
9
9
|
* @param {Array<String>} sources - array of URLs.
|
|
10
|
-
* @returns {Array<String
|
|
10
|
+
* @returns {Promise<Array<String>>} array with all non-empty and non-comment lines.
|
|
11
11
|
*/
|
|
12
12
|
async function downloadAll(sources) {
|
|
13
13
|
let list = [];
|
|
@@ -32,7 +32,7 @@ async function downloadAll(sources) {
|
|
|
32
32
|
* @param {Array<String>} rules - array of rules to apply
|
|
33
33
|
* @param {Array<String>} sources - array of rules sources
|
|
34
34
|
* (can be local or remote files)
|
|
35
|
-
* @returns {Array<utils.Wildcard
|
|
35
|
+
* @returns {Promise<Array<utils.Wildcard>>} a list of wildcards to apply
|
|
36
36
|
*/
|
|
37
37
|
async function prepareWildcards(rules, sources) {
|
|
38
38
|
let list = [];
|
package/src/index.d.ts
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
declare module '@adguard/hostlist-compiler' {
|
|
2
|
+
export type SourceType = 'adblock' | 'hosts';
|
|
3
|
+
|
|
4
|
+
export type Transformation =
|
|
5
|
+
'RemoveComments' |
|
|
6
|
+
'Compress' |
|
|
7
|
+
'RemoveModifiers' |
|
|
8
|
+
'Validate' |
|
|
9
|
+
'Deduplicate' |
|
|
10
|
+
'InvertAllow' |
|
|
11
|
+
'RemoveEmptyLines' |
|
|
12
|
+
'TrimLines' |
|
|
13
|
+
'InsertFinalNewLine';
|
|
14
|
+
|
|
15
|
+
/** A source for the filter list */
|
|
16
|
+
export interface ISource {
|
|
17
|
+
/** Name of the source */
|
|
18
|
+
name?: string;
|
|
19
|
+
/** Path to a file or a URL */
|
|
20
|
+
source: string;
|
|
21
|
+
/** Type of the source */
|
|
22
|
+
type?: SourceType;
|
|
23
|
+
/** A list of the transformations that will be applied */
|
|
24
|
+
transformations?: Transformation[];
|
|
25
|
+
/** A list of rules (or wildcards) to exclude from the source. */
|
|
26
|
+
exclusions?: string[];
|
|
27
|
+
/** An array of exclusions sources. */
|
|
28
|
+
exclusions_sources?: string[];
|
|
29
|
+
/** A list of wildcards to include from the source. All rules that don't match these wildcards won't be included. */
|
|
30
|
+
inclusions?: string[];
|
|
31
|
+
/** A list of files with inclusions. */
|
|
32
|
+
inclusions_sources?: string[];
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Configuration for the hostlist compiler */
|
|
36
|
+
export interface IConfiguration {
|
|
37
|
+
/** Filter list name */
|
|
38
|
+
name: string;
|
|
39
|
+
/** Filter list description */
|
|
40
|
+
description?: string;
|
|
41
|
+
/** Filter list homepage */
|
|
42
|
+
homepage?: string;
|
|
43
|
+
/** Filter list license */
|
|
44
|
+
license?: string;
|
|
45
|
+
/** An array of the filter list sources */
|
|
46
|
+
sources: ISource[];
|
|
47
|
+
/** A list of the transformations that will be applied */
|
|
48
|
+
transformations?: Transformation[];
|
|
49
|
+
/** A list of rules (or wildcards) to exclude from the source. */
|
|
50
|
+
exclusions?: string[];
|
|
51
|
+
/** An array of exclusions sources. */
|
|
52
|
+
exclusions_sources?: string[];
|
|
53
|
+
/** A list of wildcards to include from the source. All rules that don't match these wildcards won't be included. */
|
|
54
|
+
inclusions?: string[];
|
|
55
|
+
/** A list of files with inclusions. */
|
|
56
|
+
inclusions_sources?: string[];
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Compiles a filter list using the specified configuration.
|
|
61
|
+
*
|
|
62
|
+
* @param {*} configuration - compilation configuration.
|
|
63
|
+
See the repo README for the details on it.
|
|
64
|
+
* @returns {Promise<Array<string>>} the array of rules.
|
|
65
|
+
*/
|
|
66
|
+
declare async function compile(configuration: IConfiguration): Promise<string[]>;
|
|
67
|
+
|
|
68
|
+
export default compile;
|
|
69
|
+
}
|
package/src/index.js
CHANGED
|
@@ -65,7 +65,7 @@ function prepareSourceHeader(source) {
|
|
|
65
65
|
*
|
|
66
66
|
* @param {*} configuration - compilation configuration.
|
|
67
67
|
See the repo README for the details on it.
|
|
68
|
-
* @returns {Array<string
|
|
68
|
+
* @returns {Promise<Array<string>>} the array of rules.
|
|
69
69
|
*/
|
|
70
70
|
async function compile(configuration) {
|
|
71
71
|
consola.info('Starting the compiler');
|
|
@@ -41,7 +41,10 @@
|
|
|
41
41
|
"Compress",
|
|
42
42
|
"Validate",
|
|
43
43
|
"Deduplicate",
|
|
44
|
-
"InvertAllow"
|
|
44
|
+
"InvertAllow",
|
|
45
|
+
"RemoveEmptyLines",
|
|
46
|
+
"TrimLines",
|
|
47
|
+
"InsertFinalNewLine"
|
|
45
48
|
]
|
|
46
49
|
}
|
|
47
50
|
},
|
|
@@ -113,7 +116,10 @@
|
|
|
113
116
|
"Compress",
|
|
114
117
|
"Validate",
|
|
115
118
|
"Deduplicate",
|
|
116
|
-
"InvertAllow"
|
|
119
|
+
"InvertAllow",
|
|
120
|
+
"RemoveEmptyLines",
|
|
121
|
+
"TrimLines",
|
|
122
|
+
"InsertFinalNewLine"
|
|
117
123
|
]
|
|
118
124
|
}
|
|
119
125
|
},
|
|
@@ -9,7 +9,7 @@ const filterUtils = require('../filter');
|
|
|
9
9
|
* @param {Array<String>} exclusions - array of exclusions to apply
|
|
10
10
|
* @param {Array<String>} exclusionsSources - array of exclusion sources
|
|
11
11
|
* (can be a local or remote file)
|
|
12
|
-
* @returns {Array<String
|
|
12
|
+
* @returns {Promise<Array<String>>} filtered array of rules
|
|
13
13
|
*/
|
|
14
14
|
async function exclude(rules, exclusions, exclusionsSources) {
|
|
15
15
|
if (_.isEmpty(exclusions) && _.isEmpty(exclusionsSources)) {
|
|
@@ -9,7 +9,7 @@ const filterUtils = require('../filter');
|
|
|
9
9
|
* @param {Array<String>} inclusions - array of inclusions to apply
|
|
10
10
|
* @param {Array<String>} inclusionsSources - array of inclusions' sources
|
|
11
11
|
* (can be a local or remote file)
|
|
12
|
-
* @returns {Array<String
|
|
12
|
+
* @returns {Promise<Array<String>>} filtered array of rules
|
|
13
13
|
*/
|
|
14
14
|
async function include(rules, inclusions, inclusionsSources) {
|
|
15
15
|
if (_.isEmpty(inclusions) && _.isEmpty(inclusionsSources)) {
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
const consola = require('consola');
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* This is a very simple transformation that inserts a final newline.
|
|
5
|
+
*
|
|
6
|
+
* @param {Array<string>} lines - lines to transform
|
|
7
|
+
* @returns {Array<string>} filtered lines
|
|
8
|
+
*/
|
|
9
|
+
function insertFinalNewLine(lines) {
|
|
10
|
+
if (lines.length === 0 || (lines.length > 0 && lines[lines.length - 1].trim() !== '')) {
|
|
11
|
+
lines.push('');
|
|
12
|
+
consola.info('Final newline inserted');
|
|
13
|
+
}
|
|
14
|
+
return lines;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
module.exports = insertFinalNewLine;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
const consola = require('consola');
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* This is a very simple transformation that removes empty lines.
|
|
5
|
+
*
|
|
6
|
+
* @param {Array<string>} lines - lines to transform
|
|
7
|
+
* @returns {Array<string>} filtered lines
|
|
8
|
+
*/
|
|
9
|
+
function removeEmptyLines(lines) {
|
|
10
|
+
const filtered = lines.filter((line) => line.trim().length);
|
|
11
|
+
consola.info(`Removed ${lines.length - filtered.length} empty lines`);
|
|
12
|
+
return filtered;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
module.exports = removeEmptyLines;
|
|
@@ -4,7 +4,7 @@ const ruleUtils = require('../rule');
|
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* This transformation simply removes the following modifiers from the adblock-style rules:
|
|
7
|
-
* $document, $all, $third-party, $3p, $popup
|
|
7
|
+
* $document, $doc, $all, $third-party, $3p, $popup
|
|
8
8
|
*
|
|
9
9
|
* @param {Array<string>} rules - rules to transform
|
|
10
10
|
* @returns {Array<string>} filtered rules
|
|
@@ -32,6 +32,7 @@ function removeModifiers(rules) {
|
|
|
32
32
|
modified = ruleUtils.removeModifier(props, '3p') || modified;
|
|
33
33
|
modified = ruleUtils.removeModifier(props, 'all') || modified;
|
|
34
34
|
modified = ruleUtils.removeModifier(props, 'document') || modified;
|
|
35
|
+
modified = ruleUtils.removeModifier(props, 'doc') || modified;
|
|
35
36
|
modified = ruleUtils.removeModifier(props, 'popup') || modified;
|
|
36
37
|
filtered.push(ruleUtils.adblockRuleToString(props));
|
|
37
38
|
|
|
@@ -6,6 +6,9 @@ const include = require('./include');
|
|
|
6
6
|
const deduplicate = require('./deduplicate');
|
|
7
7
|
const compress = require('./compress');
|
|
8
8
|
const invertAllow = require('./invertallow');
|
|
9
|
+
const removeEmptyLines = require('./remove-empty-lines');
|
|
10
|
+
const trimLines = require('./trim-lines');
|
|
11
|
+
const insertFinalNewLine = require('./insert-final-newline');
|
|
9
12
|
|
|
10
13
|
/**
|
|
11
14
|
* Enum with all available transformations
|
|
@@ -17,6 +20,9 @@ const TRANSFORMATIONS = Object.freeze({
|
|
|
17
20
|
Validate: 'Validate',
|
|
18
21
|
Deduplicate: 'Deduplicate',
|
|
19
22
|
InvertAllow: 'InvertAllow',
|
|
23
|
+
RemoveEmptyLines: 'RemoveEmptyLines',
|
|
24
|
+
TrimLines: 'TrimLines',
|
|
25
|
+
InsertFinalNewLine: 'InsertFinalNewLine',
|
|
20
26
|
});
|
|
21
27
|
|
|
22
28
|
/**
|
|
@@ -25,7 +31,7 @@ const TRANSFORMATIONS = Object.freeze({
|
|
|
25
31
|
* @param {Array<string>} rules - rules to transform
|
|
26
32
|
* @param {*} configuration - transformation configuration.
|
|
27
33
|
* @param {Array<string>} transformations - a list of transformations to apply to the rules.
|
|
28
|
-
* @returns {Array<string
|
|
34
|
+
* @returns {Promise<Array<string>>} rules after applying all transformations.
|
|
29
35
|
*/
|
|
30
36
|
async function transform(rules, configuration, transformations) {
|
|
31
37
|
// If none specified -- apply all transformationss
|
|
@@ -41,6 +47,9 @@ async function transform(rules, configuration, transformations) {
|
|
|
41
47
|
transformed = await include(transformed, configuration.inclusions,
|
|
42
48
|
configuration.inclusions_sources);
|
|
43
49
|
|
|
50
|
+
if (transformations.indexOf(TRANSFORMATIONS.TrimLines) !== -1) {
|
|
51
|
+
transformed = trimLines(transformed);
|
|
52
|
+
}
|
|
44
53
|
if (transformations.indexOf(TRANSFORMATIONS.RemoveComments) !== -1) {
|
|
45
54
|
transformed = removeComments(transformed);
|
|
46
55
|
}
|
|
@@ -59,6 +68,12 @@ async function transform(rules, configuration, transformations) {
|
|
|
59
68
|
if (transformations.indexOf(TRANSFORMATIONS.Deduplicate) !== -1) {
|
|
60
69
|
transformed = deduplicate(transformed);
|
|
61
70
|
}
|
|
71
|
+
if (transformations.indexOf(TRANSFORMATIONS.RemoveEmptyLines) !== -1) {
|
|
72
|
+
transformed = removeEmptyLines(transformed);
|
|
73
|
+
}
|
|
74
|
+
if (transformations.indexOf(TRANSFORMATIONS.InsertFinalNewLine) !== -1) {
|
|
75
|
+
transformed = insertFinalNewLine(transformed);
|
|
76
|
+
}
|
|
62
77
|
return transformed;
|
|
63
78
|
}
|
|
64
79
|
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
const consola = require('consola');
|
|
2
|
+
const _ = require('lodash');
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* This is a very simple transformation that removes leading and trailing spaces/tabs.
|
|
6
|
+
*
|
|
7
|
+
* @param {Array<string>} lines - lines/rules to transform
|
|
8
|
+
* @returns {Array<string>} filtered lines/rules
|
|
9
|
+
*/
|
|
10
|
+
function trimLines(lines) {
|
|
11
|
+
const transformed = lines.map((line) => _.trim(line, ' \t'));
|
|
12
|
+
consola.info('Lines trimmed.');
|
|
13
|
+
return transformed;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
module.exports = trimLines;
|
package/src/utils.js
CHANGED
|
@@ -16,7 +16,7 @@ function isURL(str) {
|
|
|
16
16
|
* Downloads (or reads from the disk) the specified source
|
|
17
17
|
*
|
|
18
18
|
* @param {*} urlOrPath url or path to a file
|
|
19
|
-
* @returns {String} contents of the files
|
|
19
|
+
* @returns {Promise<String>} contents of the files
|
|
20
20
|
*/
|
|
21
21
|
async function download(urlOrPath) {
|
|
22
22
|
let str = '';
|
|
@@ -85,4 +85,40 @@ describe('Source compiler', () => {
|
|
|
85
85
|
// Make sure scope URLs were requested
|
|
86
86
|
scope.done();
|
|
87
87
|
});
|
|
88
|
+
|
|
89
|
+
it('compile the source with transformations', async () => {
|
|
90
|
+
const testList = `
|
|
91
|
+
|
|
92
|
+
example.org
|
|
93
|
+
|
|
94
|
+
test1.com
|
|
95
|
+
|
|
96
|
+
! comment
|
|
97
|
+
test1.com
|
|
98
|
+
|
|
99
|
+
test2.com`;
|
|
100
|
+
const scope = nock('https://example.org')
|
|
101
|
+
.get('/test-filter.txt')
|
|
102
|
+
.reply(200, testList);
|
|
103
|
+
|
|
104
|
+
const source = {
|
|
105
|
+
name: 'test source',
|
|
106
|
+
source: 'https://example.org/test-filter.txt',
|
|
107
|
+
transformations: [
|
|
108
|
+
TRANSFORMATIONS.Deduplicate,
|
|
109
|
+
TRANSFORMATIONS.RemoveComments,
|
|
110
|
+
TRANSFORMATIONS.TrimLines,
|
|
111
|
+
TRANSFORMATIONS.RemoveEmptyLines,
|
|
112
|
+
TRANSFORMATIONS.InsertFinalNewLine,
|
|
113
|
+
],
|
|
114
|
+
};
|
|
115
|
+
const compiled = await compileSource(source);
|
|
116
|
+
expect(compiled).toEqual([
|
|
117
|
+
'example.org',
|
|
118
|
+
'test1.com',
|
|
119
|
+
'test2.com',
|
|
120
|
+
'',
|
|
121
|
+
]);
|
|
122
|
+
scope.done();
|
|
123
|
+
});
|
|
88
124
|
});
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
const insertFinalNewLine = require('../../src/transformations/insert-final-newline');
|
|
2
|
+
|
|
3
|
+
describe('Insert final new line', () => {
|
|
4
|
+
it('test with empty file', () => {
|
|
5
|
+
const rules = [];
|
|
6
|
+
const filtered = insertFinalNewLine(rules);
|
|
7
|
+
expect(filtered).toEqual(['']);
|
|
8
|
+
});
|
|
9
|
+
it('test with one empty line', () => {
|
|
10
|
+
const rules = [''];
|
|
11
|
+
const filtered = insertFinalNewLine(rules);
|
|
12
|
+
expect(filtered).toEqual(['']);
|
|
13
|
+
});
|
|
14
|
+
it('test with one rule and one empty lines', () => {
|
|
15
|
+
const rules = [
|
|
16
|
+
'rule1',
|
|
17
|
+
'',
|
|
18
|
+
];
|
|
19
|
+
const filtered = insertFinalNewLine(rules);
|
|
20
|
+
expect(filtered).toEqual([
|
|
21
|
+
'rule1',
|
|
22
|
+
'',
|
|
23
|
+
]);
|
|
24
|
+
});
|
|
25
|
+
it('test with many empty lines', () => {
|
|
26
|
+
const rules = [
|
|
27
|
+
'rule1',
|
|
28
|
+
'',
|
|
29
|
+
'',
|
|
30
|
+
];
|
|
31
|
+
const filtered = insertFinalNewLine(rules);
|
|
32
|
+
expect(filtered).toEqual([
|
|
33
|
+
'rule1',
|
|
34
|
+
'',
|
|
35
|
+
'',
|
|
36
|
+
]);
|
|
37
|
+
});
|
|
38
|
+
it('test with one rule', () => {
|
|
39
|
+
const rules = [
|
|
40
|
+
'rule1',
|
|
41
|
+
];
|
|
42
|
+
const filtered = insertFinalNewLine(rules);
|
|
43
|
+
expect(filtered).toEqual([
|
|
44
|
+
'rule1',
|
|
45
|
+
'',
|
|
46
|
+
]);
|
|
47
|
+
});
|
|
48
|
+
});
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
const removeEmptyLines = require('../../src/transformations/remove-empty-lines');
|
|
2
|
+
|
|
3
|
+
describe('Remove empty lines', () => {
|
|
4
|
+
it('test with no empty lines', () => {
|
|
5
|
+
const rules = [
|
|
6
|
+
'rule1',
|
|
7
|
+
'rule2',
|
|
8
|
+
];
|
|
9
|
+
const filtered = removeEmptyLines(rules);
|
|
10
|
+
expect(filtered).toEqual([
|
|
11
|
+
'rule1',
|
|
12
|
+
'rule2',
|
|
13
|
+
]);
|
|
14
|
+
});
|
|
15
|
+
it('test with one empty line at the end', () => {
|
|
16
|
+
const rules = [
|
|
17
|
+
'rule1',
|
|
18
|
+
'rule2',
|
|
19
|
+
'',
|
|
20
|
+
];
|
|
21
|
+
const filtered = removeEmptyLines(rules);
|
|
22
|
+
expect(filtered).toEqual([
|
|
23
|
+
'rule1',
|
|
24
|
+
'rule2',
|
|
25
|
+
]);
|
|
26
|
+
});
|
|
27
|
+
it('test with one empty line between the rules', () => {
|
|
28
|
+
const rules = [
|
|
29
|
+
'rule1',
|
|
30
|
+
'',
|
|
31
|
+
'rule2',
|
|
32
|
+
];
|
|
33
|
+
const filtered = removeEmptyLines(rules);
|
|
34
|
+
expect(filtered).toEqual([
|
|
35
|
+
'rule1',
|
|
36
|
+
'rule2',
|
|
37
|
+
]);
|
|
38
|
+
});
|
|
39
|
+
it('test with two empty line', () => {
|
|
40
|
+
const rules = [
|
|
41
|
+
'rule1',
|
|
42
|
+
'',
|
|
43
|
+
'',
|
|
44
|
+
'rule2',
|
|
45
|
+
];
|
|
46
|
+
const filtered = removeEmptyLines(rules);
|
|
47
|
+
expect(filtered).toEqual([
|
|
48
|
+
'rule1',
|
|
49
|
+
'rule2',
|
|
50
|
+
]);
|
|
51
|
+
});
|
|
52
|
+
it('test with comments, empty lines', () => {
|
|
53
|
+
const rules = [
|
|
54
|
+
'! aaa',
|
|
55
|
+
'rule1',
|
|
56
|
+
'',
|
|
57
|
+
'! bbb',
|
|
58
|
+
'',
|
|
59
|
+
'',
|
|
60
|
+
'rule2',
|
|
61
|
+
'',
|
|
62
|
+
'',
|
|
63
|
+
'',
|
|
64
|
+
'!ccc',
|
|
65
|
+
'',
|
|
66
|
+
'',
|
|
67
|
+
];
|
|
68
|
+
const filtered = removeEmptyLines(rules);
|
|
69
|
+
expect(filtered).toEqual([
|
|
70
|
+
'! aaa',
|
|
71
|
+
'rule1',
|
|
72
|
+
'! bbb',
|
|
73
|
+
'rule2',
|
|
74
|
+
'!ccc',
|
|
75
|
+
]);
|
|
76
|
+
});
|
|
77
|
+
});
|
|
@@ -7,17 +7,19 @@ describe('Strip third-party', () => {
|
|
|
7
7
|
||example.net$domain=ya.ru,3p
|
|
8
8
|
||islandofadvert.com^$document,popup
|
|
9
9
|
||example.com$document
|
|
10
|
+
||example.com$doc
|
|
10
11
|
||example.com$all
|
|
11
12
|
||example.org^`.split(/\r?\n/);
|
|
12
13
|
const filtered = removeModifiers(rules);
|
|
13
14
|
|
|
14
|
-
expect(filtered).toHaveLength(
|
|
15
|
+
expect(filtered).toHaveLength(8);
|
|
15
16
|
expect(filtered[0]).toBe('! test comment');
|
|
16
17
|
expect(filtered[1]).toBe('||example.org$important');
|
|
17
18
|
expect(filtered[2]).toBe('||example.net$domain=ya.ru');
|
|
18
19
|
expect(filtered[3]).toBe('||islandofadvert.com^');
|
|
19
20
|
expect(filtered[4]).toBe('||example.com');
|
|
20
21
|
expect(filtered[5]).toBe('||example.com');
|
|
21
|
-
expect(filtered[6]).toBe('||example.
|
|
22
|
+
expect(filtered[6]).toBe('||example.com');
|
|
23
|
+
expect(filtered[7]).toBe('||example.org^');
|
|
22
24
|
});
|
|
23
25
|
});
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
const _ = require('lodash');
|
|
2
|
+
const trimLines = require('../../src/transformations/trim-lines');
|
|
3
|
+
|
|
4
|
+
describe('Trim lines', () => {
|
|
5
|
+
it('test with empty file', () => {
|
|
6
|
+
const rules = [];
|
|
7
|
+
const filtered = trimLines(rules);
|
|
8
|
+
expect(filtered).toEqual([]);
|
|
9
|
+
});
|
|
10
|
+
it('test with one empty line', () => {
|
|
11
|
+
const rules = [''];
|
|
12
|
+
const filtered = trimLines(rules);
|
|
13
|
+
expect(filtered).toEqual(['']);
|
|
14
|
+
});
|
|
15
|
+
it('test with three rules', () => {
|
|
16
|
+
const rules = [
|
|
17
|
+
'rule1',
|
|
18
|
+
' rule2 ',
|
|
19
|
+
'',
|
|
20
|
+
' rule3 ',
|
|
21
|
+
];
|
|
22
|
+
const filtered = trimLines(rules);
|
|
23
|
+
expect(filtered).toEqual([
|
|
24
|
+
'rule1',
|
|
25
|
+
'rule2',
|
|
26
|
+
'',
|
|
27
|
+
'rule3',
|
|
28
|
+
]);
|
|
29
|
+
});
|
|
30
|
+
it('test with three rules and comments', () => {
|
|
31
|
+
const rules = [
|
|
32
|
+
'rule1',
|
|
33
|
+
' rule2 ',
|
|
34
|
+
'',
|
|
35
|
+
'',
|
|
36
|
+
'! comment ',
|
|
37
|
+
' rule3 ',
|
|
38
|
+
' ! comment multiple words ',
|
|
39
|
+
'',
|
|
40
|
+
];
|
|
41
|
+
const filtered = trimLines(rules);
|
|
42
|
+
expect(filtered).toEqual([
|
|
43
|
+
'rule1',
|
|
44
|
+
'rule2',
|
|
45
|
+
'',
|
|
46
|
+
'',
|
|
47
|
+
'! comment',
|
|
48
|
+
'rule3',
|
|
49
|
+
'! comment multiple words',
|
|
50
|
+
'',
|
|
51
|
+
]);
|
|
52
|
+
});
|
|
53
|
+
it('test with three rules and comments', () => {
|
|
54
|
+
const rules = [
|
|
55
|
+
_.repeat(' ', _.random(1, 10)), // ' ', ' ', ..., ' ' etc.
|
|
56
|
+
' ! comment multiple words , ',
|
|
57
|
+
'',
|
|
58
|
+
];
|
|
59
|
+
const filtered = trimLines(rules);
|
|
60
|
+
expect(filtered).toEqual([
|
|
61
|
+
'',
|
|
62
|
+
'! comment multiple words ,',
|
|
63
|
+
'',
|
|
64
|
+
]);
|
|
65
|
+
});
|
|
66
|
+
});
|