@adguard/hostlist-compiler 1.0.11 → 1.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +152 -17
- package/bamboo-specs/bamboo.yaml +1 -1
- package/package.json +2 -1
- package/src/compile-source.js +1 -1
- package/src/filter.js +2 -2
- package/src/index.d.ts +69 -0
- package/src/index.js +1 -1
- package/src/schemas/configuration.schema.json +8 -2
- package/src/transformations/exclude.js +1 -1
- package/src/transformations/include.js +1 -1
- package/src/transformations/insert-final-newline.js +17 -0
- package/src/transformations/remove-empty-lines.js +15 -0
- package/src/transformations/remove-modifiers.js +2 -1
- package/src/transformations/transform.js +16 -1
- package/src/transformations/trim-lines.js +16 -0
- package/src/transformations/validate.js +3 -1
- package/src/utils.js +1 -1
- package/test/compile-source.test.js +36 -0
- package/test/transformations/insert-final-newline.test.js +48 -0
- package/test/transformations/remove-empty-lines.test.js +77 -0
- package/test/transformations/remove-modifiers.test.js +4 -2
- package/test/transformations/trim-lines.test.js +66 -0
- package/test/transformations/validate.test.js +4 -0
package/README.md
CHANGED
|
@@ -15,6 +15,9 @@ This is a simple tool that makes it easier to compile a [hosts blocklist](https:
|
|
|
15
15
|
- [Validate](#validate)
|
|
16
16
|
- [Deduplicate](#deduplicate)
|
|
17
17
|
- [InvertAllow](#invertallow)
|
|
18
|
+
- [RemoveEmptyLines](#removeemptylines)
|
|
19
|
+
- [TrimLines](#trimlines)
|
|
20
|
+
- [InsertFinalNewLine](#insertfinalnewline)
|
|
18
21
|
- [How to build](#how-to-build)
|
|
19
22
|
|
|
20
23
|
## <a id="usage"></a> Usage
|
|
@@ -131,28 +134,83 @@ Examples:
|
|
|
131
134
|
|
|
132
135
|
### <a id="api"></a> API
|
|
133
136
|
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
+
Install: `npm i @adguard/hostlist-compiler` or `yarn add @adguard/hostlist-compiler`
|
|
138
|
+
|
|
139
|
+
#### JavaScript example:
|
|
137
140
|
|
|
138
141
|
```javascript
|
|
139
142
|
const compile = require("@adguard/hostlist-compiler");
|
|
140
143
|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
144
|
+
;(async () => {
|
|
145
|
+
// Compile filters
|
|
146
|
+
const result = await compile({
|
|
147
|
+
name: 'Your Hostlist',
|
|
148
|
+
sources: [
|
|
149
|
+
{
|
|
150
|
+
type: 'adblock',
|
|
151
|
+
source: 'https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt', // or local file
|
|
152
|
+
transformations: ['RemoveComments', 'Validate'],
|
|
153
|
+
},
|
|
154
|
+
],
|
|
155
|
+
transformations: ['Deduplicate'],
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
// Write to file
|
|
159
|
+
writeFileSync('your-hostlist.txt', result.join('\n'));
|
|
160
|
+
})();
|
|
161
|
+
```
|
|
150
162
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
163
|
+
#### TypeScript example:
|
|
164
|
+
|
|
165
|
+
```typescript
|
|
166
|
+
import compile from '@adguard/hostlist-compiler';
|
|
167
|
+
import { writeFileSync } from 'fs';
|
|
168
|
+
|
|
169
|
+
;(async () => {
|
|
170
|
+
// Compile filters
|
|
171
|
+
const result = await compile({
|
|
172
|
+
name: 'Your Hostlist',
|
|
173
|
+
sources: [
|
|
174
|
+
{
|
|
175
|
+
type: 'adblock',
|
|
176
|
+
source: 'https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt',
|
|
177
|
+
transformations: ['RemoveComments', 'Validate'],
|
|
178
|
+
},
|
|
179
|
+
],
|
|
180
|
+
transformations: ['Deduplicate'],
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
// Write to file
|
|
184
|
+
writeFileSync('your-hostlist.txt', result.join('\n'));
|
|
185
|
+
})();
|
|
186
|
+
```
|
|
154
187
|
|
|
155
|
-
|
|
188
|
+
or:
|
|
189
|
+
|
|
190
|
+
```typescript
|
|
191
|
+
import HostlistCompiler, { IConfiguration as HostlistCompilerConfiguration } from '@adguard/hostlist-compiler';
|
|
192
|
+
import { writeFileSync } from 'fs';
|
|
193
|
+
|
|
194
|
+
;(async () => {
|
|
195
|
+
// Configuration
|
|
196
|
+
const config: HostlistCompilerConfiguration = {
|
|
197
|
+
name: 'Your Hostlist',
|
|
198
|
+
sources: [
|
|
199
|
+
{
|
|
200
|
+
type: 'adblock',
|
|
201
|
+
source: 'https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt',
|
|
202
|
+
transformations: ['RemoveComments', 'Validate'],
|
|
203
|
+
},
|
|
204
|
+
],
|
|
205
|
+
transformations: ['Deduplicate'],
|
|
206
|
+
};
|
|
207
|
+
|
|
208
|
+
// Compile filters
|
|
209
|
+
const result = await HostlistCompiler(config);
|
|
210
|
+
|
|
211
|
+
// Write to file
|
|
212
|
+
writeFileSync('your-hostlist.txt', result.join('\n'));
|
|
213
|
+
})();
|
|
156
214
|
```
|
|
157
215
|
|
|
158
216
|
## <a id="transformations"></a> Transformations
|
|
@@ -165,6 +223,9 @@ Here is the full list of transformations that are available:
|
|
|
165
223
|
4. `Validate`
|
|
166
224
|
5. `Deduplicate`
|
|
167
225
|
6. `InvertAllow`
|
|
226
|
+
7. `RemoveEmptyLines`
|
|
227
|
+
8. `TrimLines`
|
|
228
|
+
9. `InsertFinalNewLine`
|
|
168
229
|
|
|
169
230
|
Please note that these transformations are are always applied in the order specified here.
|
|
170
231
|
|
|
@@ -188,7 +249,7 @@ By default, [AdGuard Home](https://github.com/AdguardTeam/AdGuardHome) will igno
|
|
|
188
249
|
Here is the list of modifiers that will be removed:
|
|
189
250
|
|
|
190
251
|
- `$third-party` and `$3p` modifiers
|
|
191
|
-
- `$document`
|
|
252
|
+
- `$document` and `$doc` modifiers
|
|
192
253
|
- `$all` modifier
|
|
193
254
|
- `$popup` modifier
|
|
194
255
|
|
|
@@ -266,6 +327,80 @@ Here's what we will have after applying this transformation:
|
|
|
266
327
|
@@rule2
|
|
267
328
|
```
|
|
268
329
|
|
|
330
|
+
### <a id="removeemptylines"></a> RemoveEmptyLines
|
|
331
|
+
|
|
332
|
+
This is a very simple transformation that removes empty lines.
|
|
333
|
+
|
|
334
|
+
**Example:**
|
|
335
|
+
|
|
336
|
+
Original list:
|
|
337
|
+
|
|
338
|
+
```
|
|
339
|
+
rule1
|
|
340
|
+
|
|
341
|
+
rule2
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
rule3
|
|
345
|
+
```
|
|
346
|
+
|
|
347
|
+
Here's what we will have after applying this transformation:
|
|
348
|
+
|
|
349
|
+
```
|
|
350
|
+
rule1
|
|
351
|
+
rule2
|
|
352
|
+
rule3
|
|
353
|
+
```
|
|
354
|
+
|
|
355
|
+
### <a id="trimlines"></a> TrimLines
|
|
356
|
+
|
|
357
|
+
This is a very simple transformation that removes leading and trailing spaces/tabs.
|
|
358
|
+
|
|
359
|
+
**Example:**
|
|
360
|
+
|
|
361
|
+
Original list:
|
|
362
|
+
|
|
363
|
+
```
|
|
364
|
+
rule1
|
|
365
|
+
rule2
|
|
366
|
+
rule3
|
|
367
|
+
rule4
|
|
368
|
+
```
|
|
369
|
+
|
|
370
|
+
Here's what we will have after applying this transformation:
|
|
371
|
+
|
|
372
|
+
```
|
|
373
|
+
rule1
|
|
374
|
+
rule2
|
|
375
|
+
rule3
|
|
376
|
+
rule4
|
|
377
|
+
```
|
|
378
|
+
|
|
379
|
+
### <a id="insertfinalnewline"></a> InsertFinalNewLine
|
|
380
|
+
|
|
381
|
+
This is a very simple transformation that inserts a final newline.
|
|
382
|
+
|
|
383
|
+
**Example:**
|
|
384
|
+
|
|
385
|
+
Original list:
|
|
386
|
+
|
|
387
|
+
```
|
|
388
|
+
rule1
|
|
389
|
+
rule2
|
|
390
|
+
rule3
|
|
391
|
+
```
|
|
392
|
+
|
|
393
|
+
Here's what we will have after applying this transformation:
|
|
394
|
+
|
|
395
|
+
```
|
|
396
|
+
rule1
|
|
397
|
+
rule2
|
|
398
|
+
rule3
|
|
399
|
+
|
|
400
|
+
```
|
|
401
|
+
|
|
402
|
+
`RemoveEmptyLines` doesn't delete this empty row due to the execution order.
|
|
403
|
+
|
|
269
404
|
## <a id="how-to-build"></a> How to build
|
|
270
405
|
|
|
271
406
|
- `yarn install` - installs dependencies
|
package/bamboo-specs/bamboo.yaml
CHANGED
package/package.json
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adguard/hostlist-compiler",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.15",
|
|
4
4
|
"description": "A simple tool that compiles hosts blocklists from multiple sources",
|
|
5
5
|
"main": "src/index.js",
|
|
6
|
+
"types": "src/index.d.ts",
|
|
6
7
|
"repository": "https://github.com/AdguardTeam/HostlistCompiler",
|
|
7
8
|
"author": "AdGuard",
|
|
8
9
|
"license": "GPL-3.0",
|
package/src/compile-source.js
CHANGED
|
@@ -17,7 +17,7 @@ const { transform } = require('./transformations/transform');
|
|
|
17
17
|
* Compiles an individual source according to it's configuration.
|
|
18
18
|
*
|
|
19
19
|
* @param {ListSource} source - source configuration.
|
|
20
|
-
* @returns {Array<string
|
|
20
|
+
* @returns {Promise<Array<string>>} array with the source rules
|
|
21
21
|
*/
|
|
22
22
|
async function compileSource(source) {
|
|
23
23
|
consola.info(`Start compiling ${source.source}`);
|
package/src/filter.js
CHANGED
|
@@ -7,7 +7,7 @@ const ruleUtils = require('./rule');
|
|
|
7
7
|
* and return a final array with all the lines from all files.
|
|
8
8
|
*
|
|
9
9
|
* @param {Array<String>} sources - array of URLs.
|
|
10
|
-
* @returns {Array<String
|
|
10
|
+
* @returns {Promise<Array<String>>} array with all non-empty and non-comment lines.
|
|
11
11
|
*/
|
|
12
12
|
async function downloadAll(sources) {
|
|
13
13
|
let list = [];
|
|
@@ -32,7 +32,7 @@ async function downloadAll(sources) {
|
|
|
32
32
|
* @param {Array<String>} rules - array of rules to apply
|
|
33
33
|
* @param {Array<String>} sources - array of rules sources
|
|
34
34
|
* (can be local or remote files)
|
|
35
|
-
* @returns {Array<utils.Wildcard
|
|
35
|
+
* @returns {Promise<Array<utils.Wildcard>>} a list of wildcards to apply
|
|
36
36
|
*/
|
|
37
37
|
async function prepareWildcards(rules, sources) {
|
|
38
38
|
let list = [];
|
package/src/index.d.ts
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
declare module '@adguard/hostlist-compiler' {
|
|
2
|
+
export type SourceType = 'adblock' | 'hosts';
|
|
3
|
+
|
|
4
|
+
export type Transformation =
|
|
5
|
+
'RemoveComments' |
|
|
6
|
+
'Compress' |
|
|
7
|
+
'RemoveModifiers' |
|
|
8
|
+
'Validate' |
|
|
9
|
+
'Deduplicate' |
|
|
10
|
+
'InvertAllow' |
|
|
11
|
+
'RemoveEmptyLines' |
|
|
12
|
+
'TrimLines' |
|
|
13
|
+
'InsertFinalNewLine';
|
|
14
|
+
|
|
15
|
+
/** A source for the filter list */
|
|
16
|
+
export interface ISource {
|
|
17
|
+
/** Name of the source */
|
|
18
|
+
name?: string;
|
|
19
|
+
/** Path to a file or a URL */
|
|
20
|
+
source: string;
|
|
21
|
+
/** Type of the source */
|
|
22
|
+
type?: SourceType;
|
|
23
|
+
/** A list of the transformations that will be applied */
|
|
24
|
+
transformations?: Transformation[];
|
|
25
|
+
/** A list of rules (or wildcards) to exclude from the source. */
|
|
26
|
+
exclusions?: string[];
|
|
27
|
+
/** An array of exclusions sources. */
|
|
28
|
+
exclusions_sources?: string[];
|
|
29
|
+
/** A list of wildcards to include from the source. All rules that don't match these wildcards won't be included. */
|
|
30
|
+
inclusions?: string[];
|
|
31
|
+
/** A list of files with inclusions. */
|
|
32
|
+
inclusions_sources?: string[];
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Configuration for the hostlist compiler */
|
|
36
|
+
export interface IConfiguration {
|
|
37
|
+
/** Filter list name */
|
|
38
|
+
name: string;
|
|
39
|
+
/** Filter list description */
|
|
40
|
+
description?: string;
|
|
41
|
+
/** Filter list homepage */
|
|
42
|
+
homepage?: string;
|
|
43
|
+
/** Filter list license */
|
|
44
|
+
license?: string;
|
|
45
|
+
/** An array of the filter list sources */
|
|
46
|
+
sources: ISource[];
|
|
47
|
+
/** A list of the transformations that will be applied */
|
|
48
|
+
transformations?: Transformation[];
|
|
49
|
+
/** A list of rules (or wildcards) to exclude from the source. */
|
|
50
|
+
exclusions?: string[];
|
|
51
|
+
/** An array of exclusions sources. */
|
|
52
|
+
exclusions_sources?: string[];
|
|
53
|
+
/** A list of wildcards to include from the source. All rules that don't match these wildcards won't be included. */
|
|
54
|
+
inclusions?: string[];
|
|
55
|
+
/** A list of files with inclusions. */
|
|
56
|
+
inclusions_sources?: string[];
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Compiles a filter list using the specified configuration.
|
|
61
|
+
*
|
|
62
|
+
* @param {*} configuration - compilation configuration.
|
|
63
|
+
See the repo README for the details on it.
|
|
64
|
+
* @returns {Promise<Array<string>>} the array of rules.
|
|
65
|
+
*/
|
|
66
|
+
declare async function compile(configuration: IConfiguration): Promise<string[]>;
|
|
67
|
+
|
|
68
|
+
export default compile;
|
|
69
|
+
}
|
package/src/index.js
CHANGED
|
@@ -65,7 +65,7 @@ function prepareSourceHeader(source) {
|
|
|
65
65
|
*
|
|
66
66
|
* @param {*} configuration - compilation configuration.
|
|
67
67
|
See the repo README for the details on it.
|
|
68
|
-
* @returns {Array<string
|
|
68
|
+
* @returns {Promise<Array<string>>} the array of rules.
|
|
69
69
|
*/
|
|
70
70
|
async function compile(configuration) {
|
|
71
71
|
consola.info('Starting the compiler');
|
|
@@ -41,7 +41,10 @@
|
|
|
41
41
|
"Compress",
|
|
42
42
|
"Validate",
|
|
43
43
|
"Deduplicate",
|
|
44
|
-
"InvertAllow"
|
|
44
|
+
"InvertAllow",
|
|
45
|
+
"RemoveEmptyLines",
|
|
46
|
+
"TrimLines",
|
|
47
|
+
"InsertFinalNewLine"
|
|
45
48
|
]
|
|
46
49
|
}
|
|
47
50
|
},
|
|
@@ -113,7 +116,10 @@
|
|
|
113
116
|
"Compress",
|
|
114
117
|
"Validate",
|
|
115
118
|
"Deduplicate",
|
|
116
|
-
"InvertAllow"
|
|
119
|
+
"InvertAllow",
|
|
120
|
+
"RemoveEmptyLines",
|
|
121
|
+
"TrimLines",
|
|
122
|
+
"InsertFinalNewLine"
|
|
117
123
|
]
|
|
118
124
|
}
|
|
119
125
|
},
|
|
@@ -9,7 +9,7 @@ const filterUtils = require('../filter');
|
|
|
9
9
|
* @param {Array<String>} exclusions - array of exclusions to apply
|
|
10
10
|
* @param {Array<String>} exclusionsSources - array of exclusion sources
|
|
11
11
|
* (can be a local or remote file)
|
|
12
|
-
* @returns {Array<String
|
|
12
|
+
* @returns {Promise<Array<String>>} filtered array of rules
|
|
13
13
|
*/
|
|
14
14
|
async function exclude(rules, exclusions, exclusionsSources) {
|
|
15
15
|
if (_.isEmpty(exclusions) && _.isEmpty(exclusionsSources)) {
|
|
@@ -9,7 +9,7 @@ const filterUtils = require('../filter');
|
|
|
9
9
|
* @param {Array<String>} inclusions - array of inclusions to apply
|
|
10
10
|
* @param {Array<String>} inclusionsSources - array of inclusions' sources
|
|
11
11
|
* (can be a local or remote file)
|
|
12
|
-
* @returns {Array<String
|
|
12
|
+
* @returns {Promise<Array<String>>} filtered array of rules
|
|
13
13
|
*/
|
|
14
14
|
async function include(rules, inclusions, inclusionsSources) {
|
|
15
15
|
if (_.isEmpty(inclusions) && _.isEmpty(inclusionsSources)) {
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
const consola = require('consola');
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* This is a very simple transformation that inserts a final newline.
|
|
5
|
+
*
|
|
6
|
+
* @param {Array<string>} lines - lines to transform
|
|
7
|
+
* @returns {Array<string>} filtered lines
|
|
8
|
+
*/
|
|
9
|
+
function insertFinalNewLine(lines) {
|
|
10
|
+
if (lines.length === 0 || (lines.length > 0 && lines[lines.length - 1].trim() !== '')) {
|
|
11
|
+
lines.push('');
|
|
12
|
+
consola.info('Final newline inserted');
|
|
13
|
+
}
|
|
14
|
+
return lines;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
module.exports = insertFinalNewLine;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
const consola = require('consola');
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* This is a very simple transformation that removes empty lines.
|
|
5
|
+
*
|
|
6
|
+
* @param {Array<string>} lines - lines to transform
|
|
7
|
+
* @returns {Array<string>} filtered lines
|
|
8
|
+
*/
|
|
9
|
+
function removeEmptyLines(lines) {
|
|
10
|
+
const filtered = lines.filter((line) => line.trim().length);
|
|
11
|
+
consola.info(`Removed ${lines.length - filtered.length} empty lines`);
|
|
12
|
+
return filtered;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
module.exports = removeEmptyLines;
|
|
@@ -4,7 +4,7 @@ const ruleUtils = require('../rule');
|
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* This transformation simply removes the following modifiers from the adblock-style rules:
|
|
7
|
-
* $document, $all, $third-party, $3p, $popup
|
|
7
|
+
* $document, $doc, $all, $third-party, $3p, $popup
|
|
8
8
|
*
|
|
9
9
|
* @param {Array<string>} rules - rules to transform
|
|
10
10
|
* @returns {Array<string>} filtered rules
|
|
@@ -32,6 +32,7 @@ function removeModifiers(rules) {
|
|
|
32
32
|
modified = ruleUtils.removeModifier(props, '3p') || modified;
|
|
33
33
|
modified = ruleUtils.removeModifier(props, 'all') || modified;
|
|
34
34
|
modified = ruleUtils.removeModifier(props, 'document') || modified;
|
|
35
|
+
modified = ruleUtils.removeModifier(props, 'doc') || modified;
|
|
35
36
|
modified = ruleUtils.removeModifier(props, 'popup') || modified;
|
|
36
37
|
filtered.push(ruleUtils.adblockRuleToString(props));
|
|
37
38
|
|
|
@@ -6,6 +6,9 @@ const include = require('./include');
|
|
|
6
6
|
const deduplicate = require('./deduplicate');
|
|
7
7
|
const compress = require('./compress');
|
|
8
8
|
const invertAllow = require('./invertallow');
|
|
9
|
+
const removeEmptyLines = require('./remove-empty-lines');
|
|
10
|
+
const trimLines = require('./trim-lines');
|
|
11
|
+
const insertFinalNewLine = require('./insert-final-newline');
|
|
9
12
|
|
|
10
13
|
/**
|
|
11
14
|
* Enum with all available transformations
|
|
@@ -17,6 +20,9 @@ const TRANSFORMATIONS = Object.freeze({
|
|
|
17
20
|
Validate: 'Validate',
|
|
18
21
|
Deduplicate: 'Deduplicate',
|
|
19
22
|
InvertAllow: 'InvertAllow',
|
|
23
|
+
RemoveEmptyLines: 'RemoveEmptyLines',
|
|
24
|
+
TrimLines: 'TrimLines',
|
|
25
|
+
InsertFinalNewLine: 'InsertFinalNewLine',
|
|
20
26
|
});
|
|
21
27
|
|
|
22
28
|
/**
|
|
@@ -25,7 +31,7 @@ const TRANSFORMATIONS = Object.freeze({
|
|
|
25
31
|
* @param {Array<string>} rules - rules to transform
|
|
26
32
|
* @param {*} configuration - transformation configuration.
|
|
27
33
|
* @param {Array<string>} transformations - a list of transformations to apply to the rules.
|
|
28
|
-
* @returns {Array<string
|
|
34
|
+
* @returns {Promise<Array<string>>} rules after applying all transformations.
|
|
29
35
|
*/
|
|
30
36
|
async function transform(rules, configuration, transformations) {
|
|
31
37
|
// If none specified -- apply all transformationss
|
|
@@ -41,6 +47,9 @@ async function transform(rules, configuration, transformations) {
|
|
|
41
47
|
transformed = await include(transformed, configuration.inclusions,
|
|
42
48
|
configuration.inclusions_sources);
|
|
43
49
|
|
|
50
|
+
if (transformations.indexOf(TRANSFORMATIONS.TrimLines) !== -1) {
|
|
51
|
+
transformed = trimLines(transformed);
|
|
52
|
+
}
|
|
44
53
|
if (transformations.indexOf(TRANSFORMATIONS.RemoveComments) !== -1) {
|
|
45
54
|
transformed = removeComments(transformed);
|
|
46
55
|
}
|
|
@@ -59,6 +68,12 @@ async function transform(rules, configuration, transformations) {
|
|
|
59
68
|
if (transformations.indexOf(TRANSFORMATIONS.Deduplicate) !== -1) {
|
|
60
69
|
transformed = deduplicate(transformed);
|
|
61
70
|
}
|
|
71
|
+
if (transformations.indexOf(TRANSFORMATIONS.RemoveEmptyLines) !== -1) {
|
|
72
|
+
transformed = removeEmptyLines(transformed);
|
|
73
|
+
}
|
|
74
|
+
if (transformations.indexOf(TRANSFORMATIONS.InsertFinalNewLine) !== -1) {
|
|
75
|
+
transformed = insertFinalNewLine(transformed);
|
|
76
|
+
}
|
|
62
77
|
return transformed;
|
|
63
78
|
}
|
|
64
79
|
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
const consola = require('consola');
|
|
2
|
+
const _ = require('lodash');
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* This is a very simple transformation that removes leading and trailing spaces/tabs.
|
|
6
|
+
*
|
|
7
|
+
* @param {Array<string>} lines - lines/rules to transform
|
|
8
|
+
* @returns {Array<string>} filtered lines/rules
|
|
9
|
+
*/
|
|
10
|
+
function trimLines(lines) {
|
|
11
|
+
const transformed = lines.map((line) => _.trim(line, ' \t'));
|
|
12
|
+
consola.info('Lines trimmed.');
|
|
13
|
+
return transformed;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
module.exports = trimLines;
|
|
@@ -153,7 +153,9 @@ function validAdblockRule(ruleText) {
|
|
|
153
153
|
}
|
|
154
154
|
|
|
155
155
|
// If there's something after ^ in the pattern - something went wrong
|
|
156
|
-
|
|
156
|
+
// unless it's `^|` which is a rather often case
|
|
157
|
+
if (props.pattern.length > (sepIdx + 1)
|
|
158
|
+
&& props.pattern[sepIdx + 1] !== '|') {
|
|
157
159
|
return false;
|
|
158
160
|
}
|
|
159
161
|
}
|
package/src/utils.js
CHANGED
|
@@ -16,7 +16,7 @@ function isURL(str) {
|
|
|
16
16
|
* Downloads (or reads from the disk) the specified source
|
|
17
17
|
*
|
|
18
18
|
* @param {*} urlOrPath url or path to a file
|
|
19
|
-
* @returns {String} contents of the files
|
|
19
|
+
* @returns {Promise<String>} contents of the files
|
|
20
20
|
*/
|
|
21
21
|
async function download(urlOrPath) {
|
|
22
22
|
let str = '';
|
|
@@ -85,4 +85,40 @@ describe('Source compiler', () => {
|
|
|
85
85
|
// Make sure scope URLs were requested
|
|
86
86
|
scope.done();
|
|
87
87
|
});
|
|
88
|
+
|
|
89
|
+
it('compile the source with transformations', async () => {
|
|
90
|
+
const testList = `
|
|
91
|
+
|
|
92
|
+
example.org
|
|
93
|
+
|
|
94
|
+
test1.com
|
|
95
|
+
|
|
96
|
+
! comment
|
|
97
|
+
test1.com
|
|
98
|
+
|
|
99
|
+
test2.com`;
|
|
100
|
+
const scope = nock('https://example.org')
|
|
101
|
+
.get('/test-filter.txt')
|
|
102
|
+
.reply(200, testList);
|
|
103
|
+
|
|
104
|
+
const source = {
|
|
105
|
+
name: 'test source',
|
|
106
|
+
source: 'https://example.org/test-filter.txt',
|
|
107
|
+
transformations: [
|
|
108
|
+
TRANSFORMATIONS.Deduplicate,
|
|
109
|
+
TRANSFORMATIONS.RemoveComments,
|
|
110
|
+
TRANSFORMATIONS.TrimLines,
|
|
111
|
+
TRANSFORMATIONS.RemoveEmptyLines,
|
|
112
|
+
TRANSFORMATIONS.InsertFinalNewLine,
|
|
113
|
+
],
|
|
114
|
+
};
|
|
115
|
+
const compiled = await compileSource(source);
|
|
116
|
+
expect(compiled).toEqual([
|
|
117
|
+
'example.org',
|
|
118
|
+
'test1.com',
|
|
119
|
+
'test2.com',
|
|
120
|
+
'',
|
|
121
|
+
]);
|
|
122
|
+
scope.done();
|
|
123
|
+
});
|
|
88
124
|
});
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
const insertFinalNewLine = require('../../src/transformations/insert-final-newline');
|
|
2
|
+
|
|
3
|
+
describe('Insert final new line', () => {
|
|
4
|
+
it('test with empty file', () => {
|
|
5
|
+
const rules = [];
|
|
6
|
+
const filtered = insertFinalNewLine(rules);
|
|
7
|
+
expect(filtered).toEqual(['']);
|
|
8
|
+
});
|
|
9
|
+
it('test with one empty line', () => {
|
|
10
|
+
const rules = [''];
|
|
11
|
+
const filtered = insertFinalNewLine(rules);
|
|
12
|
+
expect(filtered).toEqual(['']);
|
|
13
|
+
});
|
|
14
|
+
it('test with one rule and one empty lines', () => {
|
|
15
|
+
const rules = [
|
|
16
|
+
'rule1',
|
|
17
|
+
'',
|
|
18
|
+
];
|
|
19
|
+
const filtered = insertFinalNewLine(rules);
|
|
20
|
+
expect(filtered).toEqual([
|
|
21
|
+
'rule1',
|
|
22
|
+
'',
|
|
23
|
+
]);
|
|
24
|
+
});
|
|
25
|
+
it('test with many empty lines', () => {
|
|
26
|
+
const rules = [
|
|
27
|
+
'rule1',
|
|
28
|
+
'',
|
|
29
|
+
'',
|
|
30
|
+
];
|
|
31
|
+
const filtered = insertFinalNewLine(rules);
|
|
32
|
+
expect(filtered).toEqual([
|
|
33
|
+
'rule1',
|
|
34
|
+
'',
|
|
35
|
+
'',
|
|
36
|
+
]);
|
|
37
|
+
});
|
|
38
|
+
it('test with one rule', () => {
|
|
39
|
+
const rules = [
|
|
40
|
+
'rule1',
|
|
41
|
+
];
|
|
42
|
+
const filtered = insertFinalNewLine(rules);
|
|
43
|
+
expect(filtered).toEqual([
|
|
44
|
+
'rule1',
|
|
45
|
+
'',
|
|
46
|
+
]);
|
|
47
|
+
});
|
|
48
|
+
});
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
const removeEmptyLines = require('../../src/transformations/remove-empty-lines');
|
|
2
|
+
|
|
3
|
+
describe('Remove empty lines', () => {
|
|
4
|
+
it('test with no empty lines', () => {
|
|
5
|
+
const rules = [
|
|
6
|
+
'rule1',
|
|
7
|
+
'rule2',
|
|
8
|
+
];
|
|
9
|
+
const filtered = removeEmptyLines(rules);
|
|
10
|
+
expect(filtered).toEqual([
|
|
11
|
+
'rule1',
|
|
12
|
+
'rule2',
|
|
13
|
+
]);
|
|
14
|
+
});
|
|
15
|
+
it('test with one empty line at the end', () => {
|
|
16
|
+
const rules = [
|
|
17
|
+
'rule1',
|
|
18
|
+
'rule2',
|
|
19
|
+
'',
|
|
20
|
+
];
|
|
21
|
+
const filtered = removeEmptyLines(rules);
|
|
22
|
+
expect(filtered).toEqual([
|
|
23
|
+
'rule1',
|
|
24
|
+
'rule2',
|
|
25
|
+
]);
|
|
26
|
+
});
|
|
27
|
+
it('test with one empty line between the rules', () => {
|
|
28
|
+
const rules = [
|
|
29
|
+
'rule1',
|
|
30
|
+
'',
|
|
31
|
+
'rule2',
|
|
32
|
+
];
|
|
33
|
+
const filtered = removeEmptyLines(rules);
|
|
34
|
+
expect(filtered).toEqual([
|
|
35
|
+
'rule1',
|
|
36
|
+
'rule2',
|
|
37
|
+
]);
|
|
38
|
+
});
|
|
39
|
+
it('test with two empty line', () => {
|
|
40
|
+
const rules = [
|
|
41
|
+
'rule1',
|
|
42
|
+
'',
|
|
43
|
+
'',
|
|
44
|
+
'rule2',
|
|
45
|
+
];
|
|
46
|
+
const filtered = removeEmptyLines(rules);
|
|
47
|
+
expect(filtered).toEqual([
|
|
48
|
+
'rule1',
|
|
49
|
+
'rule2',
|
|
50
|
+
]);
|
|
51
|
+
});
|
|
52
|
+
it('test with comments, empty lines', () => {
|
|
53
|
+
const rules = [
|
|
54
|
+
'! aaa',
|
|
55
|
+
'rule1',
|
|
56
|
+
'',
|
|
57
|
+
'! bbb',
|
|
58
|
+
'',
|
|
59
|
+
'',
|
|
60
|
+
'rule2',
|
|
61
|
+
'',
|
|
62
|
+
'',
|
|
63
|
+
'',
|
|
64
|
+
'!ccc',
|
|
65
|
+
'',
|
|
66
|
+
'',
|
|
67
|
+
];
|
|
68
|
+
const filtered = removeEmptyLines(rules);
|
|
69
|
+
expect(filtered).toEqual([
|
|
70
|
+
'! aaa',
|
|
71
|
+
'rule1',
|
|
72
|
+
'! bbb',
|
|
73
|
+
'rule2',
|
|
74
|
+
'!ccc',
|
|
75
|
+
]);
|
|
76
|
+
});
|
|
77
|
+
});
|
|
@@ -7,17 +7,19 @@ describe('Strip third-party', () => {
|
|
|
7
7
|
||example.net$domain=ya.ru,3p
|
|
8
8
|
||islandofadvert.com^$document,popup
|
|
9
9
|
||example.com$document
|
|
10
|
+
||example.com$doc
|
|
10
11
|
||example.com$all
|
|
11
12
|
||example.org^`.split(/\r?\n/);
|
|
12
13
|
const filtered = removeModifiers(rules);
|
|
13
14
|
|
|
14
|
-
expect(filtered).toHaveLength(
|
|
15
|
+
expect(filtered).toHaveLength(8);
|
|
15
16
|
expect(filtered[0]).toBe('! test comment');
|
|
16
17
|
expect(filtered[1]).toBe('||example.org$important');
|
|
17
18
|
expect(filtered[2]).toBe('||example.net$domain=ya.ru');
|
|
18
19
|
expect(filtered[3]).toBe('||islandofadvert.com^');
|
|
19
20
|
expect(filtered[4]).toBe('||example.com');
|
|
20
21
|
expect(filtered[5]).toBe('||example.com');
|
|
21
|
-
expect(filtered[6]).toBe('||example.
|
|
22
|
+
expect(filtered[6]).toBe('||example.com');
|
|
23
|
+
expect(filtered[7]).toBe('||example.org^');
|
|
22
24
|
});
|
|
23
25
|
});
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
const _ = require('lodash');
|
|
2
|
+
const trimLines = require('../../src/transformations/trim-lines');
|
|
3
|
+
|
|
4
|
+
describe('Trim lines', () => {
|
|
5
|
+
it('test with empty file', () => {
|
|
6
|
+
const rules = [];
|
|
7
|
+
const filtered = trimLines(rules);
|
|
8
|
+
expect(filtered).toEqual([]);
|
|
9
|
+
});
|
|
10
|
+
it('test with one empty line', () => {
|
|
11
|
+
const rules = [''];
|
|
12
|
+
const filtered = trimLines(rules);
|
|
13
|
+
expect(filtered).toEqual(['']);
|
|
14
|
+
});
|
|
15
|
+
it('test with three rules', () => {
|
|
16
|
+
const rules = [
|
|
17
|
+
'rule1',
|
|
18
|
+
' rule2 ',
|
|
19
|
+
'',
|
|
20
|
+
' rule3 ',
|
|
21
|
+
];
|
|
22
|
+
const filtered = trimLines(rules);
|
|
23
|
+
expect(filtered).toEqual([
|
|
24
|
+
'rule1',
|
|
25
|
+
'rule2',
|
|
26
|
+
'',
|
|
27
|
+
'rule3',
|
|
28
|
+
]);
|
|
29
|
+
});
|
|
30
|
+
it('test with three rules and comments', () => {
|
|
31
|
+
const rules = [
|
|
32
|
+
'rule1',
|
|
33
|
+
' rule2 ',
|
|
34
|
+
'',
|
|
35
|
+
'',
|
|
36
|
+
'! comment ',
|
|
37
|
+
' rule3 ',
|
|
38
|
+
' ! comment multiple words ',
|
|
39
|
+
'',
|
|
40
|
+
];
|
|
41
|
+
const filtered = trimLines(rules);
|
|
42
|
+
expect(filtered).toEqual([
|
|
43
|
+
'rule1',
|
|
44
|
+
'rule2',
|
|
45
|
+
'',
|
|
46
|
+
'',
|
|
47
|
+
'! comment',
|
|
48
|
+
'rule3',
|
|
49
|
+
'! comment multiple words',
|
|
50
|
+
'',
|
|
51
|
+
]);
|
|
52
|
+
});
|
|
53
|
+
it('test with three rules and comments', () => {
|
|
54
|
+
const rules = [
|
|
55
|
+
_.repeat(' ', _.random(1, 10)), // ' ', ' ', ..., ' ' etc.
|
|
56
|
+
' ! comment multiple words , ',
|
|
57
|
+
'',
|
|
58
|
+
];
|
|
59
|
+
const filtered = trimLines(rules);
|
|
60
|
+
expect(filtered).toEqual([
|
|
61
|
+
'',
|
|
62
|
+
'! comment multiple words ,',
|
|
63
|
+
'',
|
|
64
|
+
]);
|
|
65
|
+
});
|
|
66
|
+
});
|
|
@@ -45,6 +45,8 @@ describe('Validate', () => {
|
|
|
45
45
|
||example.org^$third-party
|
|
46
46
|
||example.org^$important
|
|
47
47
|
://example.org
|
|
48
|
+
||example.org^|
|
|
49
|
+
@@||example.org^|$important
|
|
48
50
|
@@||example.com^*-tracking.js
|
|
49
51
|
@@||example.com^-tracking.js`.split(/\r?\n/);
|
|
50
52
|
const filtered = validate(rules);
|
|
@@ -56,6 +58,8 @@ describe('Validate', () => {
|
|
|
56
58
|
'||ex*.org^', // valid because contains special characters
|
|
57
59
|
'||example.org^$important',
|
|
58
60
|
'://example.org',
|
|
61
|
+
'||example.org^|',
|
|
62
|
+
'@@||example.org^|$important',
|
|
59
63
|
]);
|
|
60
64
|
});
|
|
61
65
|
});
|