@adguard/hostlist-compiler 1.0.12 → 1.0.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  [![NPM](https://nodei.co/npm/@adguard/hostlist-compiler.png?compact=true)](https://www.npmjs.com/package/@adguard/hostlist-compiler/)
4
4
 
5
- This is a simple tool that makes it easier to compile a [hosts blocklist](https://github.com/AdguardTeam/AdGuardHome/wiki/Hosts-Blocklists) compatible with AdGuard Home or any other AdGuard product with **DNS filtering**.
5
+ This is a simple tool that makes it easier to compile a [hosts blocklist](https://adguard-dns.io/kb/general/dns-filtering-syntax/) compatible with AdGuard Home or any other AdGuard product with **DNS filtering**.
6
6
 
7
7
  - [Usage](#usage)
8
8
  - [Configuration](#configuration)
@@ -15,6 +15,9 @@ This is a simple tool that makes it easier to compile a [hosts blocklist](https:
15
15
  - [Validate](#validate)
16
16
  - [Deduplicate](#deduplicate)
17
17
  - [InvertAllow](#invertallow)
18
+ - [RemoveEmptyLines](#removeemptylines)
19
+ - [TrimLines](#trimlines)
20
+ - [InsertFinalNewLine](#insertfinalnewline)
18
21
  - [How to build](#how-to-build)
19
22
 
20
23
  ## <a id="usage"></a> Usage
@@ -25,6 +28,18 @@ First of all, install the `hostlist-compiler`:
25
28
  npm i -g @adguard/hostlist-compiler
26
29
  ```
27
30
 
31
+ After that you have two options.
32
+
33
+ **Quick hosts conversion**
34
+
35
+ Convert and compress a `/etc/hosts`-syntax blocklist to [AdGuard syntax](https://adguard-dns.io/kb/general/dns-filtering-syntax/).
36
+
37
+ ```
38
+ hostlist-compiler -i hosts.txt -i hosts2.txt -o output.txt
39
+ ```
40
+
41
+ **Build a configurable blocklist from multiple sources**
42
+
28
43
  Prepare the list configuration (read more about that below) and run the compiler:
29
44
 
30
45
  ```bash
@@ -131,28 +146,83 @@ Examples:
131
146
 
132
147
  ### <a id="api"></a> API
133
148
 
134
- ```
135
- npm i @adguard/hostlist-compiler
136
- ```
149
+ Install: `npm i @adguard/hostlist-compiler` or `yarn add @adguard/hostlist-compiler`
150
+
151
+ #### JavaScript example:
137
152
 
138
153
  ```javascript
139
154
  const compile = require("@adguard/hostlist-compiler");
140
155
 
141
- const configuration = {
142
- name: "test list",
143
- sources: [
144
- {
145
- source:
146
- "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt",
147
- },
148
- ],
149
- };
156
+ ;(async () => {
157
+ // Compile filters
158
+ const result = await compile({
159
+ name: 'Your Hostlist',
160
+ sources: [
161
+ {
162
+ type: 'adblock',
163
+ source: 'https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt', // or local file
164
+ transformations: ['RemoveComments', 'Validate'],
165
+ },
166
+ ],
167
+ transformations: ['Deduplicate'],
168
+ });
169
+
170
+ // Write to file
171
+ writeFileSync('your-hostlist.txt', result.join('\n'));
172
+ })();
173
+ ```
150
174
 
151
- async function main() {
152
- const compiled = compile(configuration);
153
- }
175
+ #### TypeScript example:
176
+
177
+ ```typescript
178
+ import compile from '@adguard/hostlist-compiler';
179
+ import { writeFileSync } from 'fs';
180
+
181
+ ;(async () => {
182
+ // Compile filters
183
+ const result = await compile({
184
+ name: 'Your Hostlist',
185
+ sources: [
186
+ {
187
+ type: 'adblock',
188
+ source: 'https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt',
189
+ transformations: ['RemoveComments', 'Validate'],
190
+ },
191
+ ],
192
+ transformations: ['Deduplicate'],
193
+ });
194
+
195
+ // Write to file
196
+ writeFileSync('your-hostlist.txt', result.join('\n'));
197
+ })();
198
+ ```
154
199
 
155
- main();
200
+ or:
201
+
202
+ ```typescript
203
+ import HostlistCompiler, { IConfiguration as HostlistCompilerConfiguration } from '@adguard/hostlist-compiler';
204
+ import { writeFileSync } from 'fs';
205
+
206
+ ;(async () => {
207
+ // Configuration
208
+ const config: HostlistCompilerConfiguration = {
209
+ name: 'Your Hostlist',
210
+ sources: [
211
+ {
212
+ type: 'adblock',
213
+ source: 'https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt',
214
+ transformations: ['RemoveComments', 'Validate'],
215
+ },
216
+ ],
217
+ transformations: ['Deduplicate'],
218
+ };
219
+
220
+ // Compile filters
221
+ const result = await HostlistCompiler(config);
222
+
223
+ // Write to file
224
+ writeFileSync('your-hostlist.txt', result.join('\n'));
225
+ })();
156
226
  ```
157
227
 
158
228
  ## <a id="transformations"></a> Transformations
@@ -165,6 +235,9 @@ Here is the full list of transformations that are available:
165
235
  4. `Validate`
166
236
  5. `Deduplicate`
167
237
  6. `InvertAllow`
238
+ 7. `RemoveEmptyLines`
239
+ 8. `TrimLines`
240
+ 9. `InsertFinalNewLine`
168
241
 
169
242
  Please note that these transformations are are always applied in the order specified here.
170
243
 
@@ -188,7 +261,7 @@ By default, [AdGuard Home](https://github.com/AdguardTeam/AdGuardHome) will igno
188
261
  Here is the list of modifiers that will be removed:
189
262
 
190
263
  - `$third-party` and `$3p` modifiers
191
- - `$document` modifier
264
+ - `$document` and `$doc` modifiers
192
265
  - `$all` modifier
193
266
  - `$popup` modifier
194
267
 
@@ -266,6 +339,80 @@ Here's what we will have after applying this transformation:
266
339
  @@rule2
267
340
  ```
268
341
 
342
+ ### <a id="removeemptylines"></a> RemoveEmptyLines
343
+
344
+ This is a very simple transformation that removes empty lines.
345
+
346
+ **Example:**
347
+
348
+ Original list:
349
+
350
+ ```
351
+ rule1
352
+
353
+ rule2
354
+
355
+
356
+ rule3
357
+ ```
358
+
359
+ Here's what we will have after applying this transformation:
360
+
361
+ ```
362
+ rule1
363
+ rule2
364
+ rule3
365
+ ```
366
+
367
+ ### <a id="trimlines"></a> TrimLines
368
+
369
+ This is a very simple transformation that removes leading and trailing spaces/tabs.
370
+
371
+ **Example:**
372
+
373
+ Original list:
374
+
375
+ ```
376
+ rule1
377
+ rule2
378
+ rule3
379
+ rule4
380
+ ```
381
+
382
+ Here's what we will have after applying this transformation:
383
+
384
+ ```
385
+ rule1
386
+ rule2
387
+ rule3
388
+ rule4
389
+ ```
390
+
391
+ ### <a id="insertfinalnewline"></a> InsertFinalNewLine
392
+
393
+ This is a very simple transformation that inserts a final newline.
394
+
395
+ **Example:**
396
+
397
+ Original list:
398
+
399
+ ```
400
+ rule1
401
+ rule2
402
+ rule3
403
+ ```
404
+
405
+ Here's what we will have after applying this transformation:
406
+
407
+ ```
408
+ rule1
409
+ rule2
410
+ rule3
411
+
412
+ ```
413
+
414
+ `RemoveEmptyLines` doesn't delete this empty row due to the execution order.
415
+
269
416
  ## <a id="how-to-build"></a> How to build
270
417
 
271
418
  - `yarn install` - installs dependencies
@@ -32,7 +32,7 @@ Test:
32
32
  yarn install
33
33
  yarn lint
34
34
  yarn test
35
-
35
+ rm -rf node_modules
36
36
  final-tasks:
37
37
  - clean
38
38
  requirements:
package/package.json CHANGED
@@ -1,8 +1,9 @@
1
1
  {
2
2
  "name": "@adguard/hostlist-compiler",
3
- "version": "1.0.12",
3
+ "version": "1.0.16",
4
4
  "description": "A simple tool that compiles hosts blocklists from multiple sources",
5
5
  "main": "src/index.js",
6
+ "types": "src/index.d.ts",
6
7
  "repository": "https://github.com/AdguardTeam/HostlistCompiler",
7
8
  "author": "AdGuard",
8
9
  "license": "GPL-3.0",
package/src/cli.js CHANGED
@@ -8,13 +8,32 @@ const packageJson = require('../package.json');
8
8
  // eslint-disable-next-line import/order
9
9
  const { argv } = require('yargs')
10
10
  .usage('Usage: $0 [options]')
11
- .example('$0 -c config.json -o output.txt', 'compile a blocklist and write the output to output.txt')
11
+ .example(
12
+ '$0 -c config.json -o output.txt',
13
+ 'compile a blocklist and write the output to output.txt',
14
+ )
15
+ .example(
16
+ '$0 -i https://example.org/hosts.txt -o output.txt',
17
+ 'compile a blocklist from the URL and write the output to output.txt',
18
+ )
12
19
  .option('config', {
13
20
  alias: 'c',
14
21
  type: 'string',
15
22
  description: 'Path to the compiler configuration file',
16
23
  nargs: 1,
17
24
  })
25
+ .option('input', {
26
+ alias: 'i',
27
+ array: true,
28
+ type: 'string',
29
+ description: 'URL (or path to a file) to convert to an AdGuard-syntax blocklist. Can be specified multiple times.',
30
+ })
31
+ .option('input-type', {
32
+ alias: 't',
33
+ type: 'string',
34
+ description: 'Type of the input file (/etc/hosts, adguard)',
35
+ nargs: 1,
36
+ })
18
37
  .option('output', {
19
38
  alias: 'o',
20
39
  type: 'string',
@@ -26,25 +45,77 @@ const { argv } = require('yargs')
26
45
  type: 'boolean',
27
46
  description: 'Run with verbose logging',
28
47
  })
29
- .demandOption(['c', 'o'])
48
+ .demandOption(['o'])
30
49
  .version()
31
50
  .help('h')
32
51
  .alias('h', 'help');
33
52
 
34
53
  if (argv.verbose) {
35
- // trace level
54
+ // trace level.
36
55
  consola.level = 5;
37
56
  }
38
57
 
39
58
  consola.info(`Starting ${packageJson.name} v${packageJson.version}`);
40
59
 
60
+ /**
61
+ * Reads the configuration file and returns the parsed JSON.
62
+ *
63
+ * @returns {Promise<Object> | Promise<Error>} configuration file as JSON.
64
+ */
65
+ async function readConfig() {
66
+ consola.debug(`Reading configuration from ${argv.config}`);
67
+
68
+ // Check if the configuration file exists.
69
+ await fs.access(argv.config);
70
+ const configStr = (await fs.readFile(argv.config)).toString();
71
+
72
+ return JSON.parse(configStr);
73
+ }
74
+
75
+ /**
76
+ * Creates a configuration object for the specified blocklist source.
77
+ * We assume that the input is a /etc/hosts file.
78
+ *
79
+ * @returns {Object} configuration object.
80
+ */
81
+ function createConfig() {
82
+ const inputType = argv['input-type'] || 'hosts';
83
+
84
+ consola.debug(`Creating configuration for input ${argv.input} of type ${inputType}`);
85
+
86
+ const config = {
87
+ name: 'Blocklist',
88
+ sources: [],
89
+ transformations: [
90
+ 'RemoveComments',
91
+ 'Deduplicate',
92
+ 'Compress',
93
+ 'Validate',
94
+ 'TrimLines',
95
+ 'InsertFinalNewLine',
96
+ ],
97
+ };
98
+
99
+ argv.input.forEach((input) => {
100
+ config.sources.push({
101
+ source: input,
102
+ type: inputType,
103
+ });
104
+ });
105
+
106
+ return config;
107
+ }
108
+
41
109
  async function main() {
42
110
  try {
43
- // Check if file exists
44
- await fs.access(argv.config);
111
+ if (!argv.input && !argv.config) {
112
+ throw new Error('Either --input or --config must be specified');
113
+ }
114
+
115
+ const config = argv.input ? createConfig() : await readConfig();
116
+
117
+ consola.debug(`Configuration: ${JSON.stringify(config, 0, 4)}`);
45
118
 
46
- const configStr = (await fs.readFile(argv.config)).toString();
47
- const config = JSON.parse(configStr);
48
119
  const lines = await compile(config);
49
120
 
50
121
  consola.info(`Writing output to ${argv.output}`);
@@ -17,7 +17,7 @@ const { transform } = require('./transformations/transform');
17
17
  * Compiles an individual source according to it's configuration.
18
18
  *
19
19
  * @param {ListSource} source - source configuration.
20
- * @returns {Array<string>} array with the source rules
20
+ * @returns {Promise<Array<string>>} array with the source rules
21
21
  */
22
22
  async function compileSource(source) {
23
23
  consola.info(`Start compiling ${source.source}`);
package/src/filter.js CHANGED
@@ -7,7 +7,7 @@ const ruleUtils = require('./rule');
7
7
  * and return a final array with all the lines from all files.
8
8
  *
9
9
  * @param {Array<String>} sources - array of URLs.
10
- * @returns {Array<String>} array with all non-empty and non-comment lines.
10
+ * @returns {Promise<Array<String>>} array with all non-empty and non-comment lines.
11
11
  */
12
12
  async function downloadAll(sources) {
13
13
  let list = [];
@@ -32,7 +32,7 @@ async function downloadAll(sources) {
32
32
  * @param {Array<String>} rules - array of rules to apply
33
33
  * @param {Array<String>} sources - array of rules sources
34
34
  * (can be local or remote files)
35
- * @returns {Array<utils.Wildcard>} a list of wildcards to apply
35
+ * @returns {Promise<Array<utils.Wildcard>>} a list of wildcards to apply
36
36
  */
37
37
  async function prepareWildcards(rules, sources) {
38
38
  let list = [];
package/src/index.d.ts ADDED
@@ -0,0 +1,69 @@
1
+ declare module '@adguard/hostlist-compiler' {
2
+ export type SourceType = 'adblock' | 'hosts';
3
+
4
+ export type Transformation =
5
+ 'RemoveComments' |
6
+ 'Compress' |
7
+ 'RemoveModifiers' |
8
+ 'Validate' |
9
+ 'Deduplicate' |
10
+ 'InvertAllow' |
11
+ 'RemoveEmptyLines' |
12
+ 'TrimLines' |
13
+ 'InsertFinalNewLine';
14
+
15
+ /** A source for the filter list */
16
+ export interface ISource {
17
+ /** Name of the source */
18
+ name?: string;
19
+ /** Path to a file or a URL */
20
+ source: string;
21
+ /** Type of the source */
22
+ type?: SourceType;
23
+ /** A list of the transformations that will be applied */
24
+ transformations?: Transformation[];
25
+ /** A list of rules (or wildcards) to exclude from the source. */
26
+ exclusions?: string[];
27
+ /** An array of exclusions sources. */
28
+ exclusions_sources?: string[];
29
+ /** A list of wildcards to include from the source. All rules that don't match these wildcards won't be included. */
30
+ inclusions?: string[];
31
+ /** A list of files with inclusions. */
32
+ inclusions_sources?: string[];
33
+ }
34
+
35
+ /** Configuration for the hostlist compiler */
36
+ export interface IConfiguration {
37
+ /** Filter list name */
38
+ name: string;
39
+ /** Filter list description */
40
+ description?: string;
41
+ /** Filter list homepage */
42
+ homepage?: string;
43
+ /** Filter list license */
44
+ license?: string;
45
+ /** An array of the filter list sources */
46
+ sources: ISource[];
47
+ /** A list of the transformations that will be applied */
48
+ transformations?: Transformation[];
49
+ /** A list of rules (or wildcards) to exclude from the source. */
50
+ exclusions?: string[];
51
+ /** An array of exclusions sources. */
52
+ exclusions_sources?: string[];
53
+ /** A list of wildcards to include from the source. All rules that don't match these wildcards won't be included. */
54
+ inclusions?: string[];
55
+ /** A list of files with inclusions. */
56
+ inclusions_sources?: string[];
57
+ }
58
+
59
+ /**
60
+ * Compiles a filter list using the specified configuration.
61
+ *
62
+ * @param {*} configuration - compilation configuration.
63
+ See the repo README for the details on it.
64
+ * @returns {Promise<Array<string>>} the array of rules.
65
+ */
66
+ declare async function compile(configuration: IConfiguration): Promise<string[]>;
67
+
68
+ export default compile;
69
+ }
package/src/index.js CHANGED
@@ -65,7 +65,7 @@ function prepareSourceHeader(source) {
65
65
  *
66
66
  * @param {*} configuration - compilation configuration.
67
67
  See the repo README for the details on it.
68
- * @returns {Array<string>} the array of rules.
68
+ * @returns {Promise<Array<string>>} the array of rules.
69
69
  */
70
70
  async function compile(configuration) {
71
71
  consola.info('Starting the compiler');
@@ -41,7 +41,10 @@
41
41
  "Compress",
42
42
  "Validate",
43
43
  "Deduplicate",
44
- "InvertAllow"
44
+ "InvertAllow",
45
+ "RemoveEmptyLines",
46
+ "TrimLines",
47
+ "InsertFinalNewLine"
45
48
  ]
46
49
  }
47
50
  },
@@ -113,7 +116,10 @@
113
116
  "Compress",
114
117
  "Validate",
115
118
  "Deduplicate",
116
- "InvertAllow"
119
+ "InvertAllow",
120
+ "RemoveEmptyLines",
121
+ "TrimLines",
122
+ "InsertFinalNewLine"
117
123
  ]
118
124
  }
119
125
  },
@@ -9,7 +9,7 @@ const filterUtils = require('../filter');
9
9
  * @param {Array<String>} exclusions - array of exclusions to apply
10
10
  * @param {Array<String>} exclusionsSources - array of exclusion sources
11
11
  * (can be a local or remote file)
12
- * @returns {Array<String>} filtered array of rules
12
+ * @returns {Promise<Array<String>>} filtered array of rules
13
13
  */
14
14
  async function exclude(rules, exclusions, exclusionsSources) {
15
15
  if (_.isEmpty(exclusions) && _.isEmpty(exclusionsSources)) {
@@ -9,7 +9,7 @@ const filterUtils = require('../filter');
9
9
  * @param {Array<String>} inclusions - array of inclusions to apply
10
10
  * @param {Array<String>} inclusionsSources - array of inclusions' sources
11
11
  * (can be a local or remote file)
12
- * @returns {Array<String>} filtered array of rules
12
+ * @returns {Promise<Array<String>>} filtered array of rules
13
13
  */
14
14
  async function include(rules, inclusions, inclusionsSources) {
15
15
  if (_.isEmpty(inclusions) && _.isEmpty(inclusionsSources)) {
@@ -0,0 +1,17 @@
1
+ const consola = require('consola');
2
+
3
+ /**
4
+ * This is a very simple transformation that inserts a final newline.
5
+ *
6
+ * @param {Array<string>} lines - lines to transform
7
+ * @returns {Array<string>} filtered lines
8
+ */
9
+ function insertFinalNewLine(lines) {
10
+ if (lines.length === 0 || (lines.length > 0 && lines[lines.length - 1].trim() !== '')) {
11
+ lines.push('');
12
+ consola.info('Final newline inserted');
13
+ }
14
+ return lines;
15
+ }
16
+
17
+ module.exports = insertFinalNewLine;
@@ -0,0 +1,15 @@
1
+ const consola = require('consola');
2
+
3
+ /**
4
+ * This is a very simple transformation that removes empty lines.
5
+ *
6
+ * @param {Array<string>} lines - lines to transform
7
+ * @returns {Array<string>} filtered lines
8
+ */
9
+ function removeEmptyLines(lines) {
10
+ const filtered = lines.filter((line) => line.trim().length);
11
+ consola.info(`Removed ${lines.length - filtered.length} empty lines`);
12
+ return filtered;
13
+ }
14
+
15
+ module.exports = removeEmptyLines;
@@ -4,7 +4,7 @@ const ruleUtils = require('../rule');
4
4
 
5
5
  /**
6
6
  * This transformation simply removes the following modifiers from the adblock-style rules:
7
- * $document, $all, $third-party, $3p, $popup
7
+ * $document, $doc, $all, $third-party, $3p, $popup
8
8
  *
9
9
  * @param {Array<string>} rules - rules to transform
10
10
  * @returns {Array<string>} filtered rules
@@ -32,6 +32,7 @@ function removeModifiers(rules) {
32
32
  modified = ruleUtils.removeModifier(props, '3p') || modified;
33
33
  modified = ruleUtils.removeModifier(props, 'all') || modified;
34
34
  modified = ruleUtils.removeModifier(props, 'document') || modified;
35
+ modified = ruleUtils.removeModifier(props, 'doc') || modified;
35
36
  modified = ruleUtils.removeModifier(props, 'popup') || modified;
36
37
  filtered.push(ruleUtils.adblockRuleToString(props));
37
38
 
@@ -6,6 +6,9 @@ const include = require('./include');
6
6
  const deduplicate = require('./deduplicate');
7
7
  const compress = require('./compress');
8
8
  const invertAllow = require('./invertallow');
9
+ const removeEmptyLines = require('./remove-empty-lines');
10
+ const trimLines = require('./trim-lines');
11
+ const insertFinalNewLine = require('./insert-final-newline');
9
12
 
10
13
  /**
11
14
  * Enum with all available transformations
@@ -17,6 +20,9 @@ const TRANSFORMATIONS = Object.freeze({
17
20
  Validate: 'Validate',
18
21
  Deduplicate: 'Deduplicate',
19
22
  InvertAllow: 'InvertAllow',
23
+ RemoveEmptyLines: 'RemoveEmptyLines',
24
+ TrimLines: 'TrimLines',
25
+ InsertFinalNewLine: 'InsertFinalNewLine',
20
26
  });
21
27
 
22
28
  /**
@@ -25,7 +31,7 @@ const TRANSFORMATIONS = Object.freeze({
25
31
  * @param {Array<string>} rules - rules to transform
26
32
  * @param {*} configuration - transformation configuration.
27
33
  * @param {Array<string>} transformations - a list of transformations to apply to the rules.
28
- * @returns {Array<string>} rules after applying all transformations.
34
+ * @returns {Promise<Array<string>>} rules after applying all transformations.
29
35
  */
30
36
  async function transform(rules, configuration, transformations) {
31
37
  // If none specified -- apply all transformationss
@@ -41,6 +47,9 @@ async function transform(rules, configuration, transformations) {
41
47
  transformed = await include(transformed, configuration.inclusions,
42
48
  configuration.inclusions_sources);
43
49
 
50
+ if (transformations.indexOf(TRANSFORMATIONS.TrimLines) !== -1) {
51
+ transformed = trimLines(transformed);
52
+ }
44
53
  if (transformations.indexOf(TRANSFORMATIONS.RemoveComments) !== -1) {
45
54
  transformed = removeComments(transformed);
46
55
  }
@@ -59,6 +68,12 @@ async function transform(rules, configuration, transformations) {
59
68
  if (transformations.indexOf(TRANSFORMATIONS.Deduplicate) !== -1) {
60
69
  transformed = deduplicate(transformed);
61
70
  }
71
+ if (transformations.indexOf(TRANSFORMATIONS.RemoveEmptyLines) !== -1) {
72
+ transformed = removeEmptyLines(transformed);
73
+ }
74
+ if (transformations.indexOf(TRANSFORMATIONS.InsertFinalNewLine) !== -1) {
75
+ transformed = insertFinalNewLine(transformed);
76
+ }
62
77
  return transformed;
63
78
  }
64
79
 
@@ -0,0 +1,16 @@
1
+ const consola = require('consola');
2
+ const _ = require('lodash');
3
+
4
+ /**
5
+ * This is a very simple transformation that removes leading and trailing spaces/tabs.
6
+ *
7
+ * @param {Array<string>} lines - lines/rules to transform
8
+ * @returns {Array<string>} filtered lines/rules
9
+ */
10
+ function trimLines(lines) {
11
+ const transformed = lines.map((line) => _.trim(line, ' \t'));
12
+ consola.info('Lines trimmed.');
13
+ return transformed;
14
+ }
15
+
16
+ module.exports = trimLines;
package/src/utils.js CHANGED
@@ -16,7 +16,7 @@ function isURL(str) {
16
16
  * Downloads (or reads from the disk) the specified source
17
17
  *
18
18
  * @param {*} urlOrPath url or path to a file
19
- * @returns {String} contents of the files
19
+ * @returns {Promise<String>} contents of the files
20
20
  */
21
21
  async function download(urlOrPath) {
22
22
  let str = '';
@@ -85,4 +85,40 @@ describe('Source compiler', () => {
85
85
  // Make sure scope URLs were requested
86
86
  scope.done();
87
87
  });
88
+
89
+ it('compile the source with transformations', async () => {
90
+ const testList = `
91
+
92
+ example.org
93
+
94
+ test1.com
95
+
96
+ ! comment
97
+ test1.com
98
+
99
+ test2.com`;
100
+ const scope = nock('https://example.org')
101
+ .get('/test-filter.txt')
102
+ .reply(200, testList);
103
+
104
+ const source = {
105
+ name: 'test source',
106
+ source: 'https://example.org/test-filter.txt',
107
+ transformations: [
108
+ TRANSFORMATIONS.Deduplicate,
109
+ TRANSFORMATIONS.RemoveComments,
110
+ TRANSFORMATIONS.TrimLines,
111
+ TRANSFORMATIONS.RemoveEmptyLines,
112
+ TRANSFORMATIONS.InsertFinalNewLine,
113
+ ],
114
+ };
115
+ const compiled = await compileSource(source);
116
+ expect(compiled).toEqual([
117
+ 'example.org',
118
+ 'test1.com',
119
+ 'test2.com',
120
+ '',
121
+ ]);
122
+ scope.done();
123
+ });
88
124
  });
@@ -0,0 +1,48 @@
1
+ const insertFinalNewLine = require('../../src/transformations/insert-final-newline');
2
+
3
+ describe('Insert final new line', () => {
4
+ it('test with empty file', () => {
5
+ const rules = [];
6
+ const filtered = insertFinalNewLine(rules);
7
+ expect(filtered).toEqual(['']);
8
+ });
9
+ it('test with one empty line', () => {
10
+ const rules = [''];
11
+ const filtered = insertFinalNewLine(rules);
12
+ expect(filtered).toEqual(['']);
13
+ });
14
+ it('test with one rule and one empty lines', () => {
15
+ const rules = [
16
+ 'rule1',
17
+ '',
18
+ ];
19
+ const filtered = insertFinalNewLine(rules);
20
+ expect(filtered).toEqual([
21
+ 'rule1',
22
+ '',
23
+ ]);
24
+ });
25
+ it('test with many empty lines', () => {
26
+ const rules = [
27
+ 'rule1',
28
+ '',
29
+ '',
30
+ ];
31
+ const filtered = insertFinalNewLine(rules);
32
+ expect(filtered).toEqual([
33
+ 'rule1',
34
+ '',
35
+ '',
36
+ ]);
37
+ });
38
+ it('test with one rule', () => {
39
+ const rules = [
40
+ 'rule1',
41
+ ];
42
+ const filtered = insertFinalNewLine(rules);
43
+ expect(filtered).toEqual([
44
+ 'rule1',
45
+ '',
46
+ ]);
47
+ });
48
+ });
@@ -0,0 +1,77 @@
1
+ const removeEmptyLines = require('../../src/transformations/remove-empty-lines');
2
+
3
+ describe('Remove empty lines', () => {
4
+ it('test with no empty lines', () => {
5
+ const rules = [
6
+ 'rule1',
7
+ 'rule2',
8
+ ];
9
+ const filtered = removeEmptyLines(rules);
10
+ expect(filtered).toEqual([
11
+ 'rule1',
12
+ 'rule2',
13
+ ]);
14
+ });
15
+ it('test with one empty line at the end', () => {
16
+ const rules = [
17
+ 'rule1',
18
+ 'rule2',
19
+ '',
20
+ ];
21
+ const filtered = removeEmptyLines(rules);
22
+ expect(filtered).toEqual([
23
+ 'rule1',
24
+ 'rule2',
25
+ ]);
26
+ });
27
+ it('test with one empty line between the rules', () => {
28
+ const rules = [
29
+ 'rule1',
30
+ '',
31
+ 'rule2',
32
+ ];
33
+ const filtered = removeEmptyLines(rules);
34
+ expect(filtered).toEqual([
35
+ 'rule1',
36
+ 'rule2',
37
+ ]);
38
+ });
39
+ it('test with two empty line', () => {
40
+ const rules = [
41
+ 'rule1',
42
+ '',
43
+ '',
44
+ 'rule2',
45
+ ];
46
+ const filtered = removeEmptyLines(rules);
47
+ expect(filtered).toEqual([
48
+ 'rule1',
49
+ 'rule2',
50
+ ]);
51
+ });
52
+ it('test with comments, empty lines', () => {
53
+ const rules = [
54
+ '! aaa',
55
+ 'rule1',
56
+ '',
57
+ '! bbb',
58
+ '',
59
+ '',
60
+ 'rule2',
61
+ '',
62
+ '',
63
+ '',
64
+ '!ccc',
65
+ '',
66
+ '',
67
+ ];
68
+ const filtered = removeEmptyLines(rules);
69
+ expect(filtered).toEqual([
70
+ '! aaa',
71
+ 'rule1',
72
+ '! bbb',
73
+ 'rule2',
74
+ '!ccc',
75
+ ]);
76
+ });
77
+ });
@@ -7,17 +7,19 @@ describe('Strip third-party', () => {
7
7
  ||example.net$domain=ya.ru,3p
8
8
  ||islandofadvert.com^$document,popup
9
9
  ||example.com$document
10
+ ||example.com$doc
10
11
  ||example.com$all
11
12
  ||example.org^`.split(/\r?\n/);
12
13
  const filtered = removeModifiers(rules);
13
14
 
14
- expect(filtered).toHaveLength(7);
15
+ expect(filtered).toHaveLength(8);
15
16
  expect(filtered[0]).toBe('! test comment');
16
17
  expect(filtered[1]).toBe('||example.org$important');
17
18
  expect(filtered[2]).toBe('||example.net$domain=ya.ru');
18
19
  expect(filtered[3]).toBe('||islandofadvert.com^');
19
20
  expect(filtered[4]).toBe('||example.com');
20
21
  expect(filtered[5]).toBe('||example.com');
21
- expect(filtered[6]).toBe('||example.org^');
22
+ expect(filtered[6]).toBe('||example.com');
23
+ expect(filtered[7]).toBe('||example.org^');
22
24
  });
23
25
  });
@@ -0,0 +1,66 @@
1
+ const _ = require('lodash');
2
+ const trimLines = require('../../src/transformations/trim-lines');
3
+
4
+ describe('Trim lines', () => {
5
+ it('test with empty file', () => {
6
+ const rules = [];
7
+ const filtered = trimLines(rules);
8
+ expect(filtered).toEqual([]);
9
+ });
10
+ it('test with one empty line', () => {
11
+ const rules = [''];
12
+ const filtered = trimLines(rules);
13
+ expect(filtered).toEqual(['']);
14
+ });
15
+ it('test with three rules', () => {
16
+ const rules = [
17
+ 'rule1',
18
+ ' rule2 ',
19
+ '',
20
+ ' rule3 ',
21
+ ];
22
+ const filtered = trimLines(rules);
23
+ expect(filtered).toEqual([
24
+ 'rule1',
25
+ 'rule2',
26
+ '',
27
+ 'rule3',
28
+ ]);
29
+ });
30
+ it('test with three rules and comments', () => {
31
+ const rules = [
32
+ 'rule1',
33
+ ' rule2 ',
34
+ '',
35
+ '',
36
+ '! comment ',
37
+ ' rule3 ',
38
+ ' ! comment multiple words ',
39
+ '',
40
+ ];
41
+ const filtered = trimLines(rules);
42
+ expect(filtered).toEqual([
43
+ 'rule1',
44
+ 'rule2',
45
+ '',
46
+ '',
47
+ '! comment',
48
+ 'rule3',
49
+ '! comment multiple words',
50
+ '',
51
+ ]);
52
+ });
53
+ it('test with three rules and comments', () => {
54
+ const rules = [
55
+ _.repeat(' ', _.random(1, 10)), // ' ', ' ', ..., ' ' etc.
56
+ ' ! comment multiple words , ',
57
+ '',
58
+ ];
59
+ const filtered = trimLines(rules);
60
+ expect(filtered).toEqual([
61
+ '',
62
+ '! comment multiple words ,',
63
+ '',
64
+ ]);
65
+ });
66
+ });