@adguard/hostlist-compiler 1.0.15 → 1.0.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -18
- package/bamboo-specs/bamboo.yaml +9 -53
- package/bamboo-specs/build.yaml +101 -0
- package/bamboo-specs/deploy.yaml +46 -0
- package/bamboo-specs/increment.yaml +54 -0
- package/bamboo-specs/permissions.yaml +16 -0
- package/bamboo-specs/test.yaml +65 -0
- package/package.json +4 -2
- package/src/cli.js +78 -7
- package/tools/build-txt.js +19 -0
package/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
[](https://www.npmjs.com/package/@adguard/hostlist-compiler/)
|
|
4
4
|
|
|
5
|
-
This is a simple tool that makes it easier to compile a [hosts blocklist](https://
|
|
5
|
+
This is a simple tool that makes it easier to compile a [hosts blocklist](https://adguard-dns.io/kb/general/dns-filtering-syntax/) compatible with AdGuard Home or any other AdGuard product with **DNS filtering**.
|
|
6
6
|
|
|
7
7
|
- [Usage](#usage)
|
|
8
8
|
- [Configuration](#configuration)
|
|
@@ -20,7 +20,7 @@ This is a simple tool that makes it easier to compile a [hosts blocklist](https:
|
|
|
20
20
|
- [InsertFinalNewLine](#insertfinalnewline)
|
|
21
21
|
- [How to build](#how-to-build)
|
|
22
22
|
|
|
23
|
-
## <a
|
|
23
|
+
## <a name="usage"></a> Usage
|
|
24
24
|
|
|
25
25
|
First of all, install the `hostlist-compiler`:
|
|
26
26
|
|
|
@@ -28,13 +28,48 @@ First of all, install the `hostlist-compiler`:
|
|
|
28
28
|
npm i -g @adguard/hostlist-compiler
|
|
29
29
|
```
|
|
30
30
|
|
|
31
|
-
|
|
31
|
+
After that you have two options.
|
|
32
|
+
|
|
33
|
+
**Quick hosts conversion**
|
|
34
|
+
|
|
35
|
+
Convert and compress a `/etc/hosts`-syntax blocklist to [AdGuard syntax](https://adguard-dns.io/kb/general/dns-filtering-syntax/).
|
|
36
|
+
|
|
37
|
+
```
|
|
38
|
+
hostlist-compiler -i hosts.txt -i hosts2.txt -o output.txt
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
**Build a configurable blocklist from multiple sources**
|
|
42
|
+
|
|
43
|
+
Prepare the list configuration (read more about that [below](#configuration)) and run the compiler:
|
|
32
44
|
|
|
33
45
|
```bash
|
|
34
46
|
hostlist-compiler -c configuration.json -o output.txt
|
|
35
47
|
```
|
|
36
48
|
|
|
37
|
-
|
|
49
|
+
**All command line options**
|
|
50
|
+
|
|
51
|
+
```
|
|
52
|
+
Usage: hostlist-compiler [options]
|
|
53
|
+
|
|
54
|
+
Options:
|
|
55
|
+
--config, -c Path to the compiler configuration file [string]
|
|
56
|
+
--input, -i URL (or path to a file) to convert to an AdGuard-syntax
|
|
57
|
+
blocklist. Can be specified multiple times. [array]
|
|
58
|
+
--input-type, -t Type of the input file (/etc/hosts, adguard) [string]
|
|
59
|
+
--output, -o Path to the output file [string] [required]
|
|
60
|
+
--verbose, -v Run with verbose logging [boolean]
|
|
61
|
+
--version Show version number [boolean]
|
|
62
|
+
-h, --help Show help [boolean]
|
|
63
|
+
|
|
64
|
+
Examples:
|
|
65
|
+
hostlist-compiler -c config.json -o compile a blocklist and write the
|
|
66
|
+
output.txt output to output.txt
|
|
67
|
+
hostlist-compiler -i compile a blocklist from the URL and
|
|
68
|
+
https://example.org/hosts.txt -o write the output to output.txt
|
|
69
|
+
output.txt
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
### <a name="configuration"></a> Configuration
|
|
38
73
|
|
|
39
74
|
Configuration defines your filter list sources, and the transformations that are applied to the sources.
|
|
40
75
|
|
|
@@ -113,7 +148,7 @@ Please note, that exclusion or inclusion rules may be a plain string, wildcard,
|
|
|
113
148
|
- `/regex/` - every rule that matches this regular expression, will match the rule. By default, regular expressions are case-insensitive.
|
|
114
149
|
- `! comment` - comments will be ignored.
|
|
115
150
|
|
|
116
|
-
### <a
|
|
151
|
+
### <a name="command-line"></a> Command-line
|
|
117
152
|
|
|
118
153
|
Command-line arguments.
|
|
119
154
|
|
|
@@ -132,7 +167,7 @@ Examples:
|
|
|
132
167
|
output.txt output to output.txt
|
|
133
168
|
```
|
|
134
169
|
|
|
135
|
-
### <a
|
|
170
|
+
### <a name="api"></a> API
|
|
136
171
|
|
|
137
172
|
Install: `npm i @adguard/hostlist-compiler` or `yarn add @adguard/hostlist-compiler`
|
|
138
173
|
|
|
@@ -154,7 +189,7 @@ const compile = require("@adguard/hostlist-compiler");
|
|
|
154
189
|
],
|
|
155
190
|
transformations: ['Deduplicate'],
|
|
156
191
|
});
|
|
157
|
-
|
|
192
|
+
|
|
158
193
|
// Write to file
|
|
159
194
|
writeFileSync('your-hostlist.txt', result.join('\n'));
|
|
160
195
|
})();
|
|
@@ -213,7 +248,7 @@ import { writeFileSync } from 'fs';
|
|
|
213
248
|
})();
|
|
214
249
|
```
|
|
215
250
|
|
|
216
|
-
## <a
|
|
251
|
+
## <a name="transformations"></a> Transformations
|
|
217
252
|
|
|
218
253
|
Here is the full list of transformations that are available:
|
|
219
254
|
|
|
@@ -229,11 +264,11 @@ Here is the full list of transformations that are available:
|
|
|
229
264
|
|
|
230
265
|
Please note that these transformations are are always applied in the order specified here.
|
|
231
266
|
|
|
232
|
-
### <a
|
|
267
|
+
### <a name="remove-comments"></a> RemoveComments
|
|
233
268
|
|
|
234
269
|
This is a very simple transformation that simply removes comments (e.g. all rules starting with `!` or `#`).
|
|
235
270
|
|
|
236
|
-
### <a
|
|
271
|
+
### <a name="compress"></a> Compress
|
|
237
272
|
|
|
238
273
|
> **IMPORTANT:** this transformation converts `hosts` lists into `adblock` lists.
|
|
239
274
|
|
|
@@ -242,7 +277,7 @@ Here's what it does:
|
|
|
242
277
|
1. It converts all rules to adblock-style rules. For instance, `0.0.0.0 example.org` will be converted to `||example.org^`.
|
|
243
278
|
2. It discards the rules that are now redundant because of other existing rules. For instance, `||example.org` blocks `example.org` and all it's subdomains, therefore additional rules for the subdomains are now redundant.
|
|
244
279
|
|
|
245
|
-
### <a
|
|
280
|
+
### <a name="remove-modifiers"></a> RemoveModifiers
|
|
246
281
|
|
|
247
282
|
By default, [AdGuard Home](https://github.com/AdguardTeam/AdGuardHome) will ignore rules with unsupported modifiers, and all of the modifiers listed here are unsupported. However, the rules with these modifiers are likely to be okay for DNS-level blocking, that's why you might want to remove them when importing rules from a traditional filter list.
|
|
248
283
|
|
|
@@ -255,7 +290,7 @@ Here is the list of modifiers that will be removed:
|
|
|
255
290
|
|
|
256
291
|
**IMPORTANT:** please, be cautious with this. Blindly removing `$third-party` from traditional ad blocking rules leads to lots of false-positives. This is exactly why there is an option to exclude rules - you may need to use it.
|
|
257
292
|
|
|
258
|
-
### <a
|
|
293
|
+
### <a name="validate"></a> Validate
|
|
259
294
|
|
|
260
295
|
This transformation is really crucial if you're using a filter list for a traditional ad blocker as a source.
|
|
261
296
|
|
|
@@ -269,7 +304,7 @@ So here's what it does:
|
|
|
269
304
|
|
|
270
305
|
If there are comments preceding the invalid rule, they will be removed as well.
|
|
271
306
|
|
|
272
|
-
### <a
|
|
307
|
+
### <a name="deduplicate"></a> Deduplicate
|
|
273
308
|
|
|
274
309
|
This transformation simply removes the duplicates from the specified source.
|
|
275
310
|
|
|
@@ -294,7 +329,7 @@ Here's what will be left after the transformation:
|
|
|
294
329
|
rule1
|
|
295
330
|
```
|
|
296
331
|
|
|
297
|
-
### <a
|
|
332
|
+
### <a name="invertallow"></a> InvertAllow
|
|
298
333
|
|
|
299
334
|
This transformation converts blocking rules to "allow" rules. Note, that it does nothing to /etc/hosts rules (unless they were previously converted to adblock-style syntax by a different transformation, for example [Compress](#compress)).
|
|
300
335
|
|
|
@@ -327,7 +362,7 @@ Here's what we will have after applying this transformation:
|
|
|
327
362
|
@@rule2
|
|
328
363
|
```
|
|
329
364
|
|
|
330
|
-
### <a
|
|
365
|
+
### <a name="removeemptylines"></a> RemoveEmptyLines
|
|
331
366
|
|
|
332
367
|
This is a very simple transformation that removes empty lines.
|
|
333
368
|
|
|
@@ -352,7 +387,7 @@ rule2
|
|
|
352
387
|
rule3
|
|
353
388
|
```
|
|
354
389
|
|
|
355
|
-
### <a
|
|
390
|
+
### <a name="trimlines"></a> TrimLines
|
|
356
391
|
|
|
357
392
|
This is a very simple transformation that removes leading and trailing spaces/tabs.
|
|
358
393
|
|
|
@@ -376,7 +411,7 @@ rule3
|
|
|
376
411
|
rule4
|
|
377
412
|
```
|
|
378
413
|
|
|
379
|
-
### <a
|
|
414
|
+
### <a name="insertfinalnewline"></a> InsertFinalNewLine
|
|
380
415
|
|
|
381
416
|
This is a very simple transformation that inserts a final newline.
|
|
382
417
|
|
|
@@ -401,7 +436,7 @@ rule3
|
|
|
401
436
|
|
|
402
437
|
`RemoveEmptyLines` doesn't delete this empty row due to the execution order.
|
|
403
438
|
|
|
404
|
-
## <a
|
|
439
|
+
## <a name="how-to-build"></a> How to build
|
|
405
440
|
|
|
406
441
|
- `yarn install` - installs dependencies
|
|
407
442
|
- `yarn lint` - runs eslint
|
package/bamboo-specs/bamboo.yaml
CHANGED
|
@@ -1,58 +1,14 @@
|
|
|
1
1
|
---
|
|
2
|
-
|
|
3
|
-
plan:
|
|
4
|
-
project-key: AJL
|
|
5
|
-
key: HBRTSPECS
|
|
6
|
-
name: hostlist-compiler - test
|
|
7
|
-
variables:
|
|
8
|
-
dockerJslBuilder: adguard/jsl-builder:15.8.0--1.22.5--1
|
|
2
|
+
!include 'build.yaml'
|
|
9
3
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
manual: 'false'
|
|
13
|
-
final: 'false'
|
|
14
|
-
jobs:
|
|
15
|
-
- Test
|
|
16
|
-
|
|
17
|
-
Test:
|
|
18
|
-
key: TEST
|
|
19
|
-
docker:
|
|
20
|
-
image: "${bamboo.dockerJslBuilder}"
|
|
21
|
-
volumes:
|
|
22
|
-
${system.YARN_DIR}: "${bamboo.cacheYarn}"
|
|
23
|
-
tasks:
|
|
24
|
-
- checkout:
|
|
25
|
-
force-clean-build: 'true'
|
|
26
|
-
- script:
|
|
27
|
-
interpreter: SHELL
|
|
28
|
-
scripts:
|
|
29
|
-
- |-
|
|
30
|
-
set -e
|
|
31
|
-
set -x
|
|
32
|
-
yarn install
|
|
33
|
-
yarn lint
|
|
34
|
-
yarn test
|
|
35
|
-
rm -rf node_modules
|
|
36
|
-
final-tasks:
|
|
37
|
-
- clean
|
|
38
|
-
requirements:
|
|
39
|
-
- adg-docker: 'true'
|
|
4
|
+
---
|
|
5
|
+
!include 'deploy.yaml'
|
|
40
6
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
delete:
|
|
44
|
-
after-deleted-days: '1'
|
|
45
|
-
after-inactive-days: '5'
|
|
46
|
-
link-to-jira: 'true'
|
|
7
|
+
---
|
|
8
|
+
!include 'increment.yaml'
|
|
47
9
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
- plan-status-changed
|
|
51
|
-
recipients:
|
|
52
|
-
- webhook:
|
|
53
|
-
name: Build webhook
|
|
54
|
-
url: http://prod.jirahub.service.eu.consul/v1/webhook/bamboo?channel=adguard-qa
|
|
10
|
+
---
|
|
11
|
+
!include 'permissions.yaml'
|
|
55
12
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
concurrent-build-plugin: system-default
|
|
13
|
+
---
|
|
14
|
+
!include 'test.yaml'
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
---
|
|
2
|
+
version: 2
|
|
3
|
+
plan:
|
|
4
|
+
project-key: AJL
|
|
5
|
+
key: HOSTLISTCMPLRBUILD
|
|
6
|
+
name: hostlist-compiler - build
|
|
7
|
+
variables:
|
|
8
|
+
dockerContainer: adguard/node-ssh:18.13--0
|
|
9
|
+
|
|
10
|
+
stages:
|
|
11
|
+
- Build:
|
|
12
|
+
manual: false
|
|
13
|
+
final: false
|
|
14
|
+
jobs:
|
|
15
|
+
- Build
|
|
16
|
+
|
|
17
|
+
Build:
|
|
18
|
+
key: BUILD
|
|
19
|
+
other:
|
|
20
|
+
clean-working-dir: true
|
|
21
|
+
docker:
|
|
22
|
+
image: "${bamboo.dockerContainer}"
|
|
23
|
+
volumes:
|
|
24
|
+
${system.YARN_DIR}: "${bamboo.cacheYarn}"
|
|
25
|
+
tasks:
|
|
26
|
+
- checkout:
|
|
27
|
+
force-clean-build: true
|
|
28
|
+
- script:
|
|
29
|
+
interpreter: SHELL
|
|
30
|
+
scripts:
|
|
31
|
+
- |-
|
|
32
|
+
set -e
|
|
33
|
+
set -x
|
|
34
|
+
|
|
35
|
+
# Fix mixed logs
|
|
36
|
+
exec 2>&1
|
|
37
|
+
|
|
38
|
+
# install dependencies at the beginning
|
|
39
|
+
yarn install ${bamboo.varsYarn}
|
|
40
|
+
|
|
41
|
+
yarn build-txt
|
|
42
|
+
- inject-variables:
|
|
43
|
+
file: dist/build.txt
|
|
44
|
+
scope: RESULT
|
|
45
|
+
namespace: inject
|
|
46
|
+
- script:
|
|
47
|
+
interpreter: SHELL
|
|
48
|
+
scripts:
|
|
49
|
+
- |-
|
|
50
|
+
#!/bin/bash
|
|
51
|
+
set -e
|
|
52
|
+
set -x
|
|
53
|
+
|
|
54
|
+
# Fix mixed logs
|
|
55
|
+
exec 2>&1
|
|
56
|
+
|
|
57
|
+
# Fail if version was not incremented
|
|
58
|
+
if [ "$(git tag -l "v${bamboo.inject.version}")" ]; then
|
|
59
|
+
echo "Build failed!"
|
|
60
|
+
echo "Error: Tag v${bamboo.inject.version} already exists. Increment version before build"
|
|
61
|
+
exit 1
|
|
62
|
+
fi
|
|
63
|
+
|
|
64
|
+
yarn pack --filename hostlist-compiler.tgz
|
|
65
|
+
|
|
66
|
+
# clean up at the end
|
|
67
|
+
rm -rf node_modules
|
|
68
|
+
- any-task:
|
|
69
|
+
plugin-key: com.atlassian.bamboo.plugins.vcs:task.vcs.tagging
|
|
70
|
+
configuration:
|
|
71
|
+
selectedRepository: defaultRepository
|
|
72
|
+
tagName: v${bamboo.inject.version}
|
|
73
|
+
artifacts:
|
|
74
|
+
- name: hostlist-compiler.tgz
|
|
75
|
+
location: ./
|
|
76
|
+
pattern: hostlist-compiler.tgz
|
|
77
|
+
shared: true
|
|
78
|
+
required: true
|
|
79
|
+
requirements:
|
|
80
|
+
- adg-docker: true
|
|
81
|
+
|
|
82
|
+
triggers: []
|
|
83
|
+
|
|
84
|
+
branches:
|
|
85
|
+
create: manually
|
|
86
|
+
delete: never
|
|
87
|
+
link-to-jira: true
|
|
88
|
+
|
|
89
|
+
notifications:
|
|
90
|
+
- events:
|
|
91
|
+
- plan-status-changed
|
|
92
|
+
recipients:
|
|
93
|
+
- webhook:
|
|
94
|
+
name: Build webhook
|
|
95
|
+
# TODO: maybe channel should be 'adguard-filters'
|
|
96
|
+
url: http://prod.jirahub.service.eu.consul/v1/webhook/bamboo?channel=adguard-qa
|
|
97
|
+
|
|
98
|
+
labels: []
|
|
99
|
+
|
|
100
|
+
other:
|
|
101
|
+
concurrent-build-plugin: system-default
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
---
|
|
2
|
+
version: 2
|
|
3
|
+
deployment:
|
|
4
|
+
name: hostlist-compiler - deploy
|
|
5
|
+
source-plan: AJL-HOSTLISTCMPLRBUILD
|
|
6
|
+
release-naming: ${bamboo.inject.version}
|
|
7
|
+
environments:
|
|
8
|
+
- npmjs
|
|
9
|
+
|
|
10
|
+
npmjs:
|
|
11
|
+
docker:
|
|
12
|
+
image: adguard/node-ssh:18.13--0
|
|
13
|
+
volumes:
|
|
14
|
+
${system.YARN_DIR}: "${bamboo.cacheYarn}"
|
|
15
|
+
triggers: []
|
|
16
|
+
tasks:
|
|
17
|
+
- checkout:
|
|
18
|
+
force-clean-build: true
|
|
19
|
+
- artifact-download:
|
|
20
|
+
artifacts:
|
|
21
|
+
- name: hostlist-compiler.tgz
|
|
22
|
+
- script:
|
|
23
|
+
interpreter: SHELL
|
|
24
|
+
scripts:
|
|
25
|
+
- |-
|
|
26
|
+
set -e
|
|
27
|
+
set -x
|
|
28
|
+
|
|
29
|
+
# Fix mixed logs
|
|
30
|
+
exec 2>&1
|
|
31
|
+
|
|
32
|
+
ls -alt
|
|
33
|
+
|
|
34
|
+
export NPM_TOKEN=${bamboo.npmSecretToken}
|
|
35
|
+
echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > .npmrc
|
|
36
|
+
npm publish hostlist-compiler.tgz --access public
|
|
37
|
+
requirements:
|
|
38
|
+
- adg-docker: true
|
|
39
|
+
notifications:
|
|
40
|
+
- events:
|
|
41
|
+
- deployment-started-and-finished
|
|
42
|
+
recipients:
|
|
43
|
+
- webhook:
|
|
44
|
+
name: Deploy webhook
|
|
45
|
+
# TODO: maybe channel should be 'adguard-filters'
|
|
46
|
+
url: http://prod.jirahub.service.eu.consul/v1/webhook/bamboo?channel=adguard-qa
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
---
|
|
2
|
+
version: 2
|
|
3
|
+
plan:
|
|
4
|
+
project-key: AJL
|
|
5
|
+
key: HOSTLISTCMPLRINCR
|
|
6
|
+
name: hostlist-compiler - increment
|
|
7
|
+
variables:
|
|
8
|
+
dockerContainer: adguard/node-ssh:18.13--0
|
|
9
|
+
|
|
10
|
+
stages:
|
|
11
|
+
- Increment:
|
|
12
|
+
manual: false
|
|
13
|
+
final: false
|
|
14
|
+
jobs:
|
|
15
|
+
- Increment
|
|
16
|
+
|
|
17
|
+
Increment:
|
|
18
|
+
key: INCR
|
|
19
|
+
docker:
|
|
20
|
+
image: "${bamboo.dockerContainer}"
|
|
21
|
+
volumes:
|
|
22
|
+
${system.YARN_DIR}: "${bamboo.cacheYarn}"
|
|
23
|
+
other:
|
|
24
|
+
clean-working-dir: true
|
|
25
|
+
tasks:
|
|
26
|
+
- checkout:
|
|
27
|
+
force-clean-build: true
|
|
28
|
+
- script:
|
|
29
|
+
interpreter: SHELL
|
|
30
|
+
scripts:
|
|
31
|
+
- |-
|
|
32
|
+
set -e
|
|
33
|
+
set -x
|
|
34
|
+
|
|
35
|
+
# Fix mixed logs
|
|
36
|
+
exec 2>&1
|
|
37
|
+
|
|
38
|
+
yarn increment
|
|
39
|
+
- any-task:
|
|
40
|
+
plugin-key: com.atlassian.bamboo.plugins.vcs:task.vcs.commit
|
|
41
|
+
configuration:
|
|
42
|
+
commitMessage: 'skipci: Automatic increment build number'
|
|
43
|
+
selectedRepository: defaultRepository
|
|
44
|
+
requirements:
|
|
45
|
+
- adg-docker: true
|
|
46
|
+
|
|
47
|
+
branches:
|
|
48
|
+
create: manually
|
|
49
|
+
delete: never
|
|
50
|
+
link-to-jira: true
|
|
51
|
+
|
|
52
|
+
labels: []
|
|
53
|
+
other:
|
|
54
|
+
concurrent-build-plugin: system-default
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
---
|
|
2
|
+
version: 2
|
|
3
|
+
deployment:
|
|
4
|
+
name: hostlist-compiler - deploy
|
|
5
|
+
deployment-permissions:
|
|
6
|
+
- groups:
|
|
7
|
+
- extensions-developers
|
|
8
|
+
permissions:
|
|
9
|
+
- view
|
|
10
|
+
environment-permissions:
|
|
11
|
+
- npmjs:
|
|
12
|
+
- groups:
|
|
13
|
+
- extensions-developers
|
|
14
|
+
permissions:
|
|
15
|
+
- view
|
|
16
|
+
- deploy
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
---
|
|
2
|
+
version: 2
|
|
3
|
+
plan:
|
|
4
|
+
project-key: AJL
|
|
5
|
+
key: HOSTLISTCMPLRTEST
|
|
6
|
+
name: hostlist-compiler - tests
|
|
7
|
+
variables:
|
|
8
|
+
dockerContainer: adguard/node-ssh:18.13--0
|
|
9
|
+
|
|
10
|
+
stages:
|
|
11
|
+
- Test:
|
|
12
|
+
manual: false
|
|
13
|
+
final: false
|
|
14
|
+
jobs:
|
|
15
|
+
- Test
|
|
16
|
+
|
|
17
|
+
Test:
|
|
18
|
+
key: TEST
|
|
19
|
+
docker:
|
|
20
|
+
image: "${bamboo.dockerContainer}"
|
|
21
|
+
volumes:
|
|
22
|
+
${system.YARN_DIR}: "${bamboo.cacheYarn}"
|
|
23
|
+
tasks:
|
|
24
|
+
- checkout:
|
|
25
|
+
force-clean-build: true
|
|
26
|
+
- script:
|
|
27
|
+
interpreter: SHELL
|
|
28
|
+
scripts:
|
|
29
|
+
- |-
|
|
30
|
+
set -e
|
|
31
|
+
set -x
|
|
32
|
+
|
|
33
|
+
# Fix mixed logs
|
|
34
|
+
exec 2>&1
|
|
35
|
+
|
|
36
|
+
yarn install ${bamboo.varsYarn}
|
|
37
|
+
|
|
38
|
+
yarn lint
|
|
39
|
+
yarn test
|
|
40
|
+
|
|
41
|
+
rm -rf node_modules
|
|
42
|
+
final-tasks:
|
|
43
|
+
- clean
|
|
44
|
+
requirements:
|
|
45
|
+
- adg-docker: true
|
|
46
|
+
|
|
47
|
+
branches:
|
|
48
|
+
create: for-pull-request
|
|
49
|
+
delete:
|
|
50
|
+
after-deleted-days: '1'
|
|
51
|
+
after-inactive-days: '5'
|
|
52
|
+
link-to-jira: true
|
|
53
|
+
|
|
54
|
+
notifications:
|
|
55
|
+
- events:
|
|
56
|
+
- plan-status-changed
|
|
57
|
+
recipients:
|
|
58
|
+
- webhook:
|
|
59
|
+
name: Build webhook
|
|
60
|
+
# TODO: maybe channel should be 'adguard-filters'
|
|
61
|
+
url: http://prod.jirahub.service.eu.consul/v1/webhook/bamboo?channel=adguard-qa
|
|
62
|
+
|
|
63
|
+
labels: []
|
|
64
|
+
other:
|
|
65
|
+
concurrent-build-plugin: system-default
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adguard/hostlist-compiler",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.19",
|
|
4
4
|
"description": "A simple tool that compiles hosts blocklists from multiple sources",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"types": "src/index.d.ts",
|
|
@@ -11,8 +11,10 @@
|
|
|
11
11
|
"hostlist-compiler": "src/cli.js"
|
|
12
12
|
},
|
|
13
13
|
"scripts": {
|
|
14
|
-
"lint": "eslint .",
|
|
14
|
+
"lint": "eslint . --cache",
|
|
15
15
|
"test": "jest --runInBand .",
|
|
16
|
+
"increment": "yarn version --patch --no-git-tag-version",
|
|
17
|
+
"build-txt": "node tools/build-txt.js",
|
|
16
18
|
"compile": "node src/cli.js"
|
|
17
19
|
},
|
|
18
20
|
"devDependencies": {
|
package/src/cli.js
CHANGED
|
@@ -8,13 +8,32 @@ const packageJson = require('../package.json');
|
|
|
8
8
|
// eslint-disable-next-line import/order
|
|
9
9
|
const { argv } = require('yargs')
|
|
10
10
|
.usage('Usage: $0 [options]')
|
|
11
|
-
.example(
|
|
11
|
+
.example(
|
|
12
|
+
'$0 -c config.json -o output.txt',
|
|
13
|
+
'compile a blocklist and write the output to output.txt',
|
|
14
|
+
)
|
|
15
|
+
.example(
|
|
16
|
+
'$0 -i https://example.org/hosts.txt -o output.txt',
|
|
17
|
+
'compile a blocklist from the URL and write the output to output.txt',
|
|
18
|
+
)
|
|
12
19
|
.option('config', {
|
|
13
20
|
alias: 'c',
|
|
14
21
|
type: 'string',
|
|
15
22
|
description: 'Path to the compiler configuration file',
|
|
16
23
|
nargs: 1,
|
|
17
24
|
})
|
|
25
|
+
.option('input', {
|
|
26
|
+
alias: 'i',
|
|
27
|
+
array: true,
|
|
28
|
+
type: 'string',
|
|
29
|
+
description: 'URL (or path to a file) to convert to an AdGuard-syntax blocklist. Can be specified multiple times.',
|
|
30
|
+
})
|
|
31
|
+
.option('input-type', {
|
|
32
|
+
alias: 't',
|
|
33
|
+
type: 'string',
|
|
34
|
+
description: 'Type of the input file (/etc/hosts, adguard)',
|
|
35
|
+
nargs: 1,
|
|
36
|
+
})
|
|
18
37
|
.option('output', {
|
|
19
38
|
alias: 'o',
|
|
20
39
|
type: 'string',
|
|
@@ -26,25 +45,77 @@ const { argv } = require('yargs')
|
|
|
26
45
|
type: 'boolean',
|
|
27
46
|
description: 'Run with verbose logging',
|
|
28
47
|
})
|
|
29
|
-
.demandOption(['
|
|
48
|
+
.demandOption(['o'])
|
|
30
49
|
.version()
|
|
31
50
|
.help('h')
|
|
32
51
|
.alias('h', 'help');
|
|
33
52
|
|
|
34
53
|
if (argv.verbose) {
|
|
35
|
-
// trace level
|
|
54
|
+
// trace level.
|
|
36
55
|
consola.level = 5;
|
|
37
56
|
}
|
|
38
57
|
|
|
39
58
|
consola.info(`Starting ${packageJson.name} v${packageJson.version}`);
|
|
40
59
|
|
|
60
|
+
/**
|
|
61
|
+
* Reads the configuration file and returns the parsed JSON.
|
|
62
|
+
*
|
|
63
|
+
* @returns {Promise<Object> | Promise<Error>} configuration file as JSON.
|
|
64
|
+
*/
|
|
65
|
+
async function readConfig() {
|
|
66
|
+
consola.debug(`Reading configuration from ${argv.config}`);
|
|
67
|
+
|
|
68
|
+
// Check if the configuration file exists.
|
|
69
|
+
await fs.access(argv.config);
|
|
70
|
+
const configStr = (await fs.readFile(argv.config)).toString();
|
|
71
|
+
|
|
72
|
+
return JSON.parse(configStr);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Creates a configuration object for the specified blocklist source.
|
|
77
|
+
* We assume that the input is a /etc/hosts file.
|
|
78
|
+
*
|
|
79
|
+
* @returns {Object} configuration object.
|
|
80
|
+
*/
|
|
81
|
+
function createConfig() {
|
|
82
|
+
const inputType = argv['input-type'] || 'hosts';
|
|
83
|
+
|
|
84
|
+
consola.debug(`Creating configuration for input ${argv.input} of type ${inputType}`);
|
|
85
|
+
|
|
86
|
+
const config = {
|
|
87
|
+
name: 'Blocklist',
|
|
88
|
+
sources: [],
|
|
89
|
+
transformations: [
|
|
90
|
+
'RemoveComments',
|
|
91
|
+
'Deduplicate',
|
|
92
|
+
'Compress',
|
|
93
|
+
'Validate',
|
|
94
|
+
'TrimLines',
|
|
95
|
+
'InsertFinalNewLine',
|
|
96
|
+
],
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
argv.input.forEach((input) => {
|
|
100
|
+
config.sources.push({
|
|
101
|
+
source: input,
|
|
102
|
+
type: inputType,
|
|
103
|
+
});
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
return config;
|
|
107
|
+
}
|
|
108
|
+
|
|
41
109
|
async function main() {
|
|
42
110
|
try {
|
|
43
|
-
|
|
44
|
-
|
|
111
|
+
if (!argv.input && !argv.config) {
|
|
112
|
+
throw new Error('Either --input or --config must be specified');
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
const config = argv.input ? createConfig() : await readConfig();
|
|
116
|
+
|
|
117
|
+
consola.debug(`Configuration: ${JSON.stringify(config, 0, 4)}`);
|
|
45
118
|
|
|
46
|
-
const configStr = (await fs.readFile(argv.config)).toString();
|
|
47
|
-
const config = JSON.parse(configStr);
|
|
48
119
|
const lines = await compile(config);
|
|
49
120
|
|
|
50
121
|
consola.info(`Writing output to ${argv.output}`);
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
const fs = require('fs');
|
|
2
|
+
const path = require('path');
|
|
3
|
+
const { version } = require('../package.json');
|
|
4
|
+
|
|
5
|
+
const PATH = '../dist';
|
|
6
|
+
const FILENAME = 'build.txt';
|
|
7
|
+
|
|
8
|
+
const buildTxt = () => {
|
|
9
|
+
const dir = path.resolve(__dirname, PATH);
|
|
10
|
+
|
|
11
|
+
if (!fs.existsSync(dir)) {
|
|
12
|
+
fs.mkdirSync(dir);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
const content = `version=${version}`;
|
|
16
|
+
fs.writeFileSync(path.resolve(__dirname, PATH, FILENAME), content);
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
buildTxt();
|