@adguard/hostlist-compiler 2.0.0 → 2.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dockerignore +29 -0
- package/.github/workflows/ci.yml +22 -0
- package/.github/workflows/mirror.yml +24 -0
- package/.github/workflows/prepare-release.yml +26 -0
- package/.github/workflows/publish-release.yml +44 -0
- package/AGENTS.md +15 -16
- package/CHANGELOG.md +44 -0
- package/DEVELOPMENT.md +16 -17
- package/README.md +42 -3
- package/package.json +10 -11
- package/src/cli.js +6 -2
- package/src/index.d.ts +1 -0
- package/src/index.js +6 -1
- package/src/schemas/configuration.schema.json +98 -0
- package/src/transformations/ip-normalize.js +182 -0
- package/src/transformations/transform.js +6 -4
- package/src/transformations/validate-allow-ip-and-public-suffix.js +33 -0
- package/src/transformations/validate-allow-ip.js +16 -3
- package/src/transformations/validate.js +118 -53
- package/src/utils.js +137 -0
- package/test/configuration.test.js +6 -0
- package/test/transformations/ip-alignment.test.js +65 -0
- package/test/transformations/ip-normalize.test.js +331 -0
- package/test/transformations/transform.test.js +6 -0
- package/test/transformations/validate-allow-ip-and-public-suffix.test.js +167 -0
- package/test/transformations/validate-allow-ip.test.js +46 -18
- package/test/transformations/validate-allow-public-suffix.test.js +20 -0
- package/test/transformations/validate.test.js +57 -0
- package/bamboo-specs/bamboo.yaml +0 -14
- package/bamboo-specs/build.yaml +0 -92
- package/bamboo-specs/deploy.yaml +0 -38
- package/bamboo-specs/increment.yaml +0 -54
- package/bamboo-specs/permissions.yaml +0 -16
- package/bamboo-specs/test.yaml +0 -56
- package/tools/build-txt.js +0 -19
package/.dockerignore
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Version control
|
|
2
|
+
.git
|
|
3
|
+
|
|
4
|
+
# IDE / editor
|
|
5
|
+
.idea
|
|
6
|
+
.vscode
|
|
7
|
+
|
|
8
|
+
# macOS
|
|
9
|
+
.DS_Store
|
|
10
|
+
|
|
11
|
+
# Lint cache
|
|
12
|
+
.eslintcache
|
|
13
|
+
|
|
14
|
+
# Dependencies & build output
|
|
15
|
+
node_modules/
|
|
16
|
+
dist/
|
|
17
|
+
|
|
18
|
+
# Docker
|
|
19
|
+
Dockerfile*
|
|
20
|
+
|
|
21
|
+
# Logs & archives
|
|
22
|
+
*.log
|
|
23
|
+
*.tgz
|
|
24
|
+
|
|
25
|
+
# Local environment & secrets (may contain npm registry auth tokens)
|
|
26
|
+
.npmrc
|
|
27
|
+
.npmrc.*
|
|
28
|
+
.env
|
|
29
|
+
.env.*
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
description: Lints, tests, and builds on pull requests and pushes to master.
|
|
4
|
+
|
|
5
|
+
on:
|
|
6
|
+
pull_request:
|
|
7
|
+
push:
|
|
8
|
+
branches:
|
|
9
|
+
- master
|
|
10
|
+
|
|
11
|
+
# Runs on every PR and every push to master; cancel superseded runs to avoid
|
|
12
|
+
# spawning overlapping workflows that waste self-hosted runner capacity.
|
|
13
|
+
concurrency:
|
|
14
|
+
group: ci-${{ github.workflow }}-${{ github.ref }}
|
|
15
|
+
cancel-in-progress: true
|
|
16
|
+
|
|
17
|
+
jobs:
|
|
18
|
+
test:
|
|
19
|
+
uses: AdGuardSoftwareLimited/ext-shared-actions/.github/workflows/test-and-build.yml@master
|
|
20
|
+
with:
|
|
21
|
+
package_file: hostlist-compiler.tgz
|
|
22
|
+
team: filters
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
name: Mirror
|
|
2
|
+
|
|
3
|
+
description: Mirrors code to the public repo on every push to master.
|
|
4
|
+
|
|
5
|
+
on:
|
|
6
|
+
push:
|
|
7
|
+
branches:
|
|
8
|
+
- master
|
|
9
|
+
|
|
10
|
+
# `contents: read` — checkout source repo
|
|
11
|
+
# `actions: write` — disable workflows in the public mirror
|
|
12
|
+
# `id-token: write` — Octopass token for push to public repo
|
|
13
|
+
permissions:
|
|
14
|
+
contents: read
|
|
15
|
+
actions: write
|
|
16
|
+
id-token: write
|
|
17
|
+
|
|
18
|
+
jobs:
|
|
19
|
+
mirror:
|
|
20
|
+
uses: AdGuardSoftwareLimited/actions/.github/workflows/mirror.yml@master
|
|
21
|
+
with:
|
|
22
|
+
team: filters
|
|
23
|
+
target_repo: git@github.com:AdguardTeam/HostlistCompiler.git
|
|
24
|
+
secrets: inherit
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
name: Prepare release
|
|
2
|
+
|
|
3
|
+
description: >-
|
|
4
|
+
Opens a release PR that finalizes CHANGELOG.md for the given tag.
|
|
5
|
+
After merge, publish-release.yml takes over.
|
|
6
|
+
|
|
7
|
+
on:
|
|
8
|
+
workflow_dispatch:
|
|
9
|
+
inputs:
|
|
10
|
+
tag:
|
|
11
|
+
description: 'Release tag WITH "v" prefix (e.g. v2.1.0, v2.1.0-beta.1)'
|
|
12
|
+
required: true
|
|
13
|
+
type: string
|
|
14
|
+
|
|
15
|
+
# `contents: write` — push the release-bump branch
|
|
16
|
+
# `id-token: write` — Octopass OIDC for PR creation
|
|
17
|
+
permissions:
|
|
18
|
+
contents: write
|
|
19
|
+
id-token: write
|
|
20
|
+
|
|
21
|
+
jobs:
|
|
22
|
+
create-release-pr:
|
|
23
|
+
uses: AdGuardSoftwareLimited/actions/.github/workflows/create-release-pr.yml@master
|
|
24
|
+
with:
|
|
25
|
+
tag: ${{ inputs.tag }}
|
|
26
|
+
team: filters
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
name: Publish release
|
|
2
|
+
|
|
3
|
+
description: >-
|
|
4
|
+
Tags the release commit, builds, tests, publishes to npm,
|
|
5
|
+
mirrors to the public repo, drafts a GitHub Release, and notifies Slack.
|
|
6
|
+
|
|
7
|
+
# The `pull_request` trigger fires for ANY closed PR against master, including
|
|
8
|
+
# PRs closed without being merged. The called workflow (tag-from-changelog)
|
|
9
|
+
# only proceeds when `github.event.pull_request.merged == true` and the head
|
|
10
|
+
# ref starts with `release-bump/`, so an unmerged close is a safe no-op.
|
|
11
|
+
# Mirroring that guard here keeps the intent visible at the call site.
|
|
12
|
+
on:
|
|
13
|
+
pull_request:
|
|
14
|
+
types: [closed]
|
|
15
|
+
branches: [master]
|
|
16
|
+
workflow_dispatch:
|
|
17
|
+
inputs:
|
|
18
|
+
ref:
|
|
19
|
+
description: Ref to publish manually
|
|
20
|
+
required: false
|
|
21
|
+
type: string
|
|
22
|
+
|
|
23
|
+
# Serialize release runs: a manual dispatch racing a merge-triggered run must
|
|
24
|
+
# not concurrently create tags, publish to npm, or draft GitHub Releases.
|
|
25
|
+
concurrency:
|
|
26
|
+
group: publish-release
|
|
27
|
+
cancel-in-progress: false
|
|
28
|
+
|
|
29
|
+
jobs:
|
|
30
|
+
publish:
|
|
31
|
+
if: github.event_name != 'pull_request' || github.event.pull_request.merged == true
|
|
32
|
+
uses: AdGuardSoftwareLimited/ext-shared-actions/.github/workflows/publish-release.yml@master
|
|
33
|
+
with:
|
|
34
|
+
package_file: hostlist-compiler.tgz
|
|
35
|
+
npm_package_name: "@adguard/hostlist-compiler"
|
|
36
|
+
target_repo: AdguardTeam/HostlistCompiler
|
|
37
|
+
team: filters
|
|
38
|
+
slack_channel: "#adguard-filters-vcs"
|
|
39
|
+
ref: ${{ inputs.ref }}
|
|
40
|
+
secrets: inherit
|
|
41
|
+
permissions:
|
|
42
|
+
contents: write # create tag & GitHub release (tag-from-changelog, mirror-and-release)
|
|
43
|
+
id-token: write # OIDC trusted publishing to npm (deploy-to-npm, mirror-and-release)
|
|
44
|
+
actions: write # disable/enable workflows in the mirror repo (mirror-and-release)
|
package/AGENTS.md
CHANGED
|
@@ -53,10 +53,9 @@ inclusion/exclusion rules, and `!#include` directive resolution via
|
|
|
53
53
|
├── test/ # Jest test suites mirroring src/
|
|
54
54
|
│ ├── resources/ # Test fixture files
|
|
55
55
|
│ └── transformations/ # Transformation-specific tests
|
|
56
|
-
├── tools/ # Build helper scripts
|
|
57
|
-
│ └── build-txt.js # Generates transformations docs
|
|
58
56
|
├── examples/ # Example configurations (sdn, energized, etc.)
|
|
59
|
-
├──
|
|
57
|
+
├── .github/workflows/ # GitHub Actions CI (ci, mirror, prepare-release, publish-release)
|
|
58
|
+
├── Dockerfile # Multi-stage build: test-output / build-output targets
|
|
60
59
|
├── package.json # Package manifest and scripts
|
|
61
60
|
├── jest.config.js # Jest configuration
|
|
62
61
|
├── .eslintrc.js # ESLint configuration
|
|
@@ -70,25 +69,25 @@ inclusion/exclusion rules, and `!#include` directive resolution via
|
|
|
70
69
|
**Full verification sequence** (run all of these before considering a task done):
|
|
71
70
|
|
|
72
71
|
```bash
|
|
73
|
-
|
|
74
|
-
|
|
72
|
+
pnpm lint
|
|
73
|
+
pnpm test
|
|
75
74
|
```
|
|
76
75
|
|
|
77
76
|
| Action | Command |
|
|
78
77
|
|---------|---------|
|
|
79
|
-
| Install | `
|
|
80
|
-
| Lint | `
|
|
81
|
-
| Test | `
|
|
78
|
+
| Install | `pnpm install` |
|
|
79
|
+
| Lint | `pnpm lint` |
|
|
80
|
+
| Test | `pnpm test` |
|
|
82
81
|
| Run CLI (example) | `node src/cli.js -c examples/sdn/configuration.json -o filter.txt` |
|
|
83
|
-
| Version bump | `
|
|
82
|
+
| Version bump | `pnpm increment` |
|
|
84
83
|
|
|
85
84
|
## Verification
|
|
86
85
|
|
|
87
86
|
You MUST follow these rules for EVERY task you perform:
|
|
88
87
|
|
|
89
|
-
1. **Run the linter**: `
|
|
88
|
+
1. **Run the linter**: `pnpm lint`. Fix all warnings and errors before
|
|
90
89
|
proceeding. The project uses ESLint with `airbnb-base`.
|
|
91
|
-
2. **Run the tests**: `
|
|
90
|
+
2. **Run the tests**: `pnpm test`. All tests must pass. If you changed
|
|
92
91
|
behavior, verify that existing tests still reflect correct expectations.
|
|
93
92
|
3. **Check the outcome**. If either command fails, fix the issues and re-run
|
|
94
93
|
until both pass. A task is not complete until all checks are green.
|
|
@@ -99,7 +98,7 @@ For every task you perform, you MUST:
|
|
|
99
98
|
|
|
100
99
|
- **Update or add unit tests** for any changed or new code. Mirror the source
|
|
101
100
|
file structure under `test/`.
|
|
102
|
-
- **Run the full verification sequence** (`
|
|
101
|
+
- **Run the full verification sequence** (`pnpm lint` and `pnpm test`) and
|
|
103
102
|
confirm both pass.
|
|
104
103
|
- **Update `AGENTS.md`** if you change the project structure, add new build
|
|
105
104
|
commands, or discover a recurring agent mistake that should become a rule.
|
|
@@ -116,9 +115,9 @@ For every task you perform, you MUST:
|
|
|
116
115
|
|
|
117
116
|
## Contribution Instructions
|
|
118
117
|
|
|
119
|
-
1. **Install** dependencies with `
|
|
120
|
-
2. **Lint** before committing: `
|
|
121
|
-
3. **Test** before committing: `
|
|
118
|
+
1. **Install** dependencies with `pnpm install`.
|
|
119
|
+
2. **Lint** before committing: `pnpm lint`. Fix all warnings and errors.
|
|
120
|
+
3. **Test** before committing: `pnpm test`. All tests must pass. Add tests for
|
|
122
121
|
new functionality — mirror the source file structure under `test/`.
|
|
123
122
|
4. **Do not commit** generated files, `node_modules`, or editor configs.
|
|
124
123
|
5. **Do not create git commits or PRs** unless explicitly asked.
|
|
@@ -139,7 +138,7 @@ For every task you perform, you MUST:
|
|
|
139
138
|
|
|
140
139
|
### Code Quality
|
|
141
140
|
|
|
142
|
-
- **ESLint airbnb-base** rules apply — run `
|
|
141
|
+
- **ESLint airbnb-base** rules apply — run `pnpm lint` to check. See
|
|
143
142
|
`.eslintrc.js` for project-specific overrides.
|
|
144
143
|
- Use `lodash` helpers (e.g., `_.startsWith`, `_.isEmpty`, `_.trim`) where they
|
|
145
144
|
are already used — stay consistent with the existing style.
|
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,50 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
### Changed
|
|
13
|
+
|
|
14
|
+
### Deprecated
|
|
15
|
+
|
|
16
|
+
### Removed
|
|
17
|
+
|
|
18
|
+
### Fixed
|
|
19
|
+
|
|
20
|
+
### Security
|
|
21
|
+
|
|
22
|
+
## [2.1.1] - 2026-08-21
|
|
23
|
+
|
|
24
|
+
## [2.1.0] - 2026-04-16
|
|
25
|
+
|
|
26
|
+
### Added
|
|
27
|
+
|
|
28
|
+
- `ValidateAllowIpAndPublicSuffix` validation that keeps both IP address rules and rules matching whole public suffixes (e.g. `||185.149.120.173^`, `.org^`). [#126]
|
|
29
|
+
|
|
30
|
+
### Changed
|
|
31
|
+
|
|
32
|
+
`ValidateAllowIp` now normalizes incomplete IP rules to the safe format `||ip^`. [#127]
|
|
33
|
+
- 4-octet IPs without proper separators (e.g. `1.2.3.4`, `1.2.3.4^`, `|1.2.3.4^`) are normalized to `||1.2.3.4^`
|
|
34
|
+
- 3-octet subnet wildcards (e.g. `192.168.1.`, `192.168.1.*`) are allowed with `||` prefix
|
|
35
|
+
- 3-octet patterns with `^` are rejected (e.g. `192.168.1^`, `||192.168.1^`)
|
|
36
|
+
- 3-octet patterns without trailing dot/wildcard are rejected (e.g. `192.168.1`, `||192.168.1`)
|
|
37
|
+
- 1-2 octet patterns are rejected (too wide, use regex instead)
|
|
38
|
+
- Normalization (`ip-normalize.js`) is now purely a normalization step: it only rewrites valid patterns to canonical form and passes everything else through unchanged. Rejection of invalid patterns is the responsibility of the validator (`validate.js`), eliminating double-rejection.
|
|
39
|
+
|
|
40
|
+
### Fixed
|
|
41
|
+
|
|
42
|
+
- `ValidateAllowPublicSuffix` was letting through terminated ICANN TLDs (e.g. `xn--jlq61u9w7b`) because `tldts` v5.x had stale Public Suffix List data. Updated `tldts` from v5 to v7 which has the current PSL. [#128]
|
|
43
|
+
- Hardened `validHostname()` to reject hostnames that do not contain any alphanumeric character (e.g. `..`), covering a behavioral change in `tldts` v7.
|
|
44
|
+
|
|
45
|
+
[Unreleased]: https://github.com/AdguardTeam/HostlistCompiler/compare/v2.1.1...HEAD
|
|
46
|
+
[2.1.1]: https://github.com/AdguardTeam/HostlistCompiler/compare/v2.1.0...v2.1.1
|
|
47
|
+
[2.1.0]: https://github.com/AdguardTeam/HostlistCompiler/compare/v2.0.0...v2.1.0
|
|
48
|
+
[#126]: https://github.com/AdguardTeam/HostlistCompiler/issues/126
|
|
49
|
+
[#127]: https://github.com/AdguardTeam/HostlistCompiler/issues/127
|
|
50
|
+
[#128]: https://github.com/AdguardTeam/HostlistCompiler/issues/128
|
|
51
|
+
|
|
8
52
|
## [2.0.0] - 2026-04-02
|
|
9
53
|
|
|
10
54
|
### Added
|
package/DEVELOPMENT.md
CHANGED
|
@@ -7,8 +7,8 @@ For code guidelines and architectural decisions, see [AGENTS.md](AGENTS.md).
|
|
|
7
7
|
|
|
8
8
|
## Prerequisites
|
|
9
9
|
|
|
10
|
-
- **Node.js** — current LTS version (
|
|
11
|
-
- **
|
|
10
|
+
- **Node.js** — current LTS version (22.x recommended; CI builds with Node 22)
|
|
11
|
+
- **pnpm** — 10.x (`npm install -g pnpm@10`)
|
|
12
12
|
- **Git**
|
|
13
13
|
|
|
14
14
|
## Getting Started
|
|
@@ -23,26 +23,25 @@ For code guidelines and architectural decisions, see [AGENTS.md](AGENTS.md).
|
|
|
23
23
|
2. Install dependencies:
|
|
24
24
|
|
|
25
25
|
```bash
|
|
26
|
-
|
|
26
|
+
pnpm install
|
|
27
27
|
```
|
|
28
28
|
|
|
29
29
|
3. Verify the setup by running linter and tests:
|
|
30
30
|
|
|
31
31
|
```bash
|
|
32
|
-
|
|
33
|
-
|
|
32
|
+
pnpm lint
|
|
33
|
+
pnpm test
|
|
34
34
|
```
|
|
35
35
|
|
|
36
36
|
## Available Commands
|
|
37
37
|
|
|
38
38
|
| Command | Description |
|
|
39
39
|
|---------|-------------|
|
|
40
|
-
| `
|
|
41
|
-
| `
|
|
42
|
-
| `
|
|
43
|
-
| `
|
|
44
|
-
| `
|
|
45
|
-
| `yarn compile` | Run the compiler CLI (`node src/cli.js`) |
|
|
40
|
+
| `pnpm install` | Install all dependencies |
|
|
41
|
+
| `pnpm lint` | Run ESLint (`airbnb-base` config) |
|
|
42
|
+
| `pnpm test` | Run Jest test suite (`--runInBand --detectOpenHandles`) |
|
|
43
|
+
| `pnpm increment` | Bump the patch version in `package.json` |
|
|
44
|
+
| `pnpm compile` | Run the compiler CLI (`node src/cli.js`) |
|
|
46
45
|
|
|
47
46
|
## Running the Compiler Locally
|
|
48
47
|
|
|
@@ -77,8 +76,8 @@ The `examples/` directory contains several ready-made configurations (`sdn`,
|
|
|
77
76
|
4. Run the full verification sequence:
|
|
78
77
|
|
|
79
78
|
```bash
|
|
80
|
-
|
|
81
|
-
|
|
79
|
+
pnpm lint
|
|
80
|
+
pnpm test
|
|
82
81
|
```
|
|
83
82
|
|
|
84
83
|
5. Fix any issues until both commands pass.
|
|
@@ -88,7 +87,7 @@ The `examples/` directory contains several ready-made configurations (`sdn`,
|
|
|
88
87
|
Run the full test suite:
|
|
89
88
|
|
|
90
89
|
```bash
|
|
91
|
-
|
|
90
|
+
pnpm test
|
|
92
91
|
```
|
|
93
92
|
|
|
94
93
|
Run a specific test file:
|
|
@@ -109,7 +108,7 @@ interception. Test fixtures live in `test/resources/`.
|
|
|
109
108
|
### Linting
|
|
110
109
|
|
|
111
110
|
```bash
|
|
112
|
-
|
|
111
|
+
pnpm lint
|
|
113
112
|
```
|
|
114
113
|
|
|
115
114
|
ESLint is configured in `.eslintrc.js` with the `airbnb-base` preset. The
|
|
@@ -128,7 +127,7 @@ ESLint is configured in `.eslintrc.js` with the `airbnb-base` preset. The
|
|
|
128
127
|
4. Update `src/index.d.ts` with the new transformation name in the type union.
|
|
129
128
|
5. Create a test file in `test/transformations/`.
|
|
130
129
|
6. Update `CHANGELOG.md`.
|
|
131
|
-
7. Run `
|
|
130
|
+
7. Run `pnpm lint && pnpm test`.
|
|
132
131
|
|
|
133
132
|
### Updating the Configuration Schema
|
|
134
133
|
|
|
@@ -155,7 +154,7 @@ cache:
|
|
|
155
154
|
|
|
156
155
|
```bash
|
|
157
156
|
rm -f .eslintcache
|
|
158
|
-
|
|
157
|
+
pnpm lint
|
|
159
158
|
```
|
|
160
159
|
|
|
161
160
|
### Network Errors When Running the Compiler
|
package/README.md
CHANGED
|
@@ -4,6 +4,11 @@
|
|
|
4
4
|
|
|
5
5
|
This is a simple tool that makes it easier to compile a [hosts blocklist](https://adguard-dns.io/kb/general/dns-filtering-syntax/) compatible with AdGuard Home or any other AdGuard product with **DNS filtering**.
|
|
6
6
|
|
|
7
|
+
> **Note on repositories:** Active development happens in the private
|
|
8
|
+
> [AdGuardSoftwareLimited/filters-hostlist-compiler](https://github.com/AdGuardSoftwareLimited/filters-hostlist-compiler)
|
|
9
|
+
> repository; this public [AdguardTeam/HostlistCompiler](https://github.com/AdguardTeam/HostlistCompiler)
|
|
10
|
+
> repository is a read-only mirror that is updated automatically from it.
|
|
11
|
+
|
|
7
12
|
- [Usage](#usage)
|
|
8
13
|
- [Configuration](#configuration)
|
|
9
14
|
- [Command-line](#command-line)
|
|
@@ -18,6 +23,7 @@ This is a simple tool that makes it easier to compile a [hosts blocklist](https:
|
|
|
18
23
|
- [Validate](#validate)
|
|
19
24
|
- [ValidateAllowIp](#validate-allow-ip)
|
|
20
25
|
- [ValidateAllowPublicSuffix](#validate-allow-public-suffix)
|
|
26
|
+
- [ValidateAllowIpAndPublicSuffix](#validate-allow-ip-and-public-suffix)
|
|
21
27
|
- [Deduplicate](#deduplicate)
|
|
22
28
|
- [RemoveEmptyLines](#removeemptylines)
|
|
23
29
|
- [InsertFinalNewLine](#insertfinalnewline)
|
|
@@ -315,6 +321,7 @@ Here is the full list of transformations that are available:
|
|
|
315
321
|
1. `Validate`
|
|
316
322
|
1. `ValidateAllowIp`
|
|
317
323
|
1. `ValidateAllowPublicSuffix`
|
|
324
|
+
1. `ValidateAllowIpAndPublicSuffix`
|
|
318
325
|
1. `Deduplicate`
|
|
319
326
|
1. `RemoveEmptyLines`
|
|
320
327
|
1. `InsertFinalNewLine`
|
|
@@ -442,17 +449,41 @@ So here's what it does:
|
|
|
442
449
|
- Discards rules with unsupported modifiers. [Click here](https://github.com/AdguardTeam/AdGuardHome/wiki/Hosts-Blocklists#-adblock-style-syntax) to learn more about which modifiers are supported.
|
|
443
450
|
- Discards rules that are too short.
|
|
444
451
|
- Discards IP addresses. If you need to keep IP addresses, use [ValidateAllowIp](#validate-allow-ip) instead.
|
|
452
|
+
|
|
453
|
+
#### <a name="rejected-ip-patterns"></a>Rejected IP Patterns
|
|
454
|
+
|
|
455
|
+
The following IP patterns are rejected by all validation transformations (`Validate`, `ValidateAllowIp`, `ValidateAllowPublicSuffix`, `ValidateAllowIpAndPublicSuffix`) as they are either unsafe or ambiguous:
|
|
456
|
+
|
|
457
|
+
- `||192.168.1^` — 3-octet with `^` - does not work
|
|
458
|
+
- `192.168.1` — Ambiguous: would match `192.168.11`, `192.168.111`, etc.
|
|
459
|
+
- `1.2.` or `1.2.*` — Too wide (1-2 octets), use regex instead
|
|
460
|
+
|
|
445
461
|
- Removes rules that block entire top-level domains (TLDs) like `||*.org^`, unless they have specific limiting modifiers such as `$denyallow`, `$badfilter`, or `$client`.
|
|
446
462
|
Examples:
|
|
447
463
|
- `||*.org^` - this rule will be removed
|
|
448
464
|
- `||*.org^$denyallow=example.com` - this rule will be kept because it has a limiting modifier
|
|
449
|
-
If such rules must be saved, use [ValidateAllowPublicSuffix](#validate-allow-public-suffix).
|
|
465
|
+
If such rules must be saved, use [ValidateAllowPublicSuffix](#validate-allow-public-suffix) or [ValidateAllowIpAndPublicSuffix](#validate-allow-ip-and-public-suffix).
|
|
450
466
|
|
|
451
467
|
If there are comments preceding the invalid rule, they will be removed as well.
|
|
452
468
|
|
|
453
469
|
### <a name="validate-allow-ip"></a> ValidateAllowIp
|
|
454
470
|
|
|
455
|
-
This transformation
|
|
471
|
+
This transformation extends [Validate](#validate) to allow IP addresses in the lists. It also **normalizes IP rules** in Adblock-style to the safe format `||ip^`.
|
|
472
|
+
|
|
473
|
+
#### IP Rule Normalization
|
|
474
|
+
|
|
475
|
+
- `1.2.3.4` → `||1.2.3.4^` (add both separators)
|
|
476
|
+
- `1.2.3.4^` → `||1.2.3.4^` (add left anchor)
|
|
477
|
+
- `|1.2.3.4` → `||1.2.3.4^` (replace `|` with `||`, add `^`)
|
|
478
|
+
- `|1.2.3.4^` → `||1.2.3.4^` (replace `|` with `||`)
|
|
479
|
+
- `||1.2.3.4` → `||1.2.3.4^` (add right separator)
|
|
480
|
+
- `||1.2.3.4^` → (no change, already canonical)
|
|
481
|
+
- `192.168.1.` → `||192.168.1.` (3-octet subnet wildcard)
|
|
482
|
+
- `192.168.1.*` → `||192.168.1.*` (3-octet subnet wildcard)
|
|
483
|
+
|
|
484
|
+
Modifiers like `$important`, `$client`, `$denyallow`, `$badfilter` are preserved during normalization.
|
|
485
|
+
|
|
486
|
+
> **Note:** Invalid IP patterns are rejected solely by the base [Validate](#validate) logic — normalization only converts valid patterns to canonical form and passes everything else through unchanged. See [Rejected IP Patterns](#rejected-ip-patterns) for details.
|
|
456
487
|
|
|
457
488
|
### <a name="validate-allow-public-suffix"></a> ValidateAllowPublicSuffix
|
|
458
489
|
|
|
@@ -460,10 +491,18 @@ This transformation exactly repeats the behavior of [Validate](#validate), but l
|
|
|
460
491
|
|
|
461
492
|
It still filters out invalid syntax rules and unsupported modifiers, but does not reject public-suffix rules unless the rule itself is malformed.
|
|
462
493
|
|
|
463
|
-
> **Note:** Combining any `Validate`, `ValidateAllowIp`, and `
|
|
494
|
+
> **Note:** Combining any `Validate`, `ValidateAllowIp`, `ValidateAllowPublicSuffix`, and `ValidateAllowIpAndPublicSuffix` in one transformation list is not allowed and will result in an error. Each runs its own validator on the already-filtered output of the previous one, so allow-modes become silently ineffective.
|
|
464
495
|
|
|
465
496
|
> **Important:** Validation transformations also cannot be used at both source-level and top-level simultaneously. For example, if a source uses `ValidateAllowPublicSuffix` and the top-level configuration uses `Validate`, the compiler will throw an error. This is because the top-level `Validate` would override the source-level validation, making `ValidateAllowPublicSuffix` ineffective. Use validation transformations at only one level.
|
|
466
497
|
|
|
498
|
+
### <a name="validate-allow-ip-and-public-suffix"></a> ValidateAllowIpAndPublicSuffix
|
|
499
|
+
|
|
500
|
+
This transformation combines the behavior of [ValidateAllowIp](#validate-allow-ip) and [ValidateAllowPublicSuffix](#validate-allow-public-suffix). It allows both IP addresses and public suffix rules in the list.
|
|
501
|
+
|
|
502
|
+
Like `ValidateAllowIp`, it normalizes incomplete IP rules to the safe format `||ip^` before validation. See [IP Rule Normalization](#ip-rule-normalization) for details.
|
|
503
|
+
|
|
504
|
+
Like `ValidateAllowPublicSuffix`, it keeps rules that match whole public suffixes (e.g. `||hl.cn^`, `||org^`).
|
|
505
|
+
|
|
467
506
|
### <a name="deduplicate"></a> Deduplicate
|
|
468
507
|
|
|
469
508
|
This transformation simply removes the duplicates from the specified source.
|
package/package.json
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adguard/hostlist-compiler",
|
|
3
|
-
"version": "2.0.0",
|
|
4
3
|
"description": "A simple tool that compiles hosts blocklists from multiple sources",
|
|
5
4
|
"main": "src/index.js",
|
|
6
5
|
"types": "src/index.d.ts",
|
|
@@ -10,13 +9,6 @@
|
|
|
10
9
|
"bin": {
|
|
11
10
|
"hostlist-compiler": "src/cli.js"
|
|
12
11
|
},
|
|
13
|
-
"scripts": {
|
|
14
|
-
"lint": "eslint . --cache",
|
|
15
|
-
"test": "jest --runInBand --detectOpenHandles .",
|
|
16
|
-
"increment": "yarn version --patch --no-git-tag-version",
|
|
17
|
-
"build-txt": "node tools/build-txt.js",
|
|
18
|
-
"compile": "node src/cli.js"
|
|
19
|
-
},
|
|
20
12
|
"devDependencies": {
|
|
21
13
|
"@babel/core": "^7.21.0",
|
|
22
14
|
"@babel/eslint-parser": "^7.19.1",
|
|
@@ -31,11 +23,18 @@
|
|
|
31
23
|
"@adguard/filters-downloader": "^2.3.1",
|
|
32
24
|
"ajv": "^6.12.0",
|
|
33
25
|
"ajv-errors": "^1.0.1",
|
|
34
|
-
"axios": "^
|
|
26
|
+
"axios": "^1.15.0",
|
|
35
27
|
"better-ajv-errors": "^0.6.7",
|
|
36
28
|
"consola": "^2.11.3",
|
|
37
29
|
"lodash": "^4.17.15",
|
|
38
|
-
"tldts": "
|
|
30
|
+
"tldts": "7.0.28",
|
|
39
31
|
"yargs": "^15.3.0"
|
|
32
|
+
},
|
|
33
|
+
"version": "2.1.1",
|
|
34
|
+
"scripts": {
|
|
35
|
+
"lint": "eslint . --cache",
|
|
36
|
+
"test": "jest --runInBand --detectOpenHandles .",
|
|
37
|
+
"increment": "pnpm version patch --no-git-tag-version",
|
|
38
|
+
"compile": "node src/cli.js"
|
|
40
39
|
}
|
|
41
|
-
}
|
|
40
|
+
}
|
package/src/cli.js
CHANGED
|
@@ -5,6 +5,10 @@ const consola = require('consola');
|
|
|
5
5
|
const compile = require('./index');
|
|
6
6
|
const packageJson = require('../package.json');
|
|
7
7
|
|
|
8
|
+
// Version is not stored in the repo: CI stamps it into package.json at build
|
|
9
|
+
// time (see publish-release.yml). Fall back to 0.0.0 in development.
|
|
10
|
+
const packageVersion = packageJson.version || '0.0.0';
|
|
11
|
+
|
|
8
12
|
// eslint-disable-next-line import/order
|
|
9
13
|
const { argv } = require('yargs')
|
|
10
14
|
.usage('Usage: $0 [options]')
|
|
@@ -47,7 +51,7 @@ const { argv } = require('yargs')
|
|
|
47
51
|
description: 'Run with verbose logging',
|
|
48
52
|
})
|
|
49
53
|
.demandOption(['o'])
|
|
50
|
-
.version()
|
|
54
|
+
.version(packageVersion)
|
|
51
55
|
.help('h')
|
|
52
56
|
.alias('h', 'help');
|
|
53
57
|
|
|
@@ -56,7 +60,7 @@ if (argv.verbose) {
|
|
|
56
60
|
consola.level = 5;
|
|
57
61
|
}
|
|
58
62
|
|
|
59
|
-
consola.info(`Starting ${packageJson.name} v${
|
|
63
|
+
consola.info(`Starting ${packageJson.name} v${packageVersion}`);
|
|
60
64
|
|
|
61
65
|
/**
|
|
62
66
|
* Reads the configuration file and returns the parsed JSON.
|
package/src/index.d.ts
CHANGED
package/src/index.js
CHANGED
|
@@ -5,6 +5,10 @@ const compileSource = require('./compile-source');
|
|
|
5
5
|
const { transform, TRANSFORMATIONS } = require('./transformations/transform');
|
|
6
6
|
const packageJson = require('../package.json');
|
|
7
7
|
|
|
8
|
+
// Version is not stored in the repo: CI stamps it into package.json at build
|
|
9
|
+
// time (see publish-release.yml). Fall back to 0.0.0 in development.
|
|
10
|
+
const packageVersion = packageJson.version || '0.0.0';
|
|
11
|
+
|
|
8
12
|
/**
|
|
9
13
|
* Prepares list header
|
|
10
14
|
*
|
|
@@ -35,7 +39,7 @@ function prepareHeader(configuration) {
|
|
|
35
39
|
lines.push('!');
|
|
36
40
|
|
|
37
41
|
// Compiler info
|
|
38
|
-
lines.push(`! Compiled by ${packageJson.name} v${
|
|
42
|
+
lines.push(`! Compiled by ${packageJson.name} v${packageVersion}`);
|
|
39
43
|
lines.push('!');
|
|
40
44
|
return lines;
|
|
41
45
|
}
|
|
@@ -74,6 +78,7 @@ function checkCrossLevelValidationConflicts(configuration) {
|
|
|
74
78
|
TRANSFORMATIONS.Validate,
|
|
75
79
|
TRANSFORMATIONS.ValidateAllowIp,
|
|
76
80
|
TRANSFORMATIONS.ValidateAllowPublicSuffix,
|
|
81
|
+
TRANSFORMATIONS.ValidateAllowIpAndPublicSuffix,
|
|
77
82
|
];
|
|
78
83
|
|
|
79
84
|
const topLevelTransformations = configuration.transformations || [];
|