@adguard/hostlist-compiler 2.1.1 → 3.0.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/.markdownlint.json +29 -0
  2. package/.markdownlintignore +1 -0
  3. package/AGENTS.md +310 -98
  4. package/CHANGELOG.md +59 -8
  5. package/DEPLOYMENT.md +171 -0
  6. package/DEVELOPMENT.md +171 -72
  7. package/README.md +238 -143
  8. package/examples/china/README.md +2 -2
  9. package/examples/energized/README.md +4 -2
  10. package/examples/sdn/README.md +2 -2
  11. package/examples/whitelist/README.md +2 -2
  12. package/package.json +24 -18
  13. package/src/cli.js +53 -31
  14. package/src/compile-source.js +4 -3
  15. package/src/configuration.js +132 -2
  16. package/src/download.js +27 -0
  17. package/src/filter.js +2 -2
  18. package/src/index.d.ts +2 -0
  19. package/src/index.js +16 -50
  20. package/src/{transformations/ip-normalize.js → ip-normalize.js} +2 -2
  21. package/src/schemas/configuration.schema.js +137 -0
  22. package/src/source-types.js +17 -0
  23. package/src/transformations/enum.js +49 -0
  24. package/src/transformations/include.js +1 -1
  25. package/src/transformations/transform.js +40 -74
  26. package/src/transformations/validate-allow-ip-and-public-suffix.js +1 -1
  27. package/src/transformations/validate-allow-ip.js +1 -1
  28. package/src/utils.js +1284 -0
  29. package/src/validation-conflicts.js +106 -0
  30. package/test/cli-error-output.test.js +305 -0
  31. package/test/cli.test.js +47 -0
  32. package/test/compile-source.test.js +167 -7
  33. package/test/configuration.test.js +438 -0
  34. package/test/download.test.js +260 -0
  35. package/test/helpers/http-mock.js +94 -0
  36. package/test/index.test.js +497 -4
  37. package/test/{transformations/ip-normalize.test.js → ip-normalize.test.js} +1 -1
  38. package/test/resolve-include.test.js +8 -8
  39. package/test/transformations/covert-to-ascii.test.js +1 -1
  40. package/test/transformations/exclude.test.js +2 -2
  41. package/test/transformations/include.test.js +9 -3
  42. package/test/transformations/ip-alignment.test.js +1 -1
  43. package/test/transformations/transform.test.js +66 -1
  44. package/test/utils.test.js +1559 -0
  45. package/test/validation-conflicts.test.js +230 -0
  46. package/src/schemas/configuration.schema.json +0 -369
@@ -0,0 +1,260 @@
1
+ const mock = require('mock-fs');
2
+ const path = require('path');
3
+ const mockHttp = require('./helpers/http-mock');
4
+ const { download } = require('../src/download');
5
+
6
+ const testDirPath = path.resolve(__dirname, 'dir');
7
+
8
+ describe('Download', () => {
9
+ afterEach(() => {
10
+ // make sure FS is restored after running tests
11
+ mock.restore();
12
+ });
13
+
14
+ it('downloads a remote URL source', async () => {
15
+ const scope = mockHttp('https://example.org')
16
+ .get('/filter.txt')
17
+ .reply(200, 'rule1\nrule2\nrule3\n', {
18
+ 'Content-Type': 'text/plain',
19
+ });
20
+
21
+ const rules = await download('https://example.org/filter.txt');
22
+
23
+ expect(rules).toEqual(['rule1', 'rule2', 'rule3']);
24
+
25
+ // Make sure scope URLs were requested
26
+ scope.done();
27
+ });
28
+
29
+ it('downloads a source with credentials in the URL', async () => {
30
+ const credentials = Buffer.from('user:password').toString('base64');
31
+ const scope = mockHttp('https://example.org')
32
+ .get('/filter.txt', {
33
+ authorization: `Basic ${credentials}`,
34
+ })
35
+ .reply(200, 'rule1\nrule2\n', {
36
+ 'Content-Type': 'text/plain',
37
+ });
38
+
39
+ const rules = await download('https://user:password@example.org/filter.txt');
40
+
41
+ expect(rules).toEqual(['rule1', 'rule2']);
42
+
43
+ // Make sure scope URLs were requested
44
+ scope.done();
45
+ });
46
+
47
+ it('keeps a JSON-looking body as plain text', async () => {
48
+ const scope = mockHttp('https://example.org')
49
+ .get('/list.json')
50
+ .reply(200, '[]', {
51
+ 'Content-Type': 'text/plain',
52
+ });
53
+
54
+ expect(await download('https://example.org/list.json')).toEqual(['[]']);
55
+
56
+ // Make sure scope URLs were requested
57
+ scope.done();
58
+ });
59
+
60
+ it('does not leak the password into error messages', async () => {
61
+ const scope = mockHttp('https://example.org')
62
+ .get('/missing.txt')
63
+ .reply(404, 'Not Found', {
64
+ 'Content-Type': 'text/plain',
65
+ });
66
+
67
+ const error = await download('https://user:secret@example.org/missing.txt')
68
+ .catch((ex) => ex);
69
+
70
+ // Follow `errors` as well as `cause`: a dual-stack failure is an
71
+ // `AggregateError`, and its messages are in `errors`. This is the same
72
+ // traversal that `logError()` in `src/cli.js` uses.
73
+ const messages = [];
74
+ const logged = new Set();
75
+ const collect = (current) => {
76
+ if (current === null || current === undefined || logged.has(current)) {
77
+ return;
78
+ }
79
+
80
+ if (typeof current === 'object') {
81
+ logged.add(current);
82
+ }
83
+
84
+ messages.push(String(current.message || current));
85
+
86
+ if (Array.isArray(current.errors)) {
87
+ current.errors.forEach(collect);
88
+ }
89
+
90
+ collect(current.cause);
91
+ };
92
+ collect(error);
93
+
94
+ expect(error.message).toBe(
95
+ 'Response status for url https://example.org/missing.txt is invalid: 404',
96
+ );
97
+ expect(messages.join('\n')).not.toContain('secret');
98
+
99
+ // Make sure scope URLs were requested
100
+ scope.done();
101
+ });
102
+
103
+ it.each([
104
+ [
105
+ 'a triple slash',
106
+ 'https:///user:secret@example.invalid/list.txt',
107
+ 'https:///example.invalid/list.txt',
108
+ ],
109
+ [
110
+ 'a single slash',
111
+ 'http:/user:secret@example.invalid/list.txt',
112
+ 'http:/example.invalid/list.txt',
113
+ ],
114
+ [
115
+ 'no slashes',
116
+ 'https:user:secret@example.invalid/list.txt',
117
+ 'https:example.invalid/list.txt',
118
+ ],
119
+ [
120
+ 'backslashes',
121
+ 'https:\\user:secret@example.invalid\\list.txt',
122
+ 'https:\\example.invalid\\list.txt',
123
+ ],
124
+ [
125
+ 'a quote in the username',
126
+ "https:///o'brien:secret@example.invalid/list.txt",
127
+ 'https:///example.invalid/list.txt',
128
+ ],
129
+ [
130
+ 'a space in the username',
131
+ 'https:///John Smith:secret@example.invalid/list.txt',
132
+ 'https:///example.invalid/list.txt',
133
+ ],
134
+ [
135
+ 'a password that starts with a space',
136
+ 'https:///user: secret@example.invalid/list.txt',
137
+ 'https:///example.invalid/list.txt',
138
+ ],
139
+ ])('does not leak credentials from a source with %s into the error chain', async (_, source, sanitizedSource) => {
140
+ const error = await download(source).catch((ex) => ex);
141
+
142
+ const messages = [];
143
+ const logged = new Set();
144
+ const collect = (current) => {
145
+ if (current === null || current === undefined || logged.has(current)) {
146
+ return;
147
+ }
148
+
149
+ if (typeof current === 'object') {
150
+ logged.add(current);
151
+ }
152
+
153
+ messages.push(String(current.message || current));
154
+
155
+ if (Array.isArray(current.errors)) {
156
+ current.errors.forEach(collect);
157
+ }
158
+
159
+ collect(current.cause);
160
+ };
161
+ collect(error);
162
+
163
+ expect(error.message).not.toContain('secret');
164
+ // The sanitized URL must still be present, so a change that drops the
165
+ // whole message cannot pass this test.
166
+ expect(error.message).toContain(sanitizedSource);
167
+ expect(error.stack).not.toContain('secret');
168
+ expect(messages.join('\n')).not.toContain('secret');
169
+ });
170
+
171
+ it.each([
172
+ ['spaces', 'https:///user:p s s@example.invalid/list.txt', 'p s s'],
173
+ ['a quote and a space', 'https:///user:p"s s@example.invalid/list.txt', 'p"s s'],
174
+ ['two quotes', 'https:///user:p"q"@example.invalid/list.txt', 'p"q"'],
175
+ ['a tab and a space', 'https:///user:p\t s@example.invalid/list.txt', 'p\t s'],
176
+ ])('does not leak a password with %s from a malformed source into the error chain', async (_, source, password) => {
177
+ const error = await download(source).catch((ex) => ex);
178
+
179
+ expect(error.message).not.toContain(password);
180
+ expect(error.message).toContain('https:///example.invalid/list.txt');
181
+ expect(error.stack).not.toContain(password);
182
+ expect(String(error.cause && (error.cause.message || error.cause))).not.toContain(password);
183
+ });
184
+
185
+ it('does not leak the part of a password after a literal @ and a space', async () => {
186
+ // The generic scanner uses the first `@` with a prose guard, so it
187
+ // leaves `s s@...` behind. The exact source replacement must run first.
188
+ const source = 'https:///user:p@s s@example.invalid/list.txt';
189
+ const error = await download(source).catch((ex) => ex);
190
+
191
+ expect(error.message).not.toContain('s s');
192
+ expect(error.message).toContain('https:///example.invalid/list.txt');
193
+ expect(error.stack).not.toContain('s s');
194
+ });
195
+
196
+ it('downloads a local file source', async () => {
197
+ mock({
198
+ [testDirPath]: {
199
+ 'rules.txt': 'rule1\nrule2\n',
200
+ },
201
+ });
202
+
203
+ const rules = await download(path.join(testDirPath, 'rules.txt'));
204
+
205
+ expect(rules).toEqual(['rule1', 'rule2']);
206
+ });
207
+
208
+ it('resolves !#include directives in a local file source', async () => {
209
+ mock({
210
+ [testDirPath]: {
211
+ 'main.txt': 'rule1\n!#include included.txt',
212
+ 'included.txt': 'rule2\n',
213
+ },
214
+ });
215
+
216
+ const rules = await download(path.join(testDirPath, 'main.txt'));
217
+
218
+ expect(rules).toEqual(['rule1', 'rule2']);
219
+ });
220
+
221
+ it('downloads an empty local file source', async () => {
222
+ mock({
223
+ [testDirPath]: {
224
+ 'empty.txt': '',
225
+ },
226
+ });
227
+
228
+ const rules = await download(path.join(testDirPath, 'empty.txt'));
229
+
230
+ expect(rules).toEqual(['']);
231
+ });
232
+
233
+ it('downloads an empty remote source', async () => {
234
+ const scope = mockHttp('https://example.org')
235
+ .get('/empty.txt')
236
+ .reply(200, '', {
237
+ 'Content-Type': 'text/plain',
238
+ });
239
+
240
+ const rules = await download('https://example.org/empty.txt');
241
+
242
+ expect(rules).toEqual(['']);
243
+
244
+ // Make sure scope URLs were requested
245
+ scope.done();
246
+ });
247
+
248
+ it('throws an error when the source cannot be downloaded', async () => {
249
+ const scope = mockHttp('https://example.org')
250
+ .get('/missing.txt')
251
+ .reply(404, 'Not Found', {
252
+ 'Content-Type': 'text/plain',
253
+ });
254
+
255
+ await expect(download('https://example.org/missing.txt')).rejects.toThrow();
256
+
257
+ // Make sure scope URLs were requested
258
+ scope.done();
259
+ });
260
+ });
@@ -0,0 +1,94 @@
1
+ /**
2
+ * Minimal HTTP mocking helper for tests, backed by undici's MockAgent.
3
+ *
4
+ * The library under test downloads sources with the global `fetch`. The
5
+ * `undici` package installed in this project is a different copy than the one
6
+ * bundled into Node.js, so set the project's fetch implementation together
7
+ * with its MockAgent dispatcher.
8
+ *
9
+ * Keep `undici` on 7.x. Version 8.12.3 fails on Node.js 22.22.2 with
10
+ * `TypeError: removeAbortListener is not a function` when `fetch` settles a
11
+ * response, and this package supports Node.js 22.22.2. The 7.x line is also
12
+ * the major that Node.js 24 bundles.
13
+ */
14
+ const {
15
+ MockAgent,
16
+ setGlobalDispatcher,
17
+ fetch: undiciFetch,
18
+ } = require('undici');
19
+
20
+ const originalFetch = global.fetch;
21
+
22
+ global.fetch = undiciFetch;
23
+
24
+ let mockAgent;
25
+
26
+ // Create an agent per test, so `assertNoPendingInterceptors()` only sees the
27
+ // interceptors of the current test. A failed test cannot break the `done()`
28
+ // assertions of the later tests in the same file.
29
+ beforeEach(() => {
30
+ mockAgent = new MockAgent();
31
+ mockAgent.disableNetConnect();
32
+ setGlobalDispatcher(mockAgent);
33
+ });
34
+
35
+ afterEach(async () => {
36
+ await mockAgent.close();
37
+ });
38
+
39
+ afterAll(() => {
40
+ global.fetch = originalFetch;
41
+ });
42
+
43
+ /**
44
+ * Creates a mock scope for one origin.
45
+ *
46
+ * @param {string} origin - origin to mock, e.g. `https://example.org`.
47
+ * @returns {object} scope with `get()` and `done()` methods.
48
+ */
49
+ function mockHttp(origin) {
50
+ const pool = mockAgent.get(origin);
51
+
52
+ const scope = {
53
+ /**
54
+ * Adds an expected GET request.
55
+ *
56
+ * @param {string} requestPath - request path.
57
+ * @param {object} [requestHeaders] - request headers to match.
58
+ * @returns {object} interceptor with a `reply()` method.
59
+ */
60
+ get(requestPath, requestHeaders = {}) {
61
+ const interceptor = pool.intercept({
62
+ path: requestPath,
63
+ method: 'GET',
64
+ headers: requestHeaders,
65
+ });
66
+
67
+ return {
68
+ /**
69
+ * Sets the response for the expected request.
70
+ *
71
+ * @param {number} statusCode - response status code.
72
+ * @param {string} body - response body.
73
+ * @param {object} [headers] - response headers.
74
+ * @returns {object} the same scope, to allow chaining.
75
+ */
76
+ reply(statusCode, body, headers = {}) {
77
+ interceptor.reply(statusCode, body, { headers });
78
+ return scope;
79
+ },
80
+ };
81
+ },
82
+
83
+ /**
84
+ * Asserts that all expected requests were made.
85
+ */
86
+ done() {
87
+ mockAgent.assertNoPendingInterceptors();
88
+ },
89
+ };
90
+
91
+ return scope;
92
+ }
93
+
94
+ module.exports = mockHttp;