@adguard/hostlist-compiler 2.1.0 → 3.0.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dockerignore +29 -0
- package/.github/workflows/ci.yml +22 -0
- package/.github/workflows/mirror.yml +24 -0
- package/.github/workflows/prepare-release.yml +26 -0
- package/.github/workflows/publish-release.yml +44 -0
- package/.markdownlint.json +29 -0
- package/.markdownlintignore +1 -0
- package/AGENTS.md +314 -103
- package/CHANGELOG.md +76 -7
- package/DEPLOYMENT.md +171 -0
- package/DEVELOPMENT.md +174 -76
- package/README.md +240 -140
- package/examples/china/README.md +2 -2
- package/examples/energized/README.md +4 -2
- package/examples/sdn/README.md +2 -2
- package/examples/whitelist/README.md +2 -2
- package/package.json +29 -24
- package/src/cli.js +59 -33
- package/src/compile-source.js +4 -3
- package/src/configuration.js +132 -2
- package/src/download.js +27 -0
- package/src/filter.js +2 -2
- package/src/index.d.ts +2 -0
- package/src/index.js +21 -51
- package/src/{transformations/ip-normalize.js → ip-normalize.js} +2 -2
- package/src/schemas/configuration.schema.js +137 -0
- package/src/source-types.js +17 -0
- package/src/transformations/enum.js +49 -0
- package/src/transformations/include.js +1 -1
- package/src/transformations/transform.js +40 -74
- package/src/transformations/validate-allow-ip-and-public-suffix.js +1 -1
- package/src/transformations/validate-allow-ip.js +1 -1
- package/src/utils.js +1284 -0
- package/src/validation-conflicts.js +106 -0
- package/test/cli-error-output.test.js +305 -0
- package/test/cli.test.js +47 -0
- package/test/compile-source.test.js +167 -7
- package/test/configuration.test.js +438 -0
- package/test/download.test.js +260 -0
- package/test/helpers/http-mock.js +94 -0
- package/test/index.test.js +497 -4
- package/test/{transformations/ip-normalize.test.js → ip-normalize.test.js} +1 -1
- package/test/resolve-include.test.js +8 -8
- package/test/transformations/covert-to-ascii.test.js +1 -1
- package/test/transformations/exclude.test.js +2 -2
- package/test/transformations/include.test.js +9 -3
- package/test/transformations/ip-alignment.test.js +1 -1
- package/test/transformations/transform.test.js +66 -1
- package/test/utils.test.js +1559 -0
- package/test/validation-conflicts.test.js +230 -0
- package/bamboo-specs/bamboo.yaml +0 -14
- package/bamboo-specs/build.yaml +0 -92
- package/bamboo-specs/deploy.yaml +0 -38
- package/bamboo-specs/increment.yaml +0 -54
- package/bamboo-specs/permissions.yaml +0 -16
- package/bamboo-specs/test.yaml +0 -56
- package/src/schemas/configuration.schema.json +0 -369
- package/tools/build-txt.js +0 -19
package/test/index.test.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
const nock = require('nock');
|
|
2
1
|
const consola = require('consola');
|
|
2
|
+
const mockHttp = require('./helpers/http-mock');
|
|
3
3
|
const compile = require('../src/index');
|
|
4
4
|
|
|
5
5
|
describe('Hostlist compiler', () => {
|
|
6
6
|
it('compile from multiple sources', async () => {
|
|
7
7
|
// Prepare source 1
|
|
8
|
-
const scope =
|
|
8
|
+
const scope = mockHttp('https://example.org')
|
|
9
9
|
.get('/source1.txt')
|
|
10
10
|
.reply(200, '||example.org', {
|
|
11
11
|
'Content-Type': 'text/plain',
|
|
@@ -46,6 +46,499 @@ describe('Hostlist compiler', () => {
|
|
|
46
46
|
scope.done();
|
|
47
47
|
});
|
|
48
48
|
|
|
49
|
+
it('does not expose source credentials in the compiled list or logs', async () => {
|
|
50
|
+
const scope = mockHttp('https://example.org')
|
|
51
|
+
.get('/source.txt', { Authorization: 'Basic dXNlcjpzZWNyZXQ=' })
|
|
52
|
+
.reply(200, '||example.org', {
|
|
53
|
+
'Content-Type': 'text/plain',
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
const configuration = {
|
|
57
|
+
name: 'Test filter',
|
|
58
|
+
homepage: 'https://user:secret@example.org/',
|
|
59
|
+
sources: [
|
|
60
|
+
{
|
|
61
|
+
source: 'https://user:secret@example.org/source.txt',
|
|
62
|
+
},
|
|
63
|
+
],
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
const infoSpy = jest.spyOn(consola, 'info').mockImplementation(() => {});
|
|
67
|
+
|
|
68
|
+
try {
|
|
69
|
+
const list = await compile(configuration);
|
|
70
|
+
const compiledList = list.join('\n');
|
|
71
|
+
|
|
72
|
+
expect(compiledList).toContain('! Source: https://example.org/source.txt');
|
|
73
|
+
expect(compiledList).toContain('! Homepage: https://example.org/');
|
|
74
|
+
expect(compiledList).not.toContain('secret');
|
|
75
|
+
|
|
76
|
+
const infoMessages = infoSpy.mock.calls.map(([message]) => String(message));
|
|
77
|
+
infoMessages.forEach((message) => {
|
|
78
|
+
expect(message).not.toContain('secret');
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
const configurationMessage = infoMessages
|
|
82
|
+
.find((message) => message.startsWith('Configuration:'));
|
|
83
|
+
expect(configurationMessage).toContain('"source": "https://example.org/source.txt"');
|
|
84
|
+
expect(configurationMessage).toContain('"homepage": "https://example.org/"');
|
|
85
|
+
} finally {
|
|
86
|
+
infoSpy.mockRestore();
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
scope.done();
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
it('does not expose credentials with whitespace in the homepage or logs', async () => {
|
|
93
|
+
const scope = mockHttp('https://example.org')
|
|
94
|
+
.get('/source.txt')
|
|
95
|
+
.reply(200, '||example.org', {
|
|
96
|
+
'Content-Type': 'text/plain',
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
const configuration = {
|
|
100
|
+
name: 'Test filter',
|
|
101
|
+
homepage: 'https://user:p s s@example.org/',
|
|
102
|
+
sources: [
|
|
103
|
+
{
|
|
104
|
+
source: 'https://example.org/source.txt',
|
|
105
|
+
},
|
|
106
|
+
],
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
const infoSpy = jest.spyOn(consola, 'info').mockImplementation(() => {});
|
|
110
|
+
|
|
111
|
+
try {
|
|
112
|
+
const list = await compile(configuration);
|
|
113
|
+
const compiledList = list.join('\n');
|
|
114
|
+
|
|
115
|
+
expect(compiledList).toContain('! Homepage: https://example.org/');
|
|
116
|
+
expect(compiledList).not.toContain('p s s');
|
|
117
|
+
|
|
118
|
+
const infoMessages = infoSpy.mock.calls.map(([message]) => String(message));
|
|
119
|
+
infoMessages.forEach((message) => {
|
|
120
|
+
expect(message).not.toContain('p s s');
|
|
121
|
+
});
|
|
122
|
+
} finally {
|
|
123
|
+
infoSpy.mockRestore();
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
scope.done();
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
it.each([
|
|
130
|
+
['a leading space', ' https://user:secret@example.org/', ' https://example.org/'],
|
|
131
|
+
['a tab', '\thttps://user:secret@example.org/', '\thttps://example.org/'],
|
|
132
|
+
['angle brackets', '<https://user:secret@example.org/>', '<https://example.org/>'],
|
|
133
|
+
['a double quote', '"https://user:secret@example.org/"', '"https://example.org/"'],
|
|
134
|
+
['a quote in the username', "https://o'brien:secret@example.org", 'https://example.org'],
|
|
135
|
+
['a space in the username', 'https://John Smith:secret@example.org', 'https://example.org'],
|
|
136
|
+
['a space after the password colon', 'https://user: secret@example.org', 'https://example.org'],
|
|
137
|
+
])('does not expose credentials in a homepage with %s', async (_, homepage, sanitizedHomepage) => {
|
|
138
|
+
const scope = mockHttp('https://example.org')
|
|
139
|
+
.get('/source.txt')
|
|
140
|
+
.reply(200, '||example.org', {
|
|
141
|
+
'Content-Type': 'text/plain',
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
const configuration = {
|
|
145
|
+
name: 'Test filter',
|
|
146
|
+
homepage,
|
|
147
|
+
sources: [
|
|
148
|
+
{
|
|
149
|
+
source: 'https://example.org/source.txt',
|
|
150
|
+
},
|
|
151
|
+
],
|
|
152
|
+
};
|
|
153
|
+
|
|
154
|
+
const infoSpy = jest.spyOn(consola, 'info').mockImplementation(() => {});
|
|
155
|
+
|
|
156
|
+
try {
|
|
157
|
+
const list = await compile(configuration);
|
|
158
|
+
const compiledList = list.join('\n');
|
|
159
|
+
|
|
160
|
+
expect(compiledList).toContain(`! Homepage: ${sanitizedHomepage}`);
|
|
161
|
+
expect(compiledList).not.toContain('secret');
|
|
162
|
+
|
|
163
|
+
const infoMessages = infoSpy.mock.calls.map(([message]) => String(message));
|
|
164
|
+
infoMessages.forEach((message) => {
|
|
165
|
+
expect(message).not.toContain('secret');
|
|
166
|
+
});
|
|
167
|
+
} finally {
|
|
168
|
+
infoSpy.mockRestore();
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
scope.done();
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
it('does not expose a literal @ password in a path-less homepage', async () => {
|
|
175
|
+
const scope = mockHttp('https://example.org')
|
|
176
|
+
.get('/source.txt')
|
|
177
|
+
.reply(200, '||example.org', {
|
|
178
|
+
'Content-Type': 'text/plain',
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
const configuration = {
|
|
182
|
+
name: 'Test filter',
|
|
183
|
+
homepage: 'https://user:p@s s@example.org',
|
|
184
|
+
exclusions: ['@@||good.example^'],
|
|
185
|
+
sources: [
|
|
186
|
+
{
|
|
187
|
+
source: 'https://example.org/source.txt',
|
|
188
|
+
},
|
|
189
|
+
],
|
|
190
|
+
};
|
|
191
|
+
|
|
192
|
+
const infoSpy = jest.spyOn(consola, 'info').mockImplementation(() => {});
|
|
193
|
+
|
|
194
|
+
try {
|
|
195
|
+
const list = await compile(configuration);
|
|
196
|
+
const compiledList = list.join('\n');
|
|
197
|
+
|
|
198
|
+
expect(compiledList).toContain('! Homepage: https://example.org');
|
|
199
|
+
expect(compiledList).not.toContain('p@s s');
|
|
200
|
+
|
|
201
|
+
const infoMessages = infoSpy.mock.calls.map(([message]) => String(message));
|
|
202
|
+
infoMessages.forEach((message) => {
|
|
203
|
+
expect(message).not.toContain('p@s s');
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
const configurationMessage = infoMessages
|
|
207
|
+
.find((message) => message.startsWith('Configuration:'));
|
|
208
|
+
expect(configurationMessage).toContain('"homepage": "https://example.org"');
|
|
209
|
+
expect(configurationMessage).toContain('"@@||good.example^"');
|
|
210
|
+
} finally {
|
|
211
|
+
infoSpy.mockRestore();
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
scope.done();
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
it('does not corrupt the configuration log for a path-less URL and a later email', async () => {
|
|
218
|
+
const infoSpy = jest.spyOn(consola, 'info').mockImplementation(() => {});
|
|
219
|
+
|
|
220
|
+
try {
|
|
221
|
+
await compile({
|
|
222
|
+
name: 'Test filter',
|
|
223
|
+
homepage: 'https://example.org',
|
|
224
|
+
description: 'admin@example.com',
|
|
225
|
+
sources: [{ source: 'test/resources/rules.txt' }],
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
const configurationMessage = infoSpy.mock.calls
|
|
229
|
+
.map(([message]) => String(message))
|
|
230
|
+
.find((message) => message.startsWith('Configuration:'));
|
|
231
|
+
|
|
232
|
+
expect(configurationMessage).toContain('"homepage": "https://example.org"');
|
|
233
|
+
expect(configurationMessage).toContain('"description": "admin@example.com"');
|
|
234
|
+
expect(configurationMessage).toContain('"source": "test/resources/rules.txt"');
|
|
235
|
+
} finally {
|
|
236
|
+
infoSpy.mockRestore();
|
|
237
|
+
}
|
|
238
|
+
});
|
|
239
|
+
|
|
240
|
+
it('does not expose credentials in any header field', async () => {
|
|
241
|
+
const scope = mockHttp('https://example.org')
|
|
242
|
+
.get('/source.txt')
|
|
243
|
+
.reply(200, '||example.org', {
|
|
244
|
+
'Content-Type': 'text/plain',
|
|
245
|
+
});
|
|
246
|
+
|
|
247
|
+
const configuration = {
|
|
248
|
+
name: 'Filter https://user:secret@example.org/name',
|
|
249
|
+
description: 'Mirror https://user:secret@example.org/list.txt',
|
|
250
|
+
version: '1.0 https://user:secret@example.org/version',
|
|
251
|
+
license: 'See https://user:secret@example.org/license',
|
|
252
|
+
sources: [
|
|
253
|
+
{
|
|
254
|
+
name: 'Source https://user:secret@example.org/name',
|
|
255
|
+
source: 'https://example.org/source.txt',
|
|
256
|
+
},
|
|
257
|
+
],
|
|
258
|
+
};
|
|
259
|
+
|
|
260
|
+
const infoSpy = jest.spyOn(consola, 'info').mockImplementation(() => {});
|
|
261
|
+
|
|
262
|
+
try {
|
|
263
|
+
const list = await compile(configuration);
|
|
264
|
+
const compiledList = list.join('\n');
|
|
265
|
+
|
|
266
|
+
expect(compiledList).toContain('! Title: Filter https://example.org/name');
|
|
267
|
+
expect(compiledList).toContain('! Description: Mirror https://example.org/list.txt');
|
|
268
|
+
expect(compiledList).toContain('! Version: 1.0 https://example.org/version');
|
|
269
|
+
expect(compiledList).toContain('! License: See https://example.org/license');
|
|
270
|
+
expect(compiledList).toContain('! Source name: Source https://example.org/name');
|
|
271
|
+
expect(compiledList).not.toContain('secret');
|
|
272
|
+
} finally {
|
|
273
|
+
infoSpy.mockRestore();
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
scope.done();
|
|
277
|
+
});
|
|
278
|
+
|
|
279
|
+
it.each([
|
|
280
|
+
[
|
|
281
|
+
'a space in the username and after the password colon',
|
|
282
|
+
'Mirror https://John Smith: secret@example.org',
|
|
283
|
+
'Mirror https://example.org',
|
|
284
|
+
],
|
|
285
|
+
[
|
|
286
|
+
'a space in the username before trailing text',
|
|
287
|
+
'Mirror https://John Smith:sekret@example.org by admin@example.com',
|
|
288
|
+
'Mirror https://example.org by admin@example.com',
|
|
289
|
+
],
|
|
290
|
+
[
|
|
291
|
+
'a quote in the username before trailing text',
|
|
292
|
+
"Mirror https://o'brien:sekret@example.org by admin@example.com",
|
|
293
|
+
'Mirror https://example.org by admin@example.com',
|
|
294
|
+
],
|
|
295
|
+
[
|
|
296
|
+
'a whitespace password with a quote in the username',
|
|
297
|
+
"Mirror https://o'brien:pass phrase@example.org",
|
|
298
|
+
'Mirror https://example.org',
|
|
299
|
+
],
|
|
300
|
+
[
|
|
301
|
+
'a whitespace password with a space in the username',
|
|
302
|
+
'Mirror https://John Smith:pass phrase@example.org',
|
|
303
|
+
'Mirror https://example.org',
|
|
304
|
+
],
|
|
305
|
+
[
|
|
306
|
+
'a numeric password followed by punctuation',
|
|
307
|
+
'Mirror https://user:1234, secret@example.org/list.txt',
|
|
308
|
+
'Mirror https://example.org/list.txt',
|
|
309
|
+
],
|
|
310
|
+
[
|
|
311
|
+
'an initial-style username with a path',
|
|
312
|
+
'Mirror https://J. Smith:secret@example.org/list.txt',
|
|
313
|
+
'Mirror https://example.org/list.txt',
|
|
314
|
+
],
|
|
315
|
+
[
|
|
316
|
+
'an initial-style username before trailing text',
|
|
317
|
+
'Mirror https://J. Smith:sekret@example.org by admin@example.com',
|
|
318
|
+
'Mirror https://example.org by admin@example.com',
|
|
319
|
+
],
|
|
320
|
+
[
|
|
321
|
+
'a literal @ password with a quote in the username',
|
|
322
|
+
"Mirror https://o'brien:p@ssw0rd@example.org/list.txt",
|
|
323
|
+
'Mirror https://example.org/list.txt',
|
|
324
|
+
],
|
|
325
|
+
[
|
|
326
|
+
'a literal @ password with a space in the username',
|
|
327
|
+
'Mirror https://John Smith:p@ss@example.org/list.txt',
|
|
328
|
+
'Mirror https://example.org/list.txt',
|
|
329
|
+
],
|
|
330
|
+
[
|
|
331
|
+
'a literal @ password in a path-less value',
|
|
332
|
+
'Mirror https://John Smith:p@ss@example.org',
|
|
333
|
+
'Mirror https://example.org',
|
|
334
|
+
],
|
|
335
|
+
[
|
|
336
|
+
'a literal @ password followed by a space',
|
|
337
|
+
'Mirror https://user:p@ss word@example.org',
|
|
338
|
+
'Mirror https://example.org',
|
|
339
|
+
],
|
|
340
|
+
[
|
|
341
|
+
'a literal @ password followed by a quote',
|
|
342
|
+
'Mirror https://user:p@ss"word@example.org',
|
|
343
|
+
'Mirror https://example.org',
|
|
344
|
+
],
|
|
345
|
+
[
|
|
346
|
+
'a literal @ password followed by an angle bracket',
|
|
347
|
+
'Mirror https://user:p@ss<word@example.org',
|
|
348
|
+
'Mirror https://example.org',
|
|
349
|
+
],
|
|
350
|
+
])('does not expose %s in the description header', async (_, description, sanitized) => {
|
|
351
|
+
const list = await compile({
|
|
352
|
+
name: 'Test filter',
|
|
353
|
+
description,
|
|
354
|
+
sources: [{ source: 'test/resources/rules.txt' }],
|
|
355
|
+
});
|
|
356
|
+
const header = list.slice(0, 10).join('\n');
|
|
357
|
+
|
|
358
|
+
expect(header).toContain(`! Description: ${sanitized}`);
|
|
359
|
+
expect(header).not.toContain('secret');
|
|
360
|
+
});
|
|
361
|
+
|
|
362
|
+
it('keeps prose with a port in the description header', async () => {
|
|
363
|
+
const description = 'Mirror at https://example.org:8080, contact admin@example.com';
|
|
364
|
+
const list = await compile({
|
|
365
|
+
name: 'Test filter',
|
|
366
|
+
description,
|
|
367
|
+
sources: [{ source: 'test/resources/rules.txt' }],
|
|
368
|
+
});
|
|
369
|
+
const header = list.slice(0, 10).join('\n');
|
|
370
|
+
|
|
371
|
+
expect(header).toContain(`! Description: ${description}`);
|
|
372
|
+
});
|
|
373
|
+
|
|
374
|
+
it.each([
|
|
375
|
+
'https://example.org for more info, contact admin@example.org/help.',
|
|
376
|
+
'See https://example.org or mailto:admin@example.com for support',
|
|
377
|
+
'See https://example.org or e-mail:admin@example.com for support',
|
|
378
|
+
'See https://example.org or Email:admin@example.com for support',
|
|
379
|
+
'Homepage "https://example.org". Contact: admin@example.com. Thanks!',
|
|
380
|
+
'Сайт https://фильтры.рф Контакты: пишите на admin@example.org',
|
|
381
|
+
'List for https://intranet Contact: admin@example.org',
|
|
382
|
+
'List for https://my_host.org Contact: admin@example.org',
|
|
383
|
+
'List for https://x Contact: admin@example.org',
|
|
384
|
+
'https://example.org: Contact admin@example.com',
|
|
385
|
+
'https://myhost, contact: admin@example.org',
|
|
386
|
+
])('keeps prose that follows a path-less URL in the description header: %s', async (description) => {
|
|
387
|
+
const list = await compile({
|
|
388
|
+
name: 'Test filter',
|
|
389
|
+
description,
|
|
390
|
+
sources: [{ source: 'test/resources/rules.txt' }],
|
|
391
|
+
});
|
|
392
|
+
const header = list.slice(0, 10).join('\n');
|
|
393
|
+
|
|
394
|
+
expect(header).toContain(`! Description: ${description}`);
|
|
395
|
+
});
|
|
396
|
+
|
|
397
|
+
it('keeps prose around a URL in the title, version, and license headers', async () => {
|
|
398
|
+
const prose = 'https://example.org by admin@example.com';
|
|
399
|
+
const list = await compile({
|
|
400
|
+
name: `My list ${prose}`,
|
|
401
|
+
version: `1.0 ${prose}`,
|
|
402
|
+
license: `See ${prose}`,
|
|
403
|
+
sources: [{ source: 'test/resources/rules.txt' }],
|
|
404
|
+
});
|
|
405
|
+
|
|
406
|
+
expect(list).toContain(`! Title: My list ${prose}`);
|
|
407
|
+
expect(list).toContain(`! Version: 1.0 ${prose}`);
|
|
408
|
+
expect(list).toContain(`! License: See ${prose}`);
|
|
409
|
+
});
|
|
410
|
+
|
|
411
|
+
it('does not expose a quote in a source username in the compiled list or logs', async () => {
|
|
412
|
+
const authorization = `Basic ${Buffer.from("o'brien:secret").toString('base64')}`;
|
|
413
|
+
const scope = mockHttp('https://example.org')
|
|
414
|
+
.get('/source.txt', { Authorization: authorization })
|
|
415
|
+
.reply(200, '||example.org', {
|
|
416
|
+
'Content-Type': 'text/plain',
|
|
417
|
+
});
|
|
418
|
+
|
|
419
|
+
const configuration = {
|
|
420
|
+
name: 'Test filter',
|
|
421
|
+
sources: [
|
|
422
|
+
{
|
|
423
|
+
source: "https://o'brien:secret@example.org/source.txt",
|
|
424
|
+
},
|
|
425
|
+
],
|
|
426
|
+
};
|
|
427
|
+
|
|
428
|
+
const infoSpy = jest.spyOn(consola, 'info').mockImplementation(() => {});
|
|
429
|
+
|
|
430
|
+
try {
|
|
431
|
+
const list = await compile(configuration);
|
|
432
|
+
const compiledList = list.join('\n');
|
|
433
|
+
|
|
434
|
+
expect(compiledList).toContain('! Source: https://example.org/source.txt');
|
|
435
|
+
expect(compiledList).not.toContain('secret');
|
|
436
|
+
|
|
437
|
+
const infoMessages = infoSpy.mock.calls.map(([message]) => String(message));
|
|
438
|
+
infoMessages.forEach((message) => {
|
|
439
|
+
expect(message).not.toContain('secret');
|
|
440
|
+
});
|
|
441
|
+
} finally {
|
|
442
|
+
infoSpy.mockRestore();
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
scope.done();
|
|
446
|
+
});
|
|
447
|
+
|
|
448
|
+
it('does not expose a literal @ password in a path-less source', async () => {
|
|
449
|
+
const authorization = `Basic ${Buffer.from('user:p@s s').toString('base64')}`;
|
|
450
|
+
const scope = mockHttp('https://example.org')
|
|
451
|
+
.get('/', { Authorization: authorization })
|
|
452
|
+
.reply(200, '||example.org', {
|
|
453
|
+
'Content-Type': 'text/plain',
|
|
454
|
+
});
|
|
455
|
+
|
|
456
|
+
const configuration = {
|
|
457
|
+
name: 'Test filter',
|
|
458
|
+
sources: [
|
|
459
|
+
{
|
|
460
|
+
source: 'https://user:p@s s@example.org',
|
|
461
|
+
},
|
|
462
|
+
],
|
|
463
|
+
};
|
|
464
|
+
|
|
465
|
+
const infoSpy = jest.spyOn(consola, 'info').mockImplementation(() => {});
|
|
466
|
+
|
|
467
|
+
try {
|
|
468
|
+
const list = await compile(configuration);
|
|
469
|
+
const compiledList = list.join('\n');
|
|
470
|
+
|
|
471
|
+
expect(compiledList).toContain('! Source: https://example.org');
|
|
472
|
+
expect(compiledList).not.toContain('p@s s');
|
|
473
|
+
|
|
474
|
+
const infoMessages = infoSpy.mock.calls.map(([message]) => String(message));
|
|
475
|
+
infoMessages.forEach((message) => {
|
|
476
|
+
expect(message).not.toContain('p@s s');
|
|
477
|
+
});
|
|
478
|
+
expect(infoMessages).toContain('Start compiling https://example.org');
|
|
479
|
+
|
|
480
|
+
const configurationMessage = infoMessages
|
|
481
|
+
.find((message) => message.startsWith('Configuration:'));
|
|
482
|
+
expect(configurationMessage).toContain('"source": "https://example.org"');
|
|
483
|
+
} finally {
|
|
484
|
+
infoSpy.mockRestore();
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
scope.done();
|
|
488
|
+
});
|
|
489
|
+
|
|
490
|
+
it('does not expose source credentials when the configuration is invalid', async () => {
|
|
491
|
+
const configuration = {
|
|
492
|
+
name: 'Test filter',
|
|
493
|
+
sources: [
|
|
494
|
+
{
|
|
495
|
+
source: 'https://user:secret@example.org/source.txt',
|
|
496
|
+
transformations: ['NoSuchTransformation'],
|
|
497
|
+
},
|
|
498
|
+
],
|
|
499
|
+
};
|
|
500
|
+
|
|
501
|
+
const infoSpy = jest.spyOn(consola, 'info').mockImplementation(() => {});
|
|
502
|
+
|
|
503
|
+
try {
|
|
504
|
+
await expect(compile(configuration)).rejects.toThrow('Failed to validate configuration');
|
|
505
|
+
|
|
506
|
+
const infoMessages = infoSpy.mock.calls.map(([message]) => String(message));
|
|
507
|
+
infoMessages.forEach((message) => {
|
|
508
|
+
expect(message).not.toContain('secret');
|
|
509
|
+
});
|
|
510
|
+
|
|
511
|
+
const validationMessage = infoMessages
|
|
512
|
+
.find((message) => message.includes('ENUM should be equal to one of the allowed values'));
|
|
513
|
+
expect(validationMessage).toContain('https://example.org/source.txt');
|
|
514
|
+
} finally {
|
|
515
|
+
infoSpy.mockRestore();
|
|
516
|
+
}
|
|
517
|
+
});
|
|
518
|
+
|
|
519
|
+
it('does not modify the configuration passed to compile', async () => {
|
|
520
|
+
const scope = mockHttp('https://example.org')
|
|
521
|
+
.get('/source.txt')
|
|
522
|
+
.reply(200, '||example.org', {
|
|
523
|
+
'Content-Type': 'text/plain',
|
|
524
|
+
});
|
|
525
|
+
|
|
526
|
+
const configuration = {
|
|
527
|
+
name: 'Test filter',
|
|
528
|
+
sources: [
|
|
529
|
+
{
|
|
530
|
+
source: 'https://user:secret@example.org/source.txt',
|
|
531
|
+
},
|
|
532
|
+
],
|
|
533
|
+
};
|
|
534
|
+
|
|
535
|
+
await compile(configuration);
|
|
536
|
+
|
|
537
|
+
expect(configuration.sources[0].source).toBe('https://user:secret@example.org/source.txt');
|
|
538
|
+
|
|
539
|
+
scope.done();
|
|
540
|
+
});
|
|
541
|
+
|
|
49
542
|
it('throws error when validation transformations are used at both source and top level', async () => {
|
|
50
543
|
// No need to mock HTTP request - error is thrown before download
|
|
51
544
|
const configuration = {
|
|
@@ -65,7 +558,7 @@ describe('Hostlist compiler', () => {
|
|
|
65
558
|
});
|
|
66
559
|
|
|
67
560
|
it('allows validation transformation at source level only', async () => {
|
|
68
|
-
const scope =
|
|
561
|
+
const scope = mockHttp('https://example.org')
|
|
69
562
|
.get('/source.txt')
|
|
70
563
|
.reply(200, '||org^', {
|
|
71
564
|
'Content-Type': 'text/plain',
|
|
@@ -88,7 +581,7 @@ describe('Hostlist compiler', () => {
|
|
|
88
581
|
});
|
|
89
582
|
|
|
90
583
|
it('allows validation transformation at top level only', async () => {
|
|
91
|
-
const scope =
|
|
584
|
+
const scope = mockHttp('https://example.org')
|
|
92
585
|
.get('/source.txt')
|
|
93
586
|
.reply(200, '||org^', {
|
|
94
587
|
'Content-Type': 'text/plain',
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
const nock = require('nock');
|
|
2
1
|
const consola = require('consola');
|
|
2
|
+
const mockHttp = require('./helpers/http-mock');
|
|
3
3
|
const compile = require('../src/index');
|
|
4
4
|
|
|
5
5
|
describe('Hostlist compiler', () => {
|
|
@@ -20,7 +20,7 @@ last.include.com
|
|
|
20
20
|
non/valid_rule`;
|
|
21
21
|
|
|
22
22
|
// Prepare source
|
|
23
|
-
const scope =
|
|
23
|
+
const scope = mockHttp('https://example.org')
|
|
24
24
|
.get('/source1.txt')
|
|
25
25
|
.reply(200, filterContent1, {
|
|
26
26
|
'Content-Type': 'text/plain',
|
|
@@ -96,7 +96,7 @@ exclude.com
|
|
|
96
96
|
non/valid_rule`;
|
|
97
97
|
|
|
98
98
|
// Prepare source
|
|
99
|
-
const scope =
|
|
99
|
+
const scope = mockHttp('https://example.org')
|
|
100
100
|
.get('/source1.txt')
|
|
101
101
|
.reply(200, filterContent1, {
|
|
102
102
|
'Content-Type': 'text/plain',
|
|
@@ -209,7 +209,7 @@ non/valid_rule`;
|
|
|
209
209
|
||sub_test_final`;
|
|
210
210
|
|
|
211
211
|
// Prepare source
|
|
212
|
-
const scope =
|
|
212
|
+
const scope = mockHttp('https://example.org')
|
|
213
213
|
.get('/testdir/source1.txt')
|
|
214
214
|
.reply(200, filterContent1, {
|
|
215
215
|
'Content-Type': 'text/plain',
|
|
@@ -268,7 +268,7 @@ exclude.com
|
|
|
268
268
|
!#include https://example1.org/source.txt`;
|
|
269
269
|
|
|
270
270
|
// Prepare source
|
|
271
|
-
const scope =
|
|
271
|
+
const scope = mockHttp('https://example.org')
|
|
272
272
|
.get('/source1.txt')
|
|
273
273
|
.reply(200, filterContent1, {
|
|
274
274
|
'Content-Type': 'text/plain',
|
|
@@ -289,7 +289,7 @@ exclude.com
|
|
|
289
289
|
|
|
290
290
|
await expect(async () => {
|
|
291
291
|
await compile(configuration);
|
|
292
|
-
}).rejects.toThrow(
|
|
292
|
+
}).rejects.toThrow('Include url is rejected with origin');
|
|
293
293
|
|
|
294
294
|
scope.done();
|
|
295
295
|
});
|
|
@@ -303,7 +303,7 @@ exclude.com
|
|
|
303
303
|
!#include https://example.org/source.txt`;
|
|
304
304
|
|
|
305
305
|
// Prepare source
|
|
306
|
-
const scope =
|
|
306
|
+
const scope = mockHttp('https://example.org')
|
|
307
307
|
.get('/source1.txt')
|
|
308
308
|
.reply(200, filterContent, {
|
|
309
309
|
'Content-Type': 'text/plain',
|
|
@@ -356,7 +356,7 @@ exclude.com
|
|
|
356
356
|
|
|
357
357
|
it('should compile source with empty source', async () => {
|
|
358
358
|
// Prepare source
|
|
359
|
-
const scope =
|
|
359
|
+
const scope = mockHttp('https://example.org')
|
|
360
360
|
.get('/source.txt')
|
|
361
361
|
.reply(200, '', {
|
|
362
362
|
'Content-Type': 'text/plain',
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
const convertToAscii = require('../../src/transformations/covert-to-ascii');
|
|
2
2
|
|
|
3
|
-
describe('
|
|
3
|
+
describe('ConvertToAscii', () => {
|
|
4
4
|
it('convert non latin symbols in full domain to punycode', () => {
|
|
5
5
|
const rules = `||*.рус^$denyallow=example.com
|
|
6
6
|
||*.कॉम^$denyallow=example.com
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
const nock = require('nock');
|
|
2
1
|
const mock = require('mock-fs');
|
|
3
2
|
const path = require('path');
|
|
3
|
+
const mockHttp = require('../helpers/http-mock');
|
|
4
4
|
const exclude = require('../../src/transformations/exclude');
|
|
5
5
|
|
|
6
6
|
const testDirPath = path.resolve(__dirname, 'test/dir');
|
|
@@ -22,7 +22,7 @@ describe('Exclusions', () => {
|
|
|
22
22
|
|
|
23
23
|
it('exclusions sources', async () => {
|
|
24
24
|
// Mock exclusions
|
|
25
|
-
const scope =
|
|
25
|
+
const scope = mockHttp('https://example.org')
|
|
26
26
|
.get('/exclusions.txt')
|
|
27
27
|
.reply(200, 'rule1', {
|
|
28
28
|
'Content-Type': 'text/plain',
|
|
@@ -1,23 +1,29 @@
|
|
|
1
|
-
const nock = require('nock');
|
|
2
1
|
const mock = require('mock-fs');
|
|
2
|
+
const consola = require('consola');
|
|
3
3
|
const path = require('path');
|
|
4
|
+
const mockHttp = require('../helpers/http-mock');
|
|
4
5
|
const include = require('../../src/transformations/include');
|
|
5
6
|
|
|
6
7
|
const testDirPath = path.resolve(__dirname, 'test/dir');
|
|
7
8
|
const inclusionsFilePath = path.resolve(testDirPath, 'inclusions.txt');
|
|
8
9
|
|
|
9
|
-
describe('
|
|
10
|
+
describe('Inclusions', () => {
|
|
10
11
|
afterEach(() => {
|
|
11
12
|
// make sure FS is restored after running tests
|
|
12
13
|
mock.restore();
|
|
14
|
+
// restore spies so they do not leak into other tests
|
|
15
|
+
jest.restoreAllMocks();
|
|
13
16
|
});
|
|
14
17
|
|
|
15
18
|
it('simple inclusion', async () => {
|
|
19
|
+
const infoSpy = jest.spyOn(consola, 'info');
|
|
16
20
|
const rules = ['rule1', 'rule2'];
|
|
17
21
|
const filtered = await include(rules, ['rule2']);
|
|
18
22
|
|
|
19
23
|
expect(filtered).toHaveLength(1);
|
|
20
24
|
expect(filtered).toContain('rule2');
|
|
25
|
+
// the logged count must be the filtered count, not the input count
|
|
26
|
+
expect(infoSpy).toHaveBeenLastCalledWith('Included 1 rules');
|
|
21
27
|
});
|
|
22
28
|
|
|
23
29
|
it('include host-level only', async () => {
|
|
@@ -41,7 +47,7 @@ describe('Exclusions', () => {
|
|
|
41
47
|
|
|
42
48
|
it('inclusions sources', async () => {
|
|
43
49
|
// Mock inclusions sources
|
|
44
|
-
const scope =
|
|
50
|
+
const scope = mockHttp('https://example.org')
|
|
45
51
|
.get('/inclusions.txt')
|
|
46
52
|
.reply(200, 'rule1', {
|
|
47
53
|
'Content-Type': 'text/plain',
|