@adguard/hostlist-compiler 2.1.1 → 3.0.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/.markdownlint.json +29 -0
  2. package/.markdownlintignore +1 -0
  3. package/AGENTS.md +310 -98
  4. package/CHANGELOG.md +59 -8
  5. package/DEPLOYMENT.md +171 -0
  6. package/DEVELOPMENT.md +171 -72
  7. package/README.md +238 -143
  8. package/examples/china/README.md +2 -2
  9. package/examples/energized/README.md +4 -2
  10. package/examples/sdn/README.md +2 -2
  11. package/examples/whitelist/README.md +2 -2
  12. package/package.json +24 -18
  13. package/src/cli.js +53 -31
  14. package/src/compile-source.js +4 -3
  15. package/src/configuration.js +132 -2
  16. package/src/download.js +27 -0
  17. package/src/filter.js +2 -2
  18. package/src/index.d.ts +2 -0
  19. package/src/index.js +16 -50
  20. package/src/{transformations/ip-normalize.js → ip-normalize.js} +2 -2
  21. package/src/schemas/configuration.schema.js +137 -0
  22. package/src/source-types.js +17 -0
  23. package/src/transformations/enum.js +49 -0
  24. package/src/transformations/include.js +1 -1
  25. package/src/transformations/transform.js +40 -74
  26. package/src/transformations/validate-allow-ip-and-public-suffix.js +1 -1
  27. package/src/transformations/validate-allow-ip.js +1 -1
  28. package/src/utils.js +1284 -0
  29. package/src/validation-conflicts.js +106 -0
  30. package/test/cli-error-output.test.js +305 -0
  31. package/test/cli.test.js +47 -0
  32. package/test/compile-source.test.js +167 -7
  33. package/test/configuration.test.js +438 -0
  34. package/test/download.test.js +260 -0
  35. package/test/helpers/http-mock.js +94 -0
  36. package/test/index.test.js +497 -4
  37. package/test/{transformations/ip-normalize.test.js → ip-normalize.test.js} +1 -1
  38. package/test/resolve-include.test.js +8 -8
  39. package/test/transformations/covert-to-ascii.test.js +1 -1
  40. package/test/transformations/exclude.test.js +2 -2
  41. package/test/transformations/include.test.js +9 -3
  42. package/test/transformations/ip-alignment.test.js +1 -1
  43. package/test/transformations/transform.test.js +66 -1
  44. package/test/utils.test.js +1559 -0
  45. package/test/validation-conflicts.test.js +230 -0
  46. package/src/schemas/configuration.schema.json +0 -369
@@ -1,6 +1,79 @@
1
1
  const config = require('../src/configuration');
2
+ const { DEFAULT_TRANSFORMATIONS } = require('../src/transformations/transform');
3
+ const { TRANSFORMATIONS } = require('../src/transformations/enum');
4
+ const schema = require('../src/schemas/configuration.schema');
5
+
6
+ describe('createConfiguration', () => {
7
+ it('test createConfiguration with default source type and pipeline', () => {
8
+ const configuration = config.createConfiguration(['hosts.txt', 'hosts2.txt']);
9
+
10
+ expect(configuration.name).toBe('Blocklist');
11
+ expect(configuration.sources).toEqual([
12
+ { source: 'hosts.txt', type: 'hosts' },
13
+ { source: 'hosts2.txt', type: 'hosts' },
14
+ ]);
15
+ expect(configuration.transformations).toEqual([...DEFAULT_TRANSFORMATIONS]);
16
+
17
+ const ret = config.validateConfiguration(configuration);
18
+ expect(ret.valid).toBe(true);
19
+ expect(ret.errorsText).toBeNull();
20
+ });
21
+
22
+ it('test createConfiguration with the specified source type', () => {
23
+ const configuration = config.createConfiguration(['rules.txt'], 'adblock');
24
+
25
+ expect(configuration.sources).toEqual([
26
+ { source: 'rules.txt', type: 'adblock' },
27
+ ]);
28
+ });
29
+
30
+ it('test createConfiguration falls back to hosts for an empty source type', () => {
31
+ // The CLI passes `argv['input-type']` through; yargs yields an empty
32
+ // string for `-t ''`, which must behave like the flag being absent.
33
+ const configuration = config.createConfiguration(['hosts.txt'], '');
34
+
35
+ expect(configuration.sources).toEqual([
36
+ { source: 'hosts.txt', type: 'hosts' },
37
+ ]);
38
+ });
39
+
40
+ it('test createConfiguration returns an independent transformations copy', () => {
41
+ const configuration = config.createConfiguration(['hosts.txt']);
42
+
43
+ // Each config must own its pipeline, so customizing it later cannot
44
+ // leak into other configs or the frozen default constant.
45
+ expect(configuration.transformations).not.toBe(DEFAULT_TRANSFORMATIONS);
46
+ configuration.transformations.push(TRANSFORMATIONS.ConvertToAscii);
47
+ expect(config.createConfiguration(['hosts.txt']).transformations).toEqual(DEFAULT_TRANSFORMATIONS);
48
+ expect(configuration.transformations).not.toEqual(DEFAULT_TRANSFORMATIONS);
49
+ });
50
+ });
2
51
 
3
52
  describe('Configuration', () => {
53
+ it('renders the transformations allowed-values in the original schema order', () => {
54
+ // Pins the declaration order of the TRANSFORMATIONS enum: the schema's
55
+ // items.enum is Object.values(TRANSFORMATIONS), and that order is what
56
+ // the unknown-transformation error renders as the allowed-values list.
57
+ const expected = [
58
+ 'RemoveComments',
59
+ 'RemoveModifiers',
60
+ 'Compress',
61
+ 'Validate',
62
+ 'ValidateAllowIp',
63
+ 'ValidateAllowPublicSuffix',
64
+ 'ValidateAllowIpAndPublicSuffix',
65
+ 'Deduplicate',
66
+ 'InvertAllow',
67
+ 'RemoveEmptyLines',
68
+ 'TrimLines',
69
+ 'InsertFinalNewLine',
70
+ 'ConvertToAscii',
71
+ ];
72
+
73
+ expect(schema.properties.transformations.items.enum).toEqual(expected);
74
+ expect(schema.definitions.source.properties.transformations.items.enum).toEqual(expected);
75
+ });
76
+
4
77
  it('test invalid configuration', () => {
5
78
  const ret = config.validateConfiguration({
6
79
  name: 'test',
@@ -72,6 +145,371 @@ describe('Configuration', () => {
72
145
  expect(ret.errorsText).toBeTruthy();
73
146
  });
74
147
 
148
+ it('does not include source credentials in a validation error', () => {
149
+ const ret = config.validateConfiguration({
150
+ name: 'test',
151
+ sources: [
152
+ {
153
+ source: 'https://user:secret@example.org/source.txt',
154
+ transformations: [
155
+ 'NoSuchTransformation',
156
+ ],
157
+ },
158
+ ],
159
+ });
160
+
161
+ expect(ret.valid).toBe(false);
162
+ expect(ret.errorsText).not.toContain('secret');
163
+ expect(ret.errorsText).toContain('https://example.org/source.txt');
164
+ });
165
+
166
+ it.each([
167
+ [
168
+ 'a misspelled key',
169
+ {
170
+ name: 'test',
171
+ sources: [
172
+ {
173
+ source: 'test.txt',
174
+ exclusion_sources: ['https://user:secret@example.org/exclusions.txt'],
175
+ },
176
+ ],
177
+ },
178
+ ],
179
+ [
180
+ 'a string instead of a sources array',
181
+ {
182
+ name: 'test',
183
+ sources: 'https://user:secret@example.org/source.txt',
184
+ },
185
+ ],
186
+ [
187
+ 'a string instead of a source object',
188
+ {
189
+ name: 'test',
190
+ sources: ['https://user:secret@example.org/source.txt'],
191
+ },
192
+ ],
193
+ [
194
+ 'an array instead of a homepage string',
195
+ {
196
+ name: 'test',
197
+ homepage: ['https://user:secret@example.org/'],
198
+ sources: ['test.txt'],
199
+ },
200
+ ],
201
+ ])('does not include credentials in a validation error for %s', (_, configuration) => {
202
+ const ret = config.validateConfiguration(configuration);
203
+
204
+ expect(ret.valid).toBe(false);
205
+ expect(ret.errorsText).not.toContain('secret');
206
+ expect(ret.errorsText).toContain('https://example.org/');
207
+ });
208
+
209
+ it('does not include credentials with whitespace in userinfo from a malformed source', () => {
210
+ const ret = config.validateConfiguration({
211
+ name: 'test',
212
+ sources: ['https://user:p s s@example.org/source.txt'],
213
+ });
214
+
215
+ expect(ret.valid).toBe(false);
216
+ expect(ret.errorsText).not.toContain('p s s');
217
+ expect(ret.errorsText).toContain('https://example.org/source.txt');
218
+ });
219
+
220
+ it.each([
221
+ ['a quote in the username', "https://o'brien:secret@example.org/source.txt"],
222
+ ['a space in the username', 'https://John Smith:secret@example.org/source.txt'],
223
+ ['a space after the password colon', 'https://user: secret@example.org/source.txt'],
224
+ ['a literal @ and a space in the password', 'https://user:p@s s@example.org/source.txt'],
225
+ ])('does not include credentials with %s from a malformed source', (_, source) => {
226
+ const ret = config.validateConfiguration({
227
+ name: 'test',
228
+ sources: [source],
229
+ });
230
+
231
+ expect(ret.valid).toBe(false);
232
+ expect(ret.errorsText).not.toContain('secret');
233
+ expect(ret.errorsText).not.toContain('p@s s');
234
+ expect(ret.errorsText).toContain('https://example.org/source.txt');
235
+ });
236
+
237
+ it('does not include credentials from a quote username in a misspelled key', () => {
238
+ const ret = config.validateConfiguration({
239
+ name: 'test',
240
+ sources: [
241
+ {
242
+ url: "https://o'brien:secret@example.org",
243
+ },
244
+ ],
245
+ });
246
+
247
+ expect(ret.valid).toBe(false);
248
+ expect(ret.errorsText).not.toContain('secret');
249
+ expect(ret.errorsText).toContain('https://example.org');
250
+ });
251
+
252
+ it('does not include a password with a leading space in a misspelled key', () => {
253
+ const ret = config.validateConfiguration({
254
+ name: 'test',
255
+ sources: [
256
+ {
257
+ url: 'https://user: secret@example.org',
258
+ },
259
+ ],
260
+ });
261
+
262
+ expect(ret.valid).toBe(false);
263
+ expect(ret.errorsText).not.toContain('secret');
264
+ expect(ret.errorsText).toContain('https://example.org');
265
+ });
266
+
267
+ it('keeps prose that looks like a port in a validation error', () => {
268
+ const description = 'https://example.org:8080 by admin@example.com';
269
+ const ret = config.validateConfiguration({
270
+ name: 'test',
271
+ description,
272
+ sources: [
273
+ {
274
+ source: 'test.txt',
275
+ transformations: ['NoSuchTransformation'],
276
+ },
277
+ ],
278
+ });
279
+
280
+ expect(ret.valid).toBe(false);
281
+ expect(ret.errorsText).toContain(description);
282
+ });
283
+
284
+ it.each([
285
+ 'Mirror at https://example.org:8080, contact admin@example.com',
286
+ 'Mirror at https://example.org:8080, contact admin@example.com/path',
287
+ 'Mirror at https://example.org:8080. Contact admin@example.com/path',
288
+ 'Mirror at https://example.org:8080; contact admin@example.com/path',
289
+ 'Mirror at https://example.org:8080) contact admin@example.com/path',
290
+ ])('keeps prose with a port followed by punctuation in a validation error: %s', (description) => {
291
+ const ret = config.validateConfiguration({
292
+ name: 'test',
293
+ description,
294
+ sources: [
295
+ {
296
+ source: 'test.txt',
297
+ transformations: ['NoSuchTransformation'],
298
+ },
299
+ ],
300
+ });
301
+
302
+ expect(ret.valid).toBe(false);
303
+ expect(ret.errorsText).toContain(description);
304
+ });
305
+
306
+ it('keeps prose with a later URL in a validation error', () => {
307
+ const description = 'Maintained at https://example.org by john@example.com. '
308
+ + 'Forum: https://forum.example.org/';
309
+ const ret = config.validateConfiguration({
310
+ name: 'test',
311
+ description,
312
+ sources: [
313
+ {
314
+ source: 'test.txt',
315
+ transformations: ['NoSuchTransformation'],
316
+ },
317
+ ],
318
+ });
319
+
320
+ expect(ret.valid).toBe(false);
321
+ expect(ret.errorsText).toContain(description);
322
+ });
323
+
324
+ it('does not corrupt a validation error with a path-less URL and a later email', () => {
325
+ const ret = config.validateConfiguration({
326
+ name: 'test',
327
+ homepage: 'https://example.org',
328
+ description: 'admin@example.com',
329
+ sources: [
330
+ {
331
+ source: 'test.txt',
332
+ transformations: ['NoSuchTransformation'],
333
+ },
334
+ ],
335
+ });
336
+
337
+ expect(ret.valid).toBe(false);
338
+ expect(ret.errorsText).toContain('"homepage":"https://example.org"');
339
+ expect(ret.errorsText).toContain('"description":"admin@example.com"');
340
+ expect(ret.errorsText).toContain('"source":"test.txt"');
341
+ });
342
+
343
+ it('does not include a literal @ password from a path-less malformed source', () => {
344
+ const ret = config.validateConfiguration({
345
+ name: 'test',
346
+ sources: ['https://user:p@s s@example.org'],
347
+ });
348
+
349
+ expect(ret.valid).toBe(false);
350
+ expect(ret.errorsText).not.toContain('p@s s');
351
+ expect(ret.errorsText).toContain('https://example.org');
352
+ });
353
+
354
+ it('does not include credentials from a value under an unknown key', () => {
355
+ const ret = config.validateConfiguration({
356
+ name: 'test',
357
+ sources: [
358
+ {
359
+ url: 'https://John Smith: secret@example.org',
360
+ },
361
+ ],
362
+ });
363
+
364
+ expect(ret.valid).toBe(false);
365
+ expect(ret.errorsText).not.toContain('secret');
366
+ expect(ret.errorsText).toContain('https://example.org');
367
+ });
368
+
369
+ it('does not include a numeric password from a value under an unknown key', () => {
370
+ const ret = config.validateConfiguration({
371
+ name: 'test',
372
+ sources: [{ source: 'test.txt' }],
373
+ unknown: 'https://user:1234, secret@example.org/list.txt',
374
+ });
375
+
376
+ expect(ret.valid).toBe(false);
377
+ expect(ret.errorsText).not.toContain('secret');
378
+ expect(ret.errorsText).toContain('https://example.org/list.txt');
379
+ });
380
+
381
+ it('does not throw on an additional property URL key with whitespace in userinfo', () => {
382
+ const ret = config.validateConfiguration({
383
+ name: 'test',
384
+ sources: [{ source: 'test.txt' }],
385
+ 'https:John Smith:pa ss@example.org': 'value',
386
+ });
387
+
388
+ expect(ret.valid).toBe(false);
389
+ expect(ret.errorsText).not.toContain('pa ss');
390
+ expect(ret.errorsText).toContain('https:example.org');
391
+ });
392
+
393
+ it('does not throw on a nested additional property URL key with a path', () => {
394
+ const ret = config.validateConfiguration({
395
+ name: 'test',
396
+ sources: [
397
+ {
398
+ source: 'test.txt',
399
+ 'https://user:p"s@example.org/key.txt': 'value',
400
+ },
401
+ ],
402
+ });
403
+
404
+ expect(ret.valid).toBe(false);
405
+ expect(ret.errorsText).not.toContain('p"s');
406
+ expect(ret.errorsText).toContain('https://example.org/key.txt');
407
+ });
408
+
409
+ it('does not include free-text credentials from a validation error', () => {
410
+ const ret = config.validateConfiguration({
411
+ name: 'test',
412
+ description: 'Mirror https://John Smith: secret@example.org',
413
+ sources: [
414
+ {
415
+ source: 'test.txt',
416
+ transformations: ['NoSuchTransformation'],
417
+ },
418
+ ],
419
+ });
420
+
421
+ expect(ret.valid).toBe(false);
422
+ expect(ret.errorsText).not.toContain('secret');
423
+ expect(ret.errorsText).toContain('Mirror https://example.org');
424
+ });
425
+
426
+ it('keeps prose with a label after a URL in a validation error', () => {
427
+ const description = 'https://example.org team: contact@example.com';
428
+ const ret = config.validateConfiguration({
429
+ name: 'test',
430
+ description,
431
+ sources: [
432
+ {
433
+ source: 'test.txt',
434
+ transformations: ['NoSuchTransformation'],
435
+ },
436
+ ],
437
+ });
438
+
439
+ expect(ret.valid).toBe(false);
440
+ expect(ret.errorsText).toContain(description);
441
+ });
442
+
443
+ it('does not include credentials from a prefixed homepage', () => {
444
+ const ret = config.validateConfiguration({
445
+ name: 'test',
446
+ homepage: ' https://user:secret@example.org/',
447
+ sources: ['test.txt'],
448
+ transformations: ['NoSuchTransformation'],
449
+ });
450
+
451
+ expect(ret.valid).toBe(false);
452
+ expect(ret.errorsText).not.toContain('secret');
453
+ expect(ret.errorsText).toContain('https://example.org/');
454
+ });
455
+
456
+ it('does not include credentials from a URL object key in a validation error', () => {
457
+ const ret = config.validateConfiguration({
458
+ name: 'test',
459
+ sources: {
460
+ 'https://user:secret@example.org/source.txt': {},
461
+ },
462
+ });
463
+
464
+ expect(ret.valid).toBe(false);
465
+ expect(ret.errorsText).not.toContain('secret');
466
+ expect(ret.errorsText).toContain('https://example.org/source.txt');
467
+ });
468
+
469
+ it('does not include credentials from a serialized object in a validation error', () => {
470
+ const ret = config.validateConfiguration({
471
+ name: 'test',
472
+ homepage: {
473
+ toJSON() {
474
+ return 'https://user:secret@example.org/';
475
+ },
476
+ },
477
+ sources: ['test.txt'],
478
+ });
479
+
480
+ expect(ret.valid).toBe(false);
481
+ expect(ret.errorsText).not.toContain('secret');
482
+ expect(ret.errorsText).toContain('https://example.org/');
483
+ });
484
+
485
+ it('does not include a quote username from a serialized object without a path', () => {
486
+ const ret = config.validateConfiguration({
487
+ name: 'test',
488
+ homepage: {
489
+ toJSON() {
490
+ return "https://o'brien:secret@example.org";
491
+ },
492
+ },
493
+ sources: ['test.txt'],
494
+ });
495
+
496
+ expect(ret.valid).toBe(false);
497
+ expect(ret.errorsText).not.toContain('secret');
498
+ expect(ret.errorsText).toContain('https://example.org');
499
+ });
500
+
501
+ it('does not include credentials from a URL object in a validation error', () => {
502
+ const ret = config.validateConfiguration({
503
+ name: 'test',
504
+ homepage: new URL('https://user:secret@example.org/'),
505
+ sources: ['test.txt'],
506
+ });
507
+
508
+ expect(ret.valid).toBe(false);
509
+ expect(ret.errorsText).not.toContain('secret');
510
+ expect(ret.errorsText).toContain('https://example.org/');
511
+ });
512
+
75
513
  it('test ValidateAllowPublicSuffix transformation in configuration', () => {
76
514
  const ret = config.validateConfiguration({
77
515
  name: 'test',