@adguard/hostlist-compiler 2.1.1 → 3.0.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/.markdownlint.json +29 -0
  2. package/.markdownlintignore +1 -0
  3. package/AGENTS.md +310 -98
  4. package/CHANGELOG.md +59 -8
  5. package/DEPLOYMENT.md +171 -0
  6. package/DEVELOPMENT.md +171 -72
  7. package/README.md +238 -143
  8. package/examples/china/README.md +2 -2
  9. package/examples/energized/README.md +4 -2
  10. package/examples/sdn/README.md +2 -2
  11. package/examples/whitelist/README.md +2 -2
  12. package/package.json +24 -18
  13. package/src/cli.js +53 -31
  14. package/src/compile-source.js +4 -3
  15. package/src/configuration.js +132 -2
  16. package/src/download.js +27 -0
  17. package/src/filter.js +2 -2
  18. package/src/index.d.ts +2 -0
  19. package/src/index.js +16 -50
  20. package/src/{transformations/ip-normalize.js → ip-normalize.js} +2 -2
  21. package/src/schemas/configuration.schema.js +137 -0
  22. package/src/source-types.js +17 -0
  23. package/src/transformations/enum.js +49 -0
  24. package/src/transformations/include.js +1 -1
  25. package/src/transformations/transform.js +40 -74
  26. package/src/transformations/validate-allow-ip-and-public-suffix.js +1 -1
  27. package/src/transformations/validate-allow-ip.js +1 -1
  28. package/src/utils.js +1284 -0
  29. package/src/validation-conflicts.js +106 -0
  30. package/test/cli-error-output.test.js +305 -0
  31. package/test/cli.test.js +47 -0
  32. package/test/compile-source.test.js +167 -7
  33. package/test/configuration.test.js +438 -0
  34. package/test/download.test.js +260 -0
  35. package/test/helpers/http-mock.js +94 -0
  36. package/test/index.test.js +497 -4
  37. package/test/{transformations/ip-normalize.test.js → ip-normalize.test.js} +1 -1
  38. package/test/resolve-include.test.js +8 -8
  39. package/test/transformations/covert-to-ascii.test.js +1 -1
  40. package/test/transformations/exclude.test.js +2 -2
  41. package/test/transformations/include.test.js +9 -3
  42. package/test/transformations/ip-alignment.test.js +1 -1
  43. package/test/transformations/transform.test.js +66 -1
  44. package/test/utils.test.js +1559 -0
  45. package/test/validation-conflicts.test.js +230 -0
  46. package/src/schemas/configuration.schema.json +0 -369
@@ -1,4 +1,6 @@
1
+ /* eslint-disable class-methods-use-this, max-classes-per-file */
1
2
  const utils = require('../src/utils');
3
+ const configurationSchema = require('../src/schemas/configuration.schema');
2
4
 
3
5
  const { Wildcard } = utils;
4
6
 
@@ -19,6 +21,1489 @@ describe('Wildcard', () => {
19
21
  });
20
22
  });
21
23
 
24
+ describe('stripUrlCredentials', () => {
25
+ it.each([
26
+ [
27
+ 'https://example.org/list.txt',
28
+ 'https://example.org/list.txt',
29
+ ],
30
+ [
31
+ 'https://user:secret@example.org/list.txt',
32
+ 'https://example.org/list.txt',
33
+ ],
34
+ [
35
+ 'https://user@example.org/list.txt',
36
+ 'https://example.org/list.txt',
37
+ ],
38
+ [
39
+ 'https://user%40name:p%40ss%3Aword@example.org/list.txt',
40
+ 'https://example.org/list.txt',
41
+ ],
42
+ [
43
+ '//user:secret@example.org/list.txt',
44
+ '//example.org/list.txt',
45
+ ],
46
+ [
47
+ 'http:/user:secret@example.org/list.txt',
48
+ 'http:/example.org/list.txt',
49
+ ],
50
+ [
51
+ 'https:user:secret@example.org/list.txt',
52
+ 'https:example.org/list.txt',
53
+ ],
54
+ [
55
+ 'https:\\user:secret@example.org\\list.txt',
56
+ 'https:\\example.org\\list.txt',
57
+ ],
58
+ [
59
+ 'https:\\\\user:secret@example.org\\list.txt',
60
+ 'https:\\\\example.org\\list.txt',
61
+ ],
62
+ [
63
+ 'HTTPS://user:secret@example.org/list.txt',
64
+ 'HTTPS://example.org/list.txt',
65
+ ],
66
+ [
67
+ 'See https://user:secret@example.org/list.txt and https://other:secret@example.net/other.txt',
68
+ 'See https://example.org/list.txt and https://example.net/other.txt',
69
+ ],
70
+ [
71
+ 'https://proxy.example.org/fetch?u=https://user:secret@raw.example.org/list.txt',
72
+ 'https://proxy.example.org/fetch?u=https://raw.example.org/list.txt',
73
+ ],
74
+ [
75
+ 'https://proxy.example.org/r/https://user:secret@raw.example.org/list.txt',
76
+ 'https://proxy.example.org/r/https://raw.example.org/list.txt',
77
+ ],
78
+ [
79
+ 'https://example.org/a,https://user:secret@raw.example.org/list.txt',
80
+ 'https://example.org/a,https://raw.example.org/list.txt',
81
+ ],
82
+ [
83
+ 'https://user:secret@example.org:8443/list.txt',
84
+ 'https://example.org:8443/list.txt',
85
+ ],
86
+ [
87
+ 'https://user:secret@[2001:db8::1]:8443/list.txt?query=1#fragment',
88
+ 'https://[2001:db8::1]:8443/list.txt?query=1#fragment',
89
+ ],
90
+ [
91
+ 'https://user:secret@example.org?query=1#fragment',
92
+ 'https://example.org?query=1#fragment',
93
+ ],
94
+ [
95
+ 'https://user:secret@example.org#fragment',
96
+ 'https://example.org#fragment',
97
+ ],
98
+ [
99
+ 'https://example.org/list@with@at.txt',
100
+ 'https://example.org/list@with@at.txt',
101
+ ],
102
+ [
103
+ 'https://example.org/a//b@c',
104
+ 'https://example.org/a//b@c',
105
+ ],
106
+ [
107
+ "https://user:p'ss@example.org/list.txt",
108
+ 'https://example.org/list.txt',
109
+ ],
110
+ [
111
+ "https://o'brien:p@ssw0rd@example.org/list.txt",
112
+ 'https://example.org/list.txt',
113
+ ],
114
+ [
115
+ "https://o'brien:p@ssw0rd@example.org",
116
+ 'https://example.org',
117
+ ],
118
+ [
119
+ 'https://John Smith:p@ss@example.org/list.txt',
120
+ 'https://example.org/list.txt',
121
+ ],
122
+ [
123
+ 'https://John Smith:p@ss@example.org',
124
+ 'https://example.org',
125
+ ],
126
+ [
127
+ 'https://J. Smith:secret@example.org/list.txt',
128
+ 'https://example.org/list.txt',
129
+ ],
130
+ [
131
+ 'https://J. Smith:secret@example.org',
132
+ 'https://example.org',
133
+ ],
134
+ [
135
+ 'https://admin user:secret@example.org',
136
+ 'https://example.org',
137
+ ],
138
+ [
139
+ 'https://john smith:secret@example.org',
140
+ 'https://example.org',
141
+ ],
142
+ [
143
+ '"https://o\'brien:secret@example.org"',
144
+ '"https://example.org"',
145
+ ],
146
+ [
147
+ '<https://John Smith:secret@example.org>',
148
+ '<https://example.org>',
149
+ ],
150
+ [
151
+ 'https://user:p"s@example.org/list.txt',
152
+ 'https://example.org/list.txt',
153
+ ],
154
+ [
155
+ 'https://user:p<s@example.org/list.txt',
156
+ 'https://example.org/list.txt',
157
+ ],
158
+ [
159
+ 'https://user:p>s@example.org/list.txt',
160
+ 'https://example.org/list.txt',
161
+ ],
162
+ [
163
+ 'https://user:p s@example.org/list.txt',
164
+ 'https://example.org/list.txt',
165
+ ],
166
+ [
167
+ 'user:secret@example.org/list.txt',
168
+ 'user:secret@example.org/list.txt',
169
+ ],
170
+ [
171
+ 'test/dir/list.txt',
172
+ 'test/dir/list.txt',
173
+ ],
174
+ [
175
+ 'C:\\user:secret@example.org\\list.txt',
176
+ 'C:\\user:secret@example.org\\list.txt',
177
+ ],
178
+ [
179
+ 'C:/user@corp/list.txt',
180
+ 'C:/user@corp/list.txt',
181
+ ],
182
+ [
183
+ 'c:/user:secret@corp/list.txt',
184
+ 'c:/user:secret@corp/list.txt',
185
+ ],
186
+ [
187
+ 'ftp:/user:secret@example.org/list.txt',
188
+ 'ftp:/example.org/list.txt',
189
+ ],
190
+ [
191
+ 'ftp://user:secret@example.org/list.txt',
192
+ 'ftp://example.org/list.txt',
193
+ ],
194
+ [
195
+ 'https://user:p s s@example.org/list.txt',
196
+ 'https://example.org/list.txt',
197
+ ],
198
+ [
199
+ ' https:///user:p s s@example.invalid/list.txt',
200
+ ' https:///example.invalid/list.txt',
201
+ ],
202
+ [
203
+ 'https://user:p"s s@example.org/list.txt',
204
+ 'https://example.org/list.txt',
205
+ ],
206
+ [
207
+ 'https://user:p"q"@example.org/list.txt',
208
+ 'https://example.org/list.txt',
209
+ ],
210
+ [
211
+ 'https://example.org is maintained by user@example.com',
212
+ 'https://example.org is maintained by user@example.com',
213
+ ],
214
+ [
215
+ 'Сайт https://фильтры.рф Контакты: пишите на admin@example.org',
216
+ 'Сайт https://фильтры.рф Контакты: пишите на admin@example.org',
217
+ ],
218
+ [
219
+ 'List for https://intranet Contact: admin@example.org',
220
+ 'List for https://intranet Contact: admin@example.org',
221
+ ],
222
+ [
223
+ 'List for https://intranet Contact:admin@example.org',
224
+ 'List for https://intranet Contact:admin@example.org',
225
+ ],
226
+ [
227
+ 'List for https://my_host.org Contact: admin@example.org',
228
+ 'List for https://my_host.org Contact: admin@example.org',
229
+ ],
230
+ [
231
+ 'List for https://x Contact: admin@example.org',
232
+ 'List for https://x Contact: admin@example.org',
233
+ ],
234
+ [
235
+ 'List for https://x 联系: admin@example.org',
236
+ 'List for https://x 联系: admin@example.org',
237
+ ],
238
+ [
239
+ 'https://example.org, see foo@bar.com',
240
+ 'https://example.org, see foo@bar.com',
241
+ ],
242
+ [
243
+ 'https://user:secret@example.org and a@b',
244
+ 'https://example.org and a@b',
245
+ ],
246
+ [
247
+ 'https://user:secret@example.org and https://other:secret@example.net/x',
248
+ 'https://example.org and https://example.net/x',
249
+ ],
250
+ [
251
+ 'https://user:secret@example.org and https://other:secret@example.net',
252
+ 'https://example.org and https://example.net',
253
+ ],
254
+ [
255
+ 'https://user:secret@example.org and a@b',
256
+ 'https://example.org and a@b',
257
+ ],
258
+ [
259
+ 'https://John Smith: secret@example.org',
260
+ 'https://example.org',
261
+ ],
262
+ [
263
+ 'See https://John Smith: secret@example.org',
264
+ 'See https://example.org',
265
+ ],
266
+ [
267
+ 'https://user:1234, secret@example.org',
268
+ 'https://example.org',
269
+ ],
270
+ [
271
+ 'https://user:1234, secret@example.org/list.txt',
272
+ 'https://example.org/list.txt',
273
+ ],
274
+ [
275
+ 'See https://user:1234, secret@example.org/list.txt',
276
+ 'See https://example.org/list.txt',
277
+ ],
278
+ [
279
+ 'https://example.org:8080 by admin@example.com',
280
+ 'https://example.org:8080 by admin@example.com',
281
+ ],
282
+ [
283
+ 'https://example.org:8080 by admin@example.com/path',
284
+ 'https://example.org:8080 by admin@example.com/path',
285
+ ],
286
+ [
287
+ 'https://localhost:8080, contact admin@example.com',
288
+ 'https://localhost:8080, contact admin@example.com',
289
+ ],
290
+ [
291
+ 'https://192.168.1.1:8080, contact admin@example.com',
292
+ 'https://192.168.1.1:8080, contact admin@example.com',
293
+ ],
294
+ [
295
+ 'Mirror at https://example.org:8080, contact admin@example.com',
296
+ 'Mirror at https://example.org:8080, contact admin@example.com',
297
+ ],
298
+ [
299
+ 'Mirror at https://example.org:8080, contact admin@example.com/path',
300
+ 'Mirror at https://example.org:8080, contact admin@example.com/path',
301
+ ],
302
+ [
303
+ 'Mirror at https://example.org:8080. Contact admin@example.com/path',
304
+ 'Mirror at https://example.org:8080. Contact admin@example.com/path',
305
+ ],
306
+ [
307
+ 'Mirror at https://example.org:8080; contact admin@example.com/path',
308
+ 'Mirror at https://example.org:8080; contact admin@example.com/path',
309
+ ],
310
+ [
311
+ 'Mirror at https://example.org:8080) contact admin@example.com/path',
312
+ 'Mirror at https://example.org:8080) contact admin@example.com/path',
313
+ ],
314
+ [
315
+ 'Maintained at https://example.org by john@example.com. Forum: https://forum.example.org/',
316
+ 'Maintained at https://example.org by john@example.com. Forum: https://forum.example.org/',
317
+ ],
318
+ [
319
+ 'https://John Smith:sekret@example.org by admin@example.com',
320
+ 'https://example.org by admin@example.com',
321
+ ],
322
+ [
323
+ "https://o'brien:sekret@example.org by admin@example.com",
324
+ 'https://example.org by admin@example.com',
325
+ ],
326
+ [
327
+ "https://o'brien:pass phrase@example.org",
328
+ 'https://example.org',
329
+ ],
330
+ [
331
+ 'https://John Smith:pass phrase@example.org',
332
+ 'https://example.org',
333
+ ],
334
+ [
335
+ "https://o'brien:pass phrase@example.org by admin@example.com",
336
+ 'https://example.org by admin@example.com',
337
+ ],
338
+ [
339
+ 'https://example.org for more info, contact admin@example.org/help.',
340
+ 'https://example.org for more info, contact admin@example.org/help.',
341
+ ],
342
+ [
343
+ 'See https://example.org or mailto:admin@example.com for support',
344
+ 'See https://example.org or mailto:admin@example.com for support',
345
+ ],
346
+ [
347
+ 'See https://example.org or e-mail:admin@example.com for support',
348
+ 'See https://example.org or e-mail:admin@example.com for support',
349
+ ],
350
+ [
351
+ 'See https://example.org or Email:admin@example.com for support',
352
+ 'See https://example.org or Email:admin@example.com for support',
353
+ ],
354
+ [
355
+ 'Homepage "https://example.org". Contact: admin@example.com. Thanks!',
356
+ 'Homepage "https://example.org". Contact: admin@example.com. Thanks!',
357
+ ],
358
+ [
359
+ 'Filter list for Corp. Homepage: https://example.org. Contact: admin@example.com',
360
+ 'Filter list for Corp. Homepage: https://example.org. Contact: admin@example.com',
361
+ ],
362
+ [
363
+ 'Mirror of https://example.org, maintainer: admin@example.com',
364
+ 'Mirror of https://example.org, maintainer: admin@example.com',
365
+ ],
366
+ [
367
+ '(https://example.org) Contact: admin@example.com',
368
+ '(https://example.org) Contact: admin@example.com',
369
+ ],
370
+ [
371
+ 'https://[2001:db8::1] by admin@example.com',
372
+ 'https://[2001:db8::1] by admin@example.com',
373
+ ],
374
+ [
375
+ 'https://[2001:db8::1]. Contact: admin@example.com',
376
+ 'https://[2001:db8::1]. Contact: admin@example.com',
377
+ ],
378
+ [
379
+ 'https://user:secret@[2001:db8::1] by admin@example.com',
380
+ 'https://[2001:db8::1] by admin@example.com',
381
+ ],
382
+ [
383
+ 'https://user:secret@[2001:db8::1]. Contact: admin@example.com',
384
+ 'https://[2001:db8::1]. Contact: admin@example.com',
385
+ ],
386
+ [
387
+ 'https://user:p@ss word@example.org',
388
+ 'https://example.org',
389
+ ],
390
+ [
391
+ 'https://user:p@ss"word@example.org',
392
+ 'https://example.org',
393
+ ],
394
+ [
395
+ 'https://user:p@ss<word@example.org',
396
+ 'https://example.org',
397
+ ],
398
+ [
399
+ 'https://user:p@ss word@example.org/list.txt',
400
+ 'https://example.org/list.txt',
401
+ ],
402
+ [
403
+ 'https://user:p@ss word@example.org by admin@example.com',
404
+ 'https://example.org by admin@example.com',
405
+ ],
406
+ [
407
+ 'https://example.org: Contact admin@example.com',
408
+ 'https://example.org: Contact admin@example.com',
409
+ ],
410
+ [
411
+ 'https://myhost, contact: admin@example.org',
412
+ 'https://myhost, contact: admin@example.org',
413
+ ],
414
+ [
415
+ 'https://user, name:secret@example.org',
416
+ 'https://example.org',
417
+ ],
418
+ ])('strips credentials from %s', (url, expected) => {
419
+ expect(utils.stripUrlCredentials(url)).toBe(expected);
420
+ });
421
+
422
+ it('returns a non-string value unchanged', () => {
423
+ expect(utils.stripUrlCredentials(undefined)).toBeUndefined();
424
+ expect(utils.stripUrlCredentials(null)).toBeNull();
425
+ });
426
+
427
+ it('does not treat a double slash inside a path as a new URL', () => {
428
+ const text = 'https://example.org/a//b@c';
429
+
430
+ expect(utils.stripUrlCredentials(text)).toBe(text);
431
+ expect(utils.stripUrlCredentials(`${text} and https://user:secret@example.net/x`))
432
+ .toBe(`${text} and https://example.net/x`);
433
+ });
434
+
435
+ it('does not change a rendered configuration that has no credentials', () => {
436
+ const configuration = {
437
+ homepage: 'https://example.org',
438
+ exclusions: ['@@||good.example^'],
439
+ sources: [{ source: 'lists/a.txt' }],
440
+ };
441
+ const dump = JSON.stringify(configuration);
442
+
443
+ expect(utils.stripUrlCredentials(dump)).toBe(dump);
444
+ });
445
+
446
+ it('handles long strings in linear time', () => {
447
+ // Jest does not fail a synchronous test that runs past its timeout,
448
+ // so the test needs a wall-clock bound. The bound is generous: a
449
+ // quadratic scan needs about 19 seconds for this input, while a linear
450
+ // scan takes milliseconds. A loaded runner cannot fail the bound.
451
+ // The credentialed URL reaches the port heuristic, which used to retry
452
+ // from every position inside the long password.
453
+ const plainText = 'a'.repeat(200_000);
454
+ const longPassword = 'b'.repeat(200_000);
455
+ const credentialedUrl = `https://reader:${longPassword}@example.org/list.txt`;
456
+ // Every URL start used to rescan the rest of the string for the
457
+ // authority end, and the strict parser scanned back to index 0 when
458
+ // the input had no `@`. Both scans must use the lookup tables.
459
+ const repeatedUrlStarts = 'https:a '.repeat(20_000);
460
+ // The value parser must not scan the text up to a far `@` for every
461
+ // URL start.
462
+ const proseWithEmail = `${'https:example.org '.repeat(10_000)}admin@example.com`;
463
+ // The strict malformed-userinfo scan must stay within the value. It
464
+ // used to walk back to the last `@` in the whole string for every URL
465
+ // start, which is quadratic on many at signs.
466
+ const manyAtSigns = 'https:x#a@b '.repeat(20_000);
467
+ const startedAt = Date.now();
468
+
469
+ expect(utils.stripUrlCredentials(plainText)).toBe(plainText);
470
+ expect(utils.stripUrlCredentials(credentialedUrl)).toBe('https://example.org/list.txt');
471
+ expect(utils.stripUrlCredentials(repeatedUrlStarts)).toBe(repeatedUrlStarts);
472
+ expect(utils.stripKnownUrlCredentials(repeatedUrlStarts)).toBe(repeatedUrlStarts);
473
+ expect(utils.stripUrlCredentials(proseWithEmail)).toBe(proseWithEmail);
474
+ expect(utils.stripKnownUrlCredentials(manyAtSigns)).toBe(manyAtSigns);
475
+ expect(Date.now() - startedAt).toBeLessThan(3000);
476
+ });
477
+ });
478
+
479
+ describe('stripUrlValueCredentials', () => {
480
+ it.each([
481
+ [
482
+ 'https://user:secret@example.org/list.txt',
483
+ 'https://example.org/list.txt',
484
+ ],
485
+ [
486
+ 'https://user:p s s@example.org/list.txt',
487
+ 'https://example.org/list.txt',
488
+ ],
489
+ [
490
+ 'https://user:p"s@example.org/list.txt',
491
+ 'https://example.org/list.txt',
492
+ ],
493
+ [
494
+ "https://user:p'ss@example.org/list.txt",
495
+ 'https://example.org/list.txt',
496
+ ],
497
+ [
498
+ '//user:p s s@example.org/list.txt',
499
+ '//example.org/list.txt',
500
+ ],
501
+ [
502
+ 'https:user:p s s@example.org/list.txt',
503
+ 'https:example.org/list.txt',
504
+ ],
505
+ [
506
+ 'https:\\user:p s s@example.org\\list.txt',
507
+ 'https:\\example.org\\list.txt',
508
+ ],
509
+ [
510
+ 'https://example.org/a//b@c',
511
+ 'https://example.org/a//b@c',
512
+ ],
513
+ [
514
+ 'C:\\user:secret@example.org\\list.txt',
515
+ 'C:\\user:secret@example.org\\list.txt',
516
+ ],
517
+ [
518
+ 'C:/user@corp/list.txt',
519
+ 'C:/user@corp/list.txt',
520
+ ],
521
+ [
522
+ ' https://user:secret@example.org/list.txt',
523
+ ' https://example.org/list.txt',
524
+ ],
525
+ [
526
+ '\thttps://user:secret@example.org/list.txt',
527
+ '\thttps://example.org/list.txt',
528
+ ],
529
+ [
530
+ '<https://user:secret@example.org/>',
531
+ '<https://example.org/>',
532
+ ],
533
+ [
534
+ '"https://user:secret@example.org/"',
535
+ '"https://example.org/"',
536
+ ],
537
+ [
538
+ 'See https://user:secret@example.org/list.txt',
539
+ 'See https://example.org/list.txt',
540
+ ],
541
+ [
542
+ 'https://user:p"s s@example.org/list.txt',
543
+ 'https://example.org/list.txt',
544
+ ],
545
+ [
546
+ 'https://user:p"q"@example.org/list.txt',
547
+ 'https://example.org/list.txt',
548
+ ],
549
+ [
550
+ 'https://user:p\t s@example.org/list.txt',
551
+ 'https://example.org/list.txt',
552
+ ],
553
+ [
554
+ 'https://example.org is maintained by user@example.com',
555
+ 'https://example.org is maintained by user@example.com',
556
+ ],
557
+ [
558
+ 'Сайт https://фильтры.рф Контакты: пишите на admin@example.org',
559
+ 'Сайт https://фильтры.рф Контакты: пишите на admin@example.org',
560
+ ],
561
+ [
562
+ 'List for https://intranet Contact: admin@example.org',
563
+ 'List for https://intranet Contact: admin@example.org',
564
+ ],
565
+ [
566
+ 'List for https://intranet Contact:admin@example.org',
567
+ 'List for https://intranet Contact:admin@example.org',
568
+ ],
569
+ [
570
+ 'List for https://my_host.org Contact: admin@example.org',
571
+ 'List for https://my_host.org Contact: admin@example.org',
572
+ ],
573
+ [
574
+ 'List for https://x Contact: admin@example.org',
575
+ 'List for https://x Contact: admin@example.org',
576
+ ],
577
+ [
578
+ 'List for https://x 联系: admin@example.org',
579
+ 'List for https://x 联系: admin@example.org',
580
+ ],
581
+ [
582
+ 'https://example.org, see foo@bar.com',
583
+ 'https://example.org, see foo@bar.com',
584
+ ],
585
+ [
586
+ "https://o'brien:secret@example.org/list.txt",
587
+ 'https://example.org/list.txt',
588
+ ],
589
+ [
590
+ 'https://John Smith:secret@example.org/list.txt',
591
+ 'https://example.org/list.txt',
592
+ ],
593
+ [
594
+ "https://o'brien:p@ssw0rd@example.org/list.txt",
595
+ 'https://example.org/list.txt',
596
+ ],
597
+ [
598
+ "https://o'brien:p@ssw0rd@example.org",
599
+ 'https://example.org',
600
+ ],
601
+ [
602
+ 'https://John Smith:p@ss@example.org/list.txt',
603
+ 'https://example.org/list.txt',
604
+ ],
605
+ [
606
+ 'https://John Smith:p@ss@example.org',
607
+ 'https://example.org',
608
+ ],
609
+ [
610
+ 'https://J. Smith:secret@example.org/list.txt',
611
+ 'https://example.org/list.txt',
612
+ ],
613
+ [
614
+ 'https://user: secret@example.org/list.txt',
615
+ 'https://example.org/list.txt',
616
+ ],
617
+ [
618
+ "https://o'brien:secret@example.org",
619
+ 'https://example.org',
620
+ ],
621
+ [
622
+ 'https://John Smith:secret@example.org',
623
+ 'https://example.org',
624
+ ],
625
+ [
626
+ 'https://J. Smith:secret@example.org',
627
+ 'https://example.org',
628
+ ],
629
+ [
630
+ 'https://admin user:secret@example.org',
631
+ 'https://example.org',
632
+ ],
633
+ [
634
+ 'https://john smith:secret@example.org',
635
+ 'https://example.org',
636
+ ],
637
+ [
638
+ 'https://J. Smith:secret@example.org by admin@example.com',
639
+ 'https://example.org by admin@example.com',
640
+ ],
641
+ [
642
+ 'https://user: secret@example.org',
643
+ 'https://example.org',
644
+ ],
645
+ [
646
+ 'https://example.org team: contact@example.com',
647
+ 'https://example.org team: contact@example.com',
648
+ ],
649
+ [
650
+ 'https://John Smith: secret@example.org',
651
+ 'https://example.org',
652
+ ],
653
+ [
654
+ 'See https://John Smith: secret@example.org',
655
+ 'See https://example.org',
656
+ ],
657
+ [
658
+ 'https://user:1234, secret@example.org',
659
+ 'https://example.org',
660
+ ],
661
+ [
662
+ 'https://user:1234, secret@example.org/list.txt',
663
+ 'https://example.org/list.txt',
664
+ ],
665
+ [
666
+ 'See https://user:1234, secret@example.org/list.txt',
667
+ 'See https://example.org/list.txt',
668
+ ],
669
+ [
670
+ 'https://example.org:8080 by admin@example.com',
671
+ 'https://example.org:8080 by admin@example.com',
672
+ ],
673
+ [
674
+ 'https://example.org:8080 by admin@example.com/path',
675
+ 'https://example.org:8080 by admin@example.com/path',
676
+ ],
677
+ [
678
+ 'https://localhost:8080, contact admin@example.com',
679
+ 'https://localhost:8080, contact admin@example.com',
680
+ ],
681
+ [
682
+ 'https://192.168.1.1:8080, contact admin@example.com',
683
+ 'https://192.168.1.1:8080, contact admin@example.com',
684
+ ],
685
+ [
686
+ 'Mirror at https://example.org:8080, contact admin@example.com',
687
+ 'Mirror at https://example.org:8080, contact admin@example.com',
688
+ ],
689
+ [
690
+ 'Mirror at https://example.org:8080, contact admin@example.com/path',
691
+ 'Mirror at https://example.org:8080, contact admin@example.com/path',
692
+ ],
693
+ [
694
+ 'Mirror at https://example.org:8080. Contact admin@example.com/path',
695
+ 'Mirror at https://example.org:8080. Contact admin@example.com/path',
696
+ ],
697
+ [
698
+ 'Mirror at https://example.org:8080; contact admin@example.com/path',
699
+ 'Mirror at https://example.org:8080; contact admin@example.com/path',
700
+ ],
701
+ [
702
+ 'Mirror at https://example.org:8080) contact admin@example.com/path',
703
+ 'Mirror at https://example.org:8080) contact admin@example.com/path',
704
+ ],
705
+ [
706
+ 'Maintained at https://example.org by john@example.com. Forum: https://forum.example.org/',
707
+ 'Maintained at https://example.org by john@example.com. Forum: https://forum.example.org/',
708
+ ],
709
+ [
710
+ 'https://John Smith:sekret@example.org by admin@example.com',
711
+ 'https://example.org by admin@example.com',
712
+ ],
713
+ [
714
+ "https://o'brien:sekret@example.org by admin@example.com",
715
+ 'https://example.org by admin@example.com',
716
+ ],
717
+ [
718
+ "https://o'brien:pass phrase@example.org",
719
+ 'https://example.org',
720
+ ],
721
+ [
722
+ 'https://John Smith:pass phrase@example.org',
723
+ 'https://example.org',
724
+ ],
725
+ [
726
+ "https://o'brien:pass phrase@example.org by admin@example.com",
727
+ 'https://example.org by admin@example.com',
728
+ ],
729
+ [
730
+ 'https://example.org for more info, contact admin@example.org/help.',
731
+ 'https://example.org for more info, contact admin@example.org/help.',
732
+ ],
733
+ [
734
+ 'See https://example.org or mailto:admin@example.com for support',
735
+ 'See https://example.org or mailto:admin@example.com for support',
736
+ ],
737
+ [
738
+ 'See https://example.org or e-mail:admin@example.com for support',
739
+ 'See https://example.org or e-mail:admin@example.com for support',
740
+ ],
741
+ [
742
+ 'See https://example.org or Email:admin@example.com for support',
743
+ 'See https://example.org or Email:admin@example.com for support',
744
+ ],
745
+ [
746
+ 'Homepage "https://example.org". Contact: admin@example.com. Thanks!',
747
+ 'Homepage "https://example.org". Contact: admin@example.com. Thanks!',
748
+ ],
749
+ [
750
+ 'Filter list for Corp. Homepage: https://example.org. Contact: admin@example.com',
751
+ 'Filter list for Corp. Homepage: https://example.org. Contact: admin@example.com',
752
+ ],
753
+ [
754
+ 'Mirror of https://example.org, maintainer: admin@example.com',
755
+ 'Mirror of https://example.org, maintainer: admin@example.com',
756
+ ],
757
+ [
758
+ '(https://example.org) Contact: admin@example.com',
759
+ '(https://example.org) Contact: admin@example.com',
760
+ ],
761
+ [
762
+ 'https://[2001:db8::1] by admin@example.com',
763
+ 'https://[2001:db8::1] by admin@example.com',
764
+ ],
765
+ [
766
+ 'https://[2001:db8::1]. Contact: admin@example.com',
767
+ 'https://[2001:db8::1]. Contact: admin@example.com',
768
+ ],
769
+ [
770
+ 'https://user:secret@[2001:db8::1] by admin@example.com',
771
+ 'https://[2001:db8::1] by admin@example.com',
772
+ ],
773
+ [
774
+ 'https://user:secret@[2001:db8::1]. Contact: admin@example.com',
775
+ 'https://[2001:db8::1]. Contact: admin@example.com',
776
+ ],
777
+ [
778
+ 'https://user:secret@example.org and a@b',
779
+ 'https://example.org and a@b',
780
+ ],
781
+ [
782
+ 'https://proxy.example.org/fetch?u=https://user:secret@raw.example.org/list.txt',
783
+ 'https://proxy.example.org/fetch?u=https://raw.example.org/list.txt',
784
+ ],
785
+ [
786
+ 'https://proxy.example.org/r/https://user:secret@raw.example.org/list.txt',
787
+ 'https://proxy.example.org/r/https://raw.example.org/list.txt',
788
+ ],
789
+ [
790
+ 'https://example.org/a,https://user:secret@raw.example.org/list.txt',
791
+ 'https://example.org/a,https://raw.example.org/list.txt',
792
+ ],
793
+ [
794
+ 'https://user:p@ss word@example.org',
795
+ 'https://example.org',
796
+ ],
797
+ [
798
+ 'https://user:p@ss"word@example.org',
799
+ 'https://example.org',
800
+ ],
801
+ [
802
+ 'https://user:p@ss<word@example.org',
803
+ 'https://example.org',
804
+ ],
805
+ [
806
+ 'https://user:p@ss word@example.org/list.txt',
807
+ 'https://example.org/list.txt',
808
+ ],
809
+ [
810
+ 'https://user:p@ss word@example.org by admin@example.com',
811
+ 'https://example.org by admin@example.com',
812
+ ],
813
+ [
814
+ 'https://example.org: Contact admin@example.com',
815
+ 'https://example.org: Contact admin@example.com',
816
+ ],
817
+ [
818
+ 'https://myhost, contact: admin@example.org',
819
+ 'https://myhost, contact: admin@example.org',
820
+ ],
821
+ [
822
+ 'https://user, name:secret@example.org',
823
+ 'https://example.org',
824
+ ],
825
+ ])('strips credentials from %s', (url, expected) => {
826
+ expect(utils.stripUrlValueCredentials(url)).toBe(expected);
827
+ });
828
+
829
+ it('returns a non-string value unchanged', () => {
830
+ expect(utils.stripUrlValueCredentials(undefined)).toBeUndefined();
831
+ expect(utils.stripUrlValueCredentials(null)).toBeNull();
832
+ });
833
+
834
+ it('strips credentials when the path contains @', () => {
835
+ expect(utils.stripUrlValueCredentials('https://John Smith:secret@example.org/list@with@at.txt'))
836
+ .toBe('https://example.org/list@with@at.txt');
837
+ expect(utils.stripUrlValueCredentials("https://o'brien:secret@example.org/list@with@at.txt"))
838
+ .toBe('https://example.org/list@with@at.txt');
839
+ });
840
+ });
841
+
842
+ describe('credential sanitizer prose families', () => {
843
+ // One row per host shape and separator. A prose tail after a path-less URL
844
+ // must stay unchanged, while the same value with userinfo must lose only
845
+ // the userinfo. The cross product closes the class that single rows missed:
846
+ // `https://example.org: Contact ...` and
847
+ // `https://myhost, contact: ...` both regressed while their neighbours
848
+ // passed.
849
+ const hosts = [
850
+ 'example.org',
851
+ 'intranet',
852
+ 'my_host.org',
853
+ 'фильтры.рф',
854
+ '[2001:db8::1]',
855
+ '127.0.0.1',
856
+ ];
857
+ const separators = [' ', ':', ',', ';', '.'];
858
+ const rows = hosts.flatMap((host) => separators.map((separator) => {
859
+ const prose = `https://${host}${separator} Contact: admin@example.org`;
860
+ const credentialed = `https://user:secret@${host}${separator} Contact: admin@example.org`;
861
+
862
+ return [host, separator, prose, credentialed];
863
+ }));
864
+
865
+ it.each(rows)('keeps prose after a %s host with the %j separator', (host, separator, prose) => {
866
+ expect(utils.stripUrlCredentials(prose)).toBe(prose);
867
+ expect(utils.stripUrlValueCredentials(prose)).toBe(prose);
868
+ });
869
+
870
+ it.each(rows)('redacts userinfo before a %s host with the %j separator', (host, separator, prose, credentialed) => {
871
+ expect(utils.stripUrlCredentials(credentialed)).toBe(prose);
872
+ expect(utils.stripUrlValueCredentials(credentialed)).toBe(prose);
873
+ });
874
+ });
875
+
876
+ describe('stripKnownUrlCredentials', () => {
877
+ it.each([
878
+ [
879
+ 'https://user:secret@example.org/list.txt',
880
+ 'https://example.org/list.txt',
881
+ ],
882
+ [
883
+ 'https://user:secret@example.org',
884
+ 'https://example.org',
885
+ ],
886
+ [
887
+ "https://o'brien:secret@example.org/list.txt",
888
+ 'https://example.org/list.txt',
889
+ ],
890
+ [
891
+ "https://o'brien:secret@example.org",
892
+ 'https://example.org',
893
+ ],
894
+ [
895
+ 'https://John Smith:secret@example.org/list.txt',
896
+ 'https://example.org/list.txt',
897
+ ],
898
+ [
899
+ 'https://John Smith:secret@example.org',
900
+ 'https://example.org',
901
+ ],
902
+ [
903
+ "https://o'brien:p@ssw0rd@example.org/list.txt",
904
+ 'https://example.org/list.txt',
905
+ ],
906
+ [
907
+ "https://o'brien:p@ssw0rd@example.org",
908
+ 'https://example.org',
909
+ ],
910
+ [
911
+ 'https://John Smith:p@ss@example.org/list.txt',
912
+ 'https://example.org/list.txt',
913
+ ],
914
+ [
915
+ 'https://John Smith:p@ss@example.org',
916
+ 'https://example.org',
917
+ ],
918
+ [
919
+ 'https://J. Smith:secret@example.org/list.txt',
920
+ 'https://example.org/list.txt',
921
+ ],
922
+ [
923
+ 'https://J. Smith:secret@example.org',
924
+ 'https://example.org',
925
+ ],
926
+ [
927
+ 'https://admin user:secret@example.org',
928
+ 'https://example.org',
929
+ ],
930
+ [
931
+ 'https://john smith:secret@example.org',
932
+ 'https://example.org',
933
+ ],
934
+ [
935
+ 'https://user: secret@example.org/list.txt',
936
+ 'https://example.org/list.txt',
937
+ ],
938
+ [
939
+ 'https://user: secret@example.org',
940
+ 'https://example.org',
941
+ ],
942
+ [
943
+ 'https://user:p@s s@example.invalid/list.txt',
944
+ 'https://example.invalid/list.txt',
945
+ ],
946
+ [
947
+ ' https://user:p s s@example.invalid/list.txt',
948
+ ' https://example.invalid/list.txt',
949
+ ],
950
+ [
951
+ '<https://user:p"s s@example.org/>',
952
+ '<https://example.org/>',
953
+ ],
954
+ [
955
+ 'See https://user:p"q"@example.org/list.txt',
956
+ 'See https://example.org/list.txt',
957
+ ],
958
+ [
959
+ 'https://example.org:8080/list.txt',
960
+ 'https://example.org:8080/list.txt',
961
+ ],
962
+ [
963
+ 'C:\\user:secret@example.org\\list.txt',
964
+ 'C:\\user:secret@example.org\\list.txt',
965
+ ],
966
+ [
967
+ 'C:/user@corp/list.txt',
968
+ 'C:/user@corp/list.txt',
969
+ ],
970
+ [
971
+ 'https://proxy.example.org/fetch?u=https://user:secret@raw.example.org/list.txt',
972
+ 'https://proxy.example.org/fetch?u=https://raw.example.org/list.txt',
973
+ ],
974
+ [
975
+ 'https://proxy.example.org/r/https://user:secret@raw.example.org/list.txt',
976
+ 'https://proxy.example.org/r/https://raw.example.org/list.txt',
977
+ ],
978
+ [
979
+ 'https://example.org/a,https://user:secret@raw.example.org/list.txt',
980
+ 'https://example.org/a,https://raw.example.org/list.txt',
981
+ ],
982
+ ])('strips credentials from %s', (url, expected) => {
983
+ expect(utils.stripKnownUrlCredentials(url)).toBe(expected);
984
+ });
985
+
986
+ it.each([
987
+ [
988
+ 'https://user:pa?ss@example.org/f.txt',
989
+ 'https://example.org/f.txt',
990
+ ],
991
+ [
992
+ 'https://user:pa#ss@example.org/f.txt',
993
+ 'https://example.org/f.txt',
994
+ ],
995
+ [
996
+ 'http://admin:#Secret1@localhost/f.txt',
997
+ 'http://localhost/f.txt',
998
+ ],
999
+ [
1000
+ 'http://admin:?Secret1@localhost/f.txt',
1001
+ 'http://localhost/f.txt',
1002
+ ],
1003
+ [
1004
+ 'https://admin:2024#Secret1@example.org/list.txt',
1005
+ 'https://example.org/list.txt',
1006
+ ],
1007
+ [
1008
+ 'https://admin:2024?Secret1@example.org/list.txt',
1009
+ 'https://example.org/list.txt',
1010
+ ],
1011
+ [
1012
+ 'https://admin:2024#Secret1@b',
1013
+ 'https://b',
1014
+ ],
1015
+ [
1016
+ 'http://user:#Secret1@fileserver?format=adblock',
1017
+ 'http://fileserver?format=adblock',
1018
+ ],
1019
+ [
1020
+ 'http://user:?Secret1@fileserver',
1021
+ 'http://fileserver',
1022
+ ],
1023
+ [
1024
+ 'http://user:#Secret1@fileserver#section',
1025
+ 'http://fileserver#section',
1026
+ ],
1027
+ [
1028
+ 'https://admin:2024#Secret1@fileserver/list.txt',
1029
+ 'https://fileserver/list.txt',
1030
+ ],
1031
+ [
1032
+ 'https://admin:2024#Secret1@example.org/path@x',
1033
+ 'https://example.org/path@x',
1034
+ ],
1035
+ [
1036
+ 'https://proxy.example.org/fetch?u=https://user:pa?ss@raw.example.org/list.txt',
1037
+ 'https://proxy.example.org/fetch?u=https://raw.example.org/list.txt',
1038
+ ],
1039
+ ])('strips a malformed password that contains ? or #: %s', (url, expected) => {
1040
+ expect(utils.stripKnownUrlCredentials(url)).toBe(expected);
1041
+ });
1042
+
1043
+ it.each([
1044
+ 'https://example.org:8080#contact@example.org',
1045
+ 'https://example.org:8080?x@y',
1046
+ 'https://example.org:?email=user@example.com',
1047
+ 'https://localhost:8080#contact@example.org',
1048
+ 'https://[::1]:8080#contact@example.org',
1049
+ 'https://myhost:8080/path?email=user@example.com',
1050
+ 'https://myhost:8080/path#contact@example.org',
1051
+ ])('keeps a valid url with a port and an at sign in the query or fragment: %s', (url) => {
1052
+ expect(utils.stripKnownUrlCredentials(url)).toBe(url);
1053
+ });
1054
+
1055
+ it('treats a single-label port with a query at sign as userinfo', () => {
1056
+ // A digit-first password and a port are lexically identical. The
1057
+ // query form has no path to tell them apart, so the password form
1058
+ // wins to avoid a leak. The path form is kept by the path bound in
1059
+ // the malformed-userinfo scan.
1060
+ expect(utils.stripKnownUrlCredentials('https://myhost:8080?email=user@example.com'))
1061
+ .toBe('https://example.com');
1062
+ });
1063
+
1064
+ it('returns a non-string value unchanged', () => {
1065
+ expect(utils.stripKnownUrlCredentials(undefined)).toBeUndefined();
1066
+ expect(utils.stripKnownUrlCredentials(null)).toBeNull();
1067
+ });
1068
+ });
1069
+
1070
+ describe('stripConfigurationCredentials', () => {
1071
+ it('returns a sanitized copy without changing the original configuration', () => {
1072
+ const configuration = {
1073
+ name: 'Test filter',
1074
+ homepage: 'https://user:secret@example.org/',
1075
+ exclusions_sources: ['https://user:secret@example.org/exclusions.txt'],
1076
+ inclusions_sources: ['https://user:secret@example.org/inclusions.txt'],
1077
+ sources: [
1078
+ {
1079
+ name: 'source 1',
1080
+ source: 'https://user:secret@example.org/list.txt',
1081
+ exclusions_sources: ['https://user:secret@example.org/source-exclusions.txt'],
1082
+ inclusions_sources: ['https://user:secret@example.org/source-inclusions.txt'],
1083
+ },
1084
+ ],
1085
+ };
1086
+
1087
+ const sanitized = utils.stripConfigurationCredentials(configuration);
1088
+
1089
+ expect(sanitized).not.toBe(configuration);
1090
+ expect(sanitized).not.toEqual(configuration);
1091
+ expect(sanitized.name).toBe('Test filter');
1092
+ expect(sanitized.homepage).toBe('https://example.org/');
1093
+ expect(sanitized.exclusions_sources).toEqual(['https://example.org/exclusions.txt']);
1094
+ expect(sanitized.inclusions_sources).toEqual(['https://example.org/inclusions.txt']);
1095
+ expect(sanitized.sources[0].source).toBe('https://example.org/list.txt');
1096
+ expect(sanitized.sources[0].exclusions_sources).toEqual(['https://example.org/source-exclusions.txt']);
1097
+ expect(sanitized.sources[0].inclusions_sources).toEqual(['https://example.org/source-inclusions.txt']);
1098
+
1099
+ expect(configuration.homepage).toBe('https://user:secret@example.org/');
1100
+ expect(configuration.sources[0].source).toBe('https://user:secret@example.org/list.txt');
1101
+ });
1102
+
1103
+ it('sanitizes known URL fields with whitespace and quotes in userinfo', () => {
1104
+ const configuration = {
1105
+ homepage: 'https://user:p s s@example.org/',
1106
+ sources: [
1107
+ {
1108
+ source: 'https://user:p s s@example.org/list.txt',
1109
+ exclusions_sources: ['https://user:p s s@example.org/exclusions.txt'],
1110
+ inclusions_sources: ['https://user:p"s@example.org/inclusions.txt'],
1111
+ },
1112
+ ],
1113
+ };
1114
+
1115
+ const sanitized = utils.stripConfigurationCredentials(configuration);
1116
+
1117
+ expect(sanitized.homepage).toBe('https://example.org/');
1118
+ expect(sanitized.sources[0].source).toBe('https://example.org/list.txt');
1119
+ expect(sanitized.sources[0].exclusions_sources).toEqual(['https://example.org/exclusions.txt']);
1120
+ expect(sanitized.sources[0].inclusions_sources).toEqual(['https://example.org/inclusions.txt']);
1121
+ expect(configuration.homepage).toBe('https://user:p s s@example.org/');
1122
+ });
1123
+
1124
+ it('sanitizes known URL fields with a quote or space in the username', () => {
1125
+ const configuration = {
1126
+ homepage: "https://o'brien:secret@example.org",
1127
+ sources: [
1128
+ {
1129
+ source: 'https://John Smith:secret@example.org/list.txt',
1130
+ },
1131
+ ],
1132
+ exclusions_sources: ['https://user: secret@example.org/exclusions.txt'],
1133
+ inclusions_sources: 'https://user:p@s s@example.org/inclusions.txt',
1134
+ };
1135
+
1136
+ const sanitized = utils.stripConfigurationCredentials(configuration);
1137
+
1138
+ expect(sanitized.homepage).toBe('https://example.org');
1139
+ expect(sanitized.sources[0].source).toBe('https://example.org/list.txt');
1140
+ expect(sanitized.exclusions_sources).toEqual(['https://example.org/exclusions.txt']);
1141
+ expect(sanitized.inclusions_sources).toBe('https://example.org/inclusions.txt');
1142
+ expect(configuration.homepage).toBe("https://o'brien:secret@example.org");
1143
+ expect(configuration.sources[0].source).toBe('https://John Smith:secret@example.org/list.txt');
1144
+ });
1145
+
1146
+ it('sanitizes path-less known URL fields with a literal @ in the password', () => {
1147
+ const configuration = {
1148
+ homepage: 'https://user:p@s s@example.org',
1149
+ sources: [
1150
+ {
1151
+ source: 'https://user:p@s s@example.org',
1152
+ exclusions_sources: ['https://user:p@s s@example.org'],
1153
+ inclusions_sources: 'https://user:p@s s@example.org',
1154
+ },
1155
+ ],
1156
+ };
1157
+
1158
+ const sanitized = utils.stripConfigurationCredentials(configuration);
1159
+
1160
+ expect(sanitized.homepage).toBe('https://example.org');
1161
+ expect(sanitized.sources[0].source).toBe('https://example.org');
1162
+ expect(sanitized.sources[0].exclusions_sources).toEqual(['https://example.org']);
1163
+ expect(sanitized.sources[0].inclusions_sources).toBe('https://example.org');
1164
+ expect(configuration.homepage).toBe('https://user:p@s s@example.org');
1165
+ });
1166
+
1167
+ it('keeps prose that looks like a port', () => {
1168
+ const configuration = {
1169
+ description: 'https://example.org:8080 by admin@example.com',
1170
+ };
1171
+
1172
+ const sanitized = utils.stripConfigurationCredentials(configuration);
1173
+
1174
+ expect(sanitized.description).toBe(configuration.description);
1175
+ });
1176
+
1177
+ it('keeps prose with a port on a host-like token and punctuation', () => {
1178
+ const configuration = {
1179
+ description: 'Mirror at https://example.org:8080, contact admin@example.com',
1180
+ };
1181
+
1182
+ const sanitized = utils.stripConfigurationCredentials(configuration);
1183
+
1184
+ expect(sanitized.description).toBe(configuration.description);
1185
+ });
1186
+
1187
+ it('keeps prose that starts a label after a URL', () => {
1188
+ const configuration = {
1189
+ description: 'https://example.org team: contact@example.com',
1190
+ };
1191
+
1192
+ const sanitized = utils.stripConfigurationCredentials(configuration);
1193
+
1194
+ expect(sanitized.description).toBe(configuration.description);
1195
+ });
1196
+
1197
+ it('sanitizes a description with a space in the username and after the password colon', () => {
1198
+ const configuration = {
1199
+ description: 'Mirror https://John Smith: secret@example.org',
1200
+ };
1201
+
1202
+ const sanitized = utils.stripConfigurationCredentials(configuration);
1203
+
1204
+ expect(sanitized.description).toBe('Mirror https://example.org');
1205
+ });
1206
+
1207
+ it('sanitizes a description with a numeric password followed by punctuation', () => {
1208
+ const configuration = {
1209
+ description: 'Mirror https://user:1234, secret@example.org/list.txt',
1210
+ };
1211
+
1212
+ const sanitized = utils.stripConfigurationCredentials(configuration);
1213
+
1214
+ expect(sanitized.description).toBe('Mirror https://example.org/list.txt');
1215
+ });
1216
+
1217
+ it('sanitizes values under unknown keys with the strict parser', () => {
1218
+ const configuration = {
1219
+ name: 'test',
1220
+ sources: [
1221
+ {
1222
+ url: 'https://John Smith: secret@example.org',
1223
+ },
1224
+ ],
1225
+ unknown: 'https://user:1234, secret@example.org/list.txt',
1226
+ // A path-less value with a literal `@` and a space in the password
1227
+ // is sanitized only by the strict parser.
1228
+ unknownLiteralAt: 'https://user:p@s s@example.org',
1229
+ };
1230
+
1231
+ const sanitized = utils.stripConfigurationCredentials(configuration);
1232
+
1233
+ expect(sanitized.sources[0].url).toBe('https://example.org');
1234
+ expect(sanitized.unknown).toBe('https://example.org/list.txt');
1235
+ expect(sanitized.unknownLiteralAt).toBe('https://example.org');
1236
+ expect(configuration.sources[0].url).toBe('https://John Smith: secret@example.org');
1237
+ expect(configuration.unknown).toBe('https://user:1234, secret@example.org/list.txt');
1238
+ expect(configuration.unknownLiteralAt).toBe('https://user:p@s s@example.org');
1239
+ });
1240
+
1241
+ it('sanitizes nested values under unknown keys', () => {
1242
+ const configuration = {
1243
+ unknown: {
1244
+ nested: ['https://John Smith: secret@example.org'],
1245
+ },
1246
+ };
1247
+
1248
+ const sanitized = utils.stripConfigurationCredentials(configuration);
1249
+
1250
+ expect(sanitized.unknown.nested).toEqual(['https://example.org']);
1251
+ });
1252
+
1253
+ it('sanitizes URL object keys', () => {
1254
+ const configuration = {
1255
+ 'https://user:secret@example.org/key.txt': 'value',
1256
+ 'https://user:p"s@example.org/other.txt': 'value',
1257
+ };
1258
+
1259
+ const sanitized = utils.stripConfigurationCredentials(configuration);
1260
+
1261
+ expect(sanitized['https://example.org/key.txt']).toBe('value');
1262
+ expect(sanitized['https://example.org/other.txt']).toBe('value');
1263
+ expect(configuration['https://user:secret@example.org/key.txt']).toBe('value');
1264
+ });
1265
+
1266
+ it('sanitizes URL object keys with whitespace in userinfo', () => {
1267
+ // Keys are never prose, so the strict parser must handle a whitespace
1268
+ // in the username and in the password.
1269
+ const configuration = {
1270
+ 'https:John Smith:pa ss@example.org': 'value',
1271
+ };
1272
+
1273
+ const sanitized = utils.stripConfigurationCredentials(configuration);
1274
+
1275
+ expect(sanitized).toEqual({ 'https:example.org': 'value' });
1276
+ expect(configuration['https:John Smith:pa ss@example.org']).toBe('value');
1277
+ });
1278
+
1279
+ it('sanitizes every string in malformed configuration shapes', () => {
1280
+ const configuration = {
1281
+ homepage: ['https://user:secret@example.org/'],
1282
+ exclusions_sources: 'https://user:secret@example.org/exclusions.txt',
1283
+ sources: [
1284
+ 'https://user:secret@example.org/source.txt',
1285
+ {
1286
+ url: 'https://user:secret@example.org/misspelled.txt',
1287
+ source: ['https://user:secret@example.org/source-array.txt'],
1288
+ },
1289
+ ],
1290
+ };
1291
+
1292
+ const sanitized = utils.stripConfigurationCredentials(configuration);
1293
+
1294
+ expect(sanitized).toEqual({
1295
+ homepage: ['https://example.org/'],
1296
+ exclusions_sources: 'https://example.org/exclusions.txt',
1297
+ sources: [
1298
+ 'https://example.org/source.txt',
1299
+ {
1300
+ url: 'https://example.org/misspelled.txt',
1301
+ source: ['https://example.org/source-array.txt'],
1302
+ },
1303
+ ],
1304
+ });
1305
+ expect(configuration.homepage[0]).toBe('https://user:secret@example.org/');
1306
+ expect(configuration.sources[1].url).toBe('https://user:secret@example.org/misspelled.txt');
1307
+ });
1308
+
1309
+ it('sanitizes a malformed source string with whitespace in userinfo', () => {
1310
+ const configuration = {
1311
+ sources: ['https://user:p s s@example.org/source.txt'],
1312
+ };
1313
+
1314
+ const sanitized = utils.stripConfigurationCredentials(configuration);
1315
+
1316
+ expect(sanitized.sources).toEqual(['https://example.org/source.txt']);
1317
+ expect(configuration.sources[0]).toBe('https://user:p s s@example.org/source.txt');
1318
+ });
1319
+
1320
+ it('does not change prose that starts with a URL', () => {
1321
+ const configuration = {
1322
+ description: 'https://example.org is maintained by user@example.com',
1323
+ };
1324
+
1325
+ const sanitized = utils.stripConfigurationCredentials(configuration);
1326
+
1327
+ expect(sanitized.description).toBe(configuration.description);
1328
+ });
1329
+
1330
+ it('sanitizes strings in class instances', () => {
1331
+ class Source {
1332
+ constructor(source) {
1333
+ this.source = source;
1334
+ }
1335
+ }
1336
+
1337
+ const configuration = {
1338
+ sources: [new Source('https://user:secret@example.org/list.txt')],
1339
+ };
1340
+
1341
+ const sanitized = utils.stripConfigurationCredentials(configuration);
1342
+
1343
+ expect(sanitized.sources[0].source).toBe('https://example.org/list.txt');
1344
+ expect(configuration.sources[0].source).toBe('https://user:secret@example.org/list.txt');
1345
+ });
1346
+
1347
+ it('sanitizes a configuration that is itself a class instance', () => {
1348
+ class Configuration {
1349
+ constructor() {
1350
+ this.sources = [
1351
+ { source: 'https://user:secret@example.org/list.txt' },
1352
+ ];
1353
+ }
1354
+ }
1355
+
1356
+ const configuration = new Configuration();
1357
+ const sanitized = utils.stripConfigurationCredentials(configuration);
1358
+
1359
+ expect(sanitized).toBeInstanceOf(Configuration);
1360
+ expect(sanitized.sources[0].source).toBe('https://example.org/list.txt');
1361
+ expect(configuration.sources[0].source).toBe('https://user:secret@example.org/list.txt');
1362
+ });
1363
+ });
1364
+
1365
+ describe('sanitizeConfigurationForLog', () => {
1366
+ it('does not change a configuration with a path-less URL and a later email', () => {
1367
+ // Re-scanning the rendered document treated the text between the
1368
+ // path-less homepage and the email as userinfo. The description key
1369
+ // disappeared and the homepage changed.
1370
+ const configuration = {
1371
+ homepage: 'https://example.org',
1372
+ description: 'admin@example.com',
1373
+ sources: [{ source: 'list.txt' }],
1374
+ };
1375
+
1376
+ const sanitized = utils.sanitizeConfigurationForLog(configuration);
1377
+ const dump = JSON.stringify(sanitized, 0, 4);
1378
+
1379
+ expect(dump).toBe(JSON.stringify(configuration, 0, 4));
1380
+ });
1381
+
1382
+ it('does not change a rendered configuration that has no credentials', () => {
1383
+ const configuration = {
1384
+ homepage: 'https://example.org',
1385
+ exclusions: ['@@||good.example^'],
1386
+ sources: [{ source: 'lists/a.txt' }],
1387
+ };
1388
+
1389
+ const dump = JSON.stringify(utils.sanitizeConfigurationForLog(configuration), 0, 4);
1390
+
1391
+ expect(dump).toBe(JSON.stringify(configuration, 0, 4));
1392
+ });
1393
+
1394
+ it('sanitizes a credentialed source in a rendered configuration with exclusions', () => {
1395
+ const configuration = {
1396
+ homepage: 'https://example.org',
1397
+ exclusions: ['@@||good.example^'],
1398
+ sources: [{ source: 'https://user:secret@example.org/list.txt' }],
1399
+ };
1400
+
1401
+ const dump = JSON.stringify(utils.sanitizeConfigurationForLog(configuration), 0, 4);
1402
+
1403
+ expect(dump).toContain('"homepage": "https://example.org"');
1404
+ expect(dump).toContain('"@@||good.example^"');
1405
+ expect(dump).toContain('"source": "https://example.org/list.txt"');
1406
+ expect(dump).not.toContain('secret');
1407
+ });
1408
+
1409
+ it('sanitizes a URL object, a serialized value, and URL-like keys', () => {
1410
+ class Serialized {
1411
+ toJSON() {
1412
+ return 'https://user:secret@example.org/serialized.txt';
1413
+ }
1414
+ }
1415
+
1416
+ const configuration = {
1417
+ homepage: new URL('https://user:secret@example.org/'),
1418
+ description: new Serialized(),
1419
+ 'https://user:secret@example.org/key.txt': 'value',
1420
+ sources: [{ source: 'https://example.org/list.txt' }],
1421
+ };
1422
+
1423
+ const dump = JSON.stringify(utils.sanitizeConfigurationForLog(configuration));
1424
+
1425
+ expect(dump).toContain('"homepage":"https://example.org/"');
1426
+ expect(dump).toContain('"description":"https://example.org/serialized.txt"');
1427
+ expect(dump).toContain('"https://example.org/key.txt":"value"');
1428
+ expect(dump).not.toContain('secret');
1429
+ });
1430
+
1431
+ it('sanitizes URL-like keys introduced by toJSON', () => {
1432
+ class Configuration {
1433
+ toJSON() {
1434
+ return {
1435
+ 'https://user:secret@example.org/key.txt': 'value',
1436
+ };
1437
+ }
1438
+ }
1439
+
1440
+ const dump = JSON.stringify(utils.sanitizeConfigurationForLog(new Configuration()));
1441
+
1442
+ expect(dump).toContain('"https://example.org/key.txt":"value"');
1443
+ expect(dump).not.toContain('secret');
1444
+ });
1445
+
1446
+ it('does not change the original configuration', () => {
1447
+ const configuration = {
1448
+ homepage: new URL('https://user:secret@example.org/'),
1449
+ };
1450
+
1451
+ utils.sanitizeConfigurationForLog(configuration);
1452
+
1453
+ expect(configuration.homepage.href).toBe('https://user:secret@example.org/');
1454
+ });
1455
+ });
1456
+
1457
+ describe('stripErrorCredentials', () => {
1458
+ it('sanitizes the message, stack, and cause chain in place', () => {
1459
+ const aggregateError = new global.AggregateError([
1460
+ new Error('aggregate https://user:secret@example.net/aggregate.txt'),
1461
+ ]);
1462
+ const innerError = new Error('inner https:///user:secret@example.org/inner.txt');
1463
+ innerError.cause = aggregateError;
1464
+
1465
+ const error = new Error('outer https://user:secret@example.org/outer.txt');
1466
+ error.stack = 'Error: outer https://user:secret@example.org/outer.txt\n at test';
1467
+ error.cause = innerError;
1468
+
1469
+ const sanitized = utils.stripErrorCredentials(error);
1470
+
1471
+ expect(sanitized).toBe(error);
1472
+ expect(error.message).toBe('outer https://example.org/outer.txt');
1473
+ expect(error.stack).toBe('Error: outer https://example.org/outer.txt\n at test');
1474
+ expect(error.cause.message).toBe('inner https:///example.org/inner.txt');
1475
+ expect(error.cause.cause.errors[0].message).toBe('aggregate https://example.net/aggregate.txt');
1476
+ expect(JSON.stringify(error, Object.getOwnPropertyNames(error))).not.toContain('secret');
1477
+ });
1478
+
1479
+ it('sanitizes a string cause and a string in the errors array', () => {
1480
+ const error = new Error('outer');
1481
+ error.cause = 'cause https://user:secret@example.org/cause.txt';
1482
+ error.errors = ['aggregate https://user:secret@example.net/aggregate.txt'];
1483
+
1484
+ utils.stripErrorCredentials(error);
1485
+
1486
+ expect(error.cause).toBe('cause https://example.org/cause.txt');
1487
+ expect(error.errors).toEqual(['aggregate https://example.net/aggregate.txt']);
1488
+ });
1489
+
1490
+ it('applies exact replacements before the generic scan', () => {
1491
+ const raw = 'https:///user:p@s s@example.invalid/list.txt';
1492
+ const sanitized = 'https:///example.invalid/list.txt';
1493
+ const error = new Error(`Failed to request url '${raw}'`);
1494
+
1495
+ utils.stripErrorCredentials(error, [[raw, sanitized]]);
1496
+
1497
+ expect(error.message).toBe(`Failed to request url '${sanitized}'`);
1498
+ });
1499
+
1500
+ it('returns primitive values unchanged', () => {
1501
+ expect(utils.stripErrorCredentials('https://user:secret@example.org/')).toBe('https://user:secret@example.org/');
1502
+ expect(utils.stripErrorCredentials(undefined)).toBeUndefined();
1503
+ expect(utils.stripErrorCredentials(null)).toBeNull();
1504
+ });
1505
+ });
1506
+
22
1507
  describe('substringBetween', () => {
23
1508
  it('works', () => {
24
1509
  let substr = utils.substringBetween('<a>test</a>', '<a>', '</a>');
@@ -34,3 +1519,77 @@ describe('substringBetween', () => {
34
1519
  expect(substr).toBe(null);
35
1520
  });
36
1521
  });
1522
+
1523
+ describe('stripConfigurationCredentials with nested URLs', () => {
1524
+ it('sanitizes a URL nested in another URL with the strict parser', () => {
1525
+ const configuration = {
1526
+ homepage: 'https://proxy.example.org/fetch?u=https://user:secret@raw.example.org/list.txt',
1527
+ sources: [
1528
+ {
1529
+ source: 'https://proxy.example.org/r/https://user:secret@raw.example.org/list.txt',
1530
+ },
1531
+ ],
1532
+ };
1533
+
1534
+ const sanitized = utils.stripConfigurationCredentials(configuration);
1535
+
1536
+ expect(sanitized.homepage).toBe('https://proxy.example.org/fetch?u=https://raw.example.org/list.txt');
1537
+ expect(sanitized.sources[0].source).toBe(
1538
+ 'https://proxy.example.org/r/https://raw.example.org/list.txt',
1539
+ );
1540
+ expect(configuration.homepage).toBe(
1541
+ 'https://proxy.example.org/fetch?u=https://user:secret@raw.example.org/list.txt',
1542
+ );
1543
+ });
1544
+ });
1545
+
1546
+ describe('credential field sets', () => {
1547
+ /**
1548
+ * Collects the fields marked with `x-credential-url` in the configuration
1549
+ * schema. The marker is the schema-side source of truth for the strict
1550
+ * credential parser: a new URL property must carry it, and this helper then
1551
+ * requires the matching entry in `src/utils.js`.
1552
+ *
1553
+ * @param {Object} schema - the configuration schema.
1554
+ * @returns {{ value: Array<String>, list: Array<String> }} sorted field
1555
+ * names grouped by marker kind.
1556
+ */
1557
+ function collectCredentialUrlFields(schema) {
1558
+ const value = new Set();
1559
+ const list = new Set();
1560
+ const propertyGroups = [
1561
+ schema.properties,
1562
+ schema.definitions.source.properties,
1563
+ ];
1564
+
1565
+ propertyGroups.forEach((properties) => {
1566
+ Object.entries(properties).forEach(([key, property]) => {
1567
+ if (property['x-credential-url'] === 'value') {
1568
+ value.add(key);
1569
+ }
1570
+ if (property['x-credential-url'] === 'list') {
1571
+ list.add(key);
1572
+ }
1573
+ });
1574
+ });
1575
+
1576
+ return {
1577
+ value: [...value].sort(),
1578
+ list: [...list].sort(),
1579
+ };
1580
+ }
1581
+
1582
+ it('matches the configuration schema key sets', () => {
1583
+ expect([...utils.CONFIGURATION_KEYS].sort())
1584
+ .toEqual(Object.keys(configurationSchema.properties).sort());
1585
+ expect([...utils.SOURCE_KEYS].sort())
1586
+ .toEqual(Object.keys(configurationSchema.definitions.source.properties).sort());
1587
+ });
1588
+
1589
+ it('matches the URL fields marked in the configuration schema', () => {
1590
+ const marked = collectCredentialUrlFields(configurationSchema);
1591
+
1592
+ expect([...utils.URL_VALUE_FIELDS].sort()).toEqual(marked.value);
1593
+ expect([...utils.URL_LIST_FIELDS].sort()).toEqual(marked.list);
1594
+ });
1595
+ });