@adguard/hostlist-compiler 2.1.1 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/.github/workflows/prepare-release.yml +5 -0
  2. package/.markdownlint.json +29 -0
  3. package/.markdownlintignore +1 -0
  4. package/AGENTS.md +310 -98
  5. package/CHANGELOG.md +62 -8
  6. package/DEPLOYMENT.md +175 -0
  7. package/DEVELOPMENT.md +171 -72
  8. package/README.md +238 -143
  9. package/examples/china/README.md +2 -2
  10. package/examples/energized/README.md +4 -2
  11. package/examples/sdn/README.md +2 -2
  12. package/examples/whitelist/README.md +2 -2
  13. package/package.json +24 -18
  14. package/src/cli.js +53 -31
  15. package/src/compile-source.js +4 -3
  16. package/src/configuration.js +132 -2
  17. package/src/download.js +27 -0
  18. package/src/filter.js +2 -2
  19. package/src/index.d.ts +2 -0
  20. package/src/index.js +16 -50
  21. package/src/{transformations/ip-normalize.js → ip-normalize.js} +2 -2
  22. package/src/schemas/configuration.schema.js +137 -0
  23. package/src/source-types.js +17 -0
  24. package/src/transformations/enum.js +49 -0
  25. package/src/transformations/include.js +1 -1
  26. package/src/transformations/transform.js +40 -74
  27. package/src/transformations/validate-allow-ip-and-public-suffix.js +1 -1
  28. package/src/transformations/validate-allow-ip.js +1 -1
  29. package/src/utils.js +1284 -0
  30. package/src/validation-conflicts.js +106 -0
  31. package/test/cli-error-output.test.js +305 -0
  32. package/test/cli.test.js +47 -0
  33. package/test/compile-source.test.js +167 -7
  34. package/test/configuration.test.js +438 -0
  35. package/test/download.test.js +260 -0
  36. package/test/helpers/http-mock.js +94 -0
  37. package/test/index.test.js +497 -4
  38. package/test/{transformations/ip-normalize.test.js → ip-normalize.test.js} +1 -1
  39. package/test/resolve-include.test.js +8 -8
  40. package/test/transformations/covert-to-ascii.test.js +1 -1
  41. package/test/transformations/exclude.test.js +2 -2
  42. package/test/transformations/include.test.js +9 -3
  43. package/test/transformations/ip-alignment.test.js +1 -1
  44. package/test/transformations/transform.test.js +66 -1
  45. package/test/utils.test.js +1559 -0
  46. package/test/validation-conflicts.test.js +230 -0
  47. package/src/schemas/configuration.schema.json +0 -369
package/src/utils.js CHANGED
@@ -15,6 +15,59 @@ const MODIFIER_REGEX = /^[a-zA-Z0-9,=~_.-]+$/;
15
15
  */
16
16
  const OCTET_REGEX = /^\d{1,3}$/;
17
17
 
18
+ /**
19
+ * Sticky regex to match a URL scheme at the current position.
20
+ */
21
+ const URL_SCHEME_REGEX = /[a-z][a-z0-9+.-]*:/iy;
22
+
23
+ /**
24
+ * Sticky regex to match the slashes or backslashes that follow a URL scheme.
25
+ */
26
+ const URL_SLASHES_REGEX = /[/\\]+/y;
27
+
28
+ /**
29
+ * Regex to match a character that ends the authority component of a URL.
30
+ */
31
+ const URL_AUTHORITY_END_REGEX = /[/\\?#]/;
32
+
33
+ /**
34
+ * Regex to match a character that ends the authority component when it is not
35
+ * part of userinfo. A quote or whitespace inside userinfo is part of the URL,
36
+ * so the scanner decides whether it ends the URL or not.
37
+ */
38
+ const URL_AUTHORITY_TERMINATOR_REGEX = /[\s"'<>]/;
39
+
40
+ /**
41
+ * Regex to match a `token:digits` pair that can be a port in the authority
42
+ * text before the first `@`. The lookahead accepts a terminator or punctuation
43
+ * after the digits. A bracketed IPv6 literal is matched as one token.
44
+ *
45
+ * The lookbehind requires the token to start at a boundary. Without it, the
46
+ * regex retries from every position inside a long token, for example a long
47
+ * password, and the scan becomes quadratic.
48
+ */
49
+ const URL_PORT_REGEX = /(?<![^\s"'<>/:])(?:\[[0-9a-f:.]+\]|[^\s"'<>/:]+):\d+(?=[\s"'<>,.;:!?()[\]{}])/gi;
50
+
51
+ /**
52
+ * Modes for the authority parser.
53
+ *
54
+ * `VALUE` treats the text before the first `@` as userinfo only when it looks
55
+ * like userinfo, so it does not mangle prose that contains an email address.
56
+ * It also accepts userinfo when the URL has a path, query, or fragment, or when
57
+ * the value is a whole URL. The path or value shape shows that the text is a
58
+ * URL, not prose, so a quote or a space in the username does not hide the
59
+ * password.
60
+ *
61
+ * `STRICT` treats everything before the last `@` in the authority as userinfo,
62
+ * like the downloader does. It also recognizes malformed values where the
63
+ * password contains `?` or `#`, because the authority ends before the at sign
64
+ * there. Use it only for values that must be a URL.
65
+ */
66
+ const URL_AUTHORITY_MODE = {
67
+ VALUE: 'value',
68
+ STRICT: 'strict',
69
+ };
70
+
18
71
  /**
19
72
  * Checks if a string is a valid IPv4 octet (0-255).
20
73
  *
@@ -169,6 +222,1224 @@ function splitByDelimiterWithEscapeCharacter(
169
222
  return parts;
170
223
  }
171
224
 
225
+ /**
226
+ * Checks whether a URL scheme can start after the given character. A scheme
227
+ * cannot start in the middle of a word or another URL, but it can start after
228
+ * a slash, for example in a path such as `/https://example.org`.
229
+ *
230
+ * @param {String|undefined} char - the character before the scheme.
231
+ * @returns {Boolean} true if a scheme can start after the character.
232
+ */
233
+ function isSchemeBoundary(char) {
234
+ return char === undefined || !/[a-z0-9_+.-]/i.test(char);
235
+ }
236
+
237
+ /**
238
+ * Checks whether a scheme-relative URL (`//host`) can start after the given
239
+ * character. A double slash inside a path is not a new URL, so the previous
240
+ * character must not be a path or scheme character.
241
+ *
242
+ * @param {String|undefined} char - the character before the double slash.
243
+ * @returns {Boolean} true if a scheme-relative URL can start after it.
244
+ */
245
+ function isProtocolRelativeBoundary(char) {
246
+ return char === undefined || !/[a-z0-9_+.:/\\-]/i.test(char);
247
+ }
248
+
249
+ /**
250
+ * Checks whether the character ends the authority component of a URL.
251
+ *
252
+ * @param {String} char - character to check.
253
+ * @returns {Boolean} true if the character ends the authority.
254
+ */
255
+ function endsUrlAuthority(char) {
256
+ return URL_AUTHORITY_END_REGEX.test(char);
257
+ }
258
+
259
+ /**
260
+ * Checks whether the character ends the authority component when it is not
261
+ * part of userinfo.
262
+ *
263
+ * @param {String} char - character to check.
264
+ * @returns {Boolean} true if the character ends the authority.
265
+ */
266
+ function endsUrlAuthorityBeforeUserinfo(char) {
267
+ return URL_AUTHORITY_TERMINATOR_REGEX.test(char);
268
+ }
269
+
270
+ /**
271
+ * Checks whether a token looks like a host: a dotted name, `localhost`, or an
272
+ * IP literal. The port heuristic uses this to tell a port from a password.
273
+ *
274
+ * @param {String} value - token before a colon.
275
+ * @returns {Boolean} true if the token looks like a host.
276
+ */
277
+ function looksLikeHost(value) {
278
+ if (value.startsWith('[') && value.endsWith(']')) {
279
+ const inner = value.slice(1, -1);
280
+ return inner.includes(':') && /^[0-9a-f:.]+$/i.test(inner);
281
+ }
282
+
283
+ if (value.toLowerCase() === 'localhost') {
284
+ return true;
285
+ }
286
+
287
+ const octets = value.split('.');
288
+ if (octets.length === 4 && octets.every(isValidOctet)) {
289
+ return true;
290
+ }
291
+
292
+ // Hostnames can be internationalized and can contain underscores, for
293
+ // example an intranet host such as `my_host.org` or `фильтры.рф`.
294
+ return /^(?:[\p{L}\p{N}_-]+\.)+[\p{L}\p{N}_-]+$/iu.test(value);
295
+ }
296
+
297
+ /**
298
+ * Checks whether the text before the first `@` in an authority contains a
299
+ * port. A port looks like a password, so a URL followed by prose such as
300
+ * `https://example.org:8080 by admin@example.com` must stay unchanged. The
301
+ * port can be followed by a terminator or by punctuation, and the text before
302
+ * the colon must look like a host. Otherwise a numeric password followed by
303
+ * punctuation, such as `https://user:1234, secret@example.org`, would be
304
+ * mistaken for a port.
305
+ *
306
+ * @param {String} value - the authority text before the first `@`.
307
+ * @returns {Boolean} true if the text contains a port followed by a terminator
308
+ * or punctuation.
309
+ */
310
+ function looksLikePort(value) {
311
+ const ports = value.match(URL_PORT_REGEX) || [];
312
+
313
+ return ports.some((port) => looksLikeHost(port.slice(0, port.lastIndexOf(':'))));
314
+ }
315
+
316
+ /**
317
+ * Finds the colon that can start a password in the authority text before the
318
+ * first `@`. A colon inside a bracketed IPv6 literal is part of the host, not
319
+ * a password.
320
+ *
321
+ * @param {String} text - the string to scan.
322
+ * @param {Number} [start] - the index to start scanning from.
323
+ * @param {Number} [end] - the index to stop scanning at.
324
+ * @returns {Number} the index of the password colon, or -1.
325
+ */
326
+ function indexOfPasswordColon(text, start = 0, end = text.length) {
327
+ let inBrackets = false;
328
+
329
+ for (let i = start; i < end; i += 1) {
330
+ const char = text[i];
331
+
332
+ if (char === '[') {
333
+ inBrackets = true;
334
+ } else if (char === ']') {
335
+ inBrackets = false;
336
+ } else if (char === ':' && !inBrackets) {
337
+ return i;
338
+ }
339
+ }
340
+
341
+ return -1;
342
+ }
343
+
344
+ /**
345
+ * Checks whether the text before the first `@` in an authority looks like
346
+ * userinfo rather than prose that follows a URL. Prose can contain whitespace
347
+ * and an email-like `@`, so a colon that starts a password is the signal.
348
+ *
349
+ * `user:p s s` is userinfo, and `https://example.org is maintained by
350
+ * user@example.com` is prose. A colon that is followed by whitespace starts a
351
+ * label in prose, not a password. A colon followed by digits and then a
352
+ * terminator or punctuation is a port, not a password. This is the fast path;
353
+ * `looksLikeValueUserinfo()` handles a password with internal whitespace.
354
+ *
355
+ * @param {String} text - the string being scanned.
356
+ * @param {Number} start - index right after the URL prefix.
357
+ * @param {Number} end - index of the first `@`.
358
+ * @param {Number} firstTerminator - index of the first whitespace, quote, or
359
+ * angle bracket after `start`, bounded by `end`.
360
+ * @returns {Boolean} true if the text looks like userinfo.
361
+ */
362
+ function looksLikeUserinfo(text, start, end, firstTerminator) {
363
+ if (firstTerminator >= end) {
364
+ return true;
365
+ }
366
+
367
+ const colonIndex = indexOfPasswordColon(text, start, end);
368
+
369
+ if (colonIndex === -1 || colonIndex > firstTerminator) {
370
+ return false;
371
+ }
372
+
373
+ if (looksLikePort(text.slice(start, firstTerminator + 1))) {
374
+ return false;
375
+ }
376
+
377
+ return !/\s/.test(text[colonIndex + 1]);
378
+ }
379
+
380
+ /**
381
+ * Extracts the first word of a value for host checks. Punctuation after a
382
+ * host, for example at the end of a sentence, is ignored. A bracketed IPv6
383
+ * literal is kept intact, because its closing `]` belongs to the host.
384
+ *
385
+ * @param {String} value - the text to check.
386
+ * @returns {String} the first word without trailing punctuation.
387
+ */
388
+ function firstWordOf(value) {
389
+ const firstWordMatch = value.trim().split(/\s+/, 1)[0];
390
+ const bracketedHostEnd = firstWordMatch.startsWith('[') ? firstWordMatch.indexOf(']') : -1;
391
+
392
+ return bracketedHostEnd === -1
393
+ ? firstWordMatch.replace(/[.,;:!?)\]}"'>]+$/, '')
394
+ : firstWordMatch.slice(0, bracketedHostEnd + 1);
395
+ }
396
+
397
+ /**
398
+ * Checks whether the first word of a value looks like the host of a URL:
399
+ * a dotted name, `localhost`, or an IP literal.
400
+ *
401
+ * @param {String} value - the text to check.
402
+ * @returns {Boolean} true if the first word looks like a host.
403
+ */
404
+ function startsWithHost(value) {
405
+ return looksLikeHost(firstWordOf(value));
406
+ }
407
+
408
+ /**
409
+ * Checks whether the first word of a value looks like a single-label intranet
410
+ * host. A dotted name, `localhost`, and an IP literal are handled by
411
+ * `startsWithHost()`. The word must be lowercase, because a capitalized single
412
+ * word is a person's name in a username such as `John Smith` or `J. Smith`.
413
+ *
414
+ * @param {String} value - the text to check.
415
+ * @returns {Boolean} true if the first word looks like an intranet host.
416
+ */
417
+ function startsWithIntranetHost(value) {
418
+ return /^\p{Ll}[\p{L}\p{N}_-]*$/u.test(firstWordOf(value));
419
+ }
420
+
421
+ /**
422
+ * Checks whether the text before the first `@` is a URL with userinfo rather
423
+ * than prose. The text must contain a password colon with a value after it. A
424
+ * port, and a host-like first word followed by a label, are prose. A password
425
+ * can contain whitespace, so prose is recognized by the text before the colon
426
+ * rather than by the password. A URL can be followed by prose, so a terminator
427
+ * after the host does not make the text before the `@` prose.
428
+ *
429
+ * This covers `https://o'brien:secret@example.org`,
430
+ * `https://John Smith:secret@example.org`,
431
+ * `https://user: secret@example.org`, a password with internal whitespace such
432
+ * as `https://o'brien:pass phrase@example.org`, and the same values followed by
433
+ * text such as `https://John Smith:secret@example.org by admin@example.com`.
434
+ * Prose such as `https://example.org team: contact@example.com`,
435
+ * `https://example.org or mailto:admin@example.com for support`, and
436
+ * `Homepage "https://example.org". Contact: admin@example.com` stays unchanged.
437
+ *
438
+ * @param {String} text - the string being scanned.
439
+ * @param {Number} start - index right after the URL prefix.
440
+ * @param {Number} firstUserinfoEnd - index of the first `@`.
441
+ * @param {Number} authorityEnd - index where the authority ends.
442
+ * @param {{ nextTerminators: Array<Number>, nextNonWhitespace: Array<Number>,
443
+ * nextSpaces: Array<Number>, previousSpaces: Array<Number> }} scan - lookup
444
+ * tables built by `buildAuthorityScan()`.
445
+ * @returns {Boolean} true if the text looks like a URL with userinfo.
446
+ */
447
+ function looksLikeValueUserinfo(text, start, firstUserinfoEnd, authorityEnd, scan) {
448
+ // A port can only be in the host token, which ends at the first
449
+ // terminator. Checking the whole prefix would scan the rest of the string
450
+ // for every URL start.
451
+ const firstTerminator = Math.min(scan.nextTerminators[start], firstUserinfoEnd);
452
+ // Include the terminator so the port lookahead can see it.
453
+ const portCandidateEnd = Math.min(firstTerminator + 1, firstUserinfoEnd);
454
+
455
+ if (looksLikePort(text.slice(start, portCandidateEnd))) {
456
+ return false;
457
+ }
458
+
459
+ const colonIndex = indexOfPasswordColon(text, start, firstUserinfoEnd);
460
+
461
+ if (colonIndex === -1) {
462
+ return false;
463
+ }
464
+
465
+ if (scan.nextNonWhitespace[colonIndex + 1] >= firstUserinfoEnd) {
466
+ return false;
467
+ }
468
+
469
+ // A host-like first word followed by whitespace before the colon makes the
470
+ // colon a label in prose, whatever follows it. The value starts with the
471
+ // host, so this also keeps `https://example.org or mailto:admin@example.com`
472
+ // and `Homepage "https://example.org". Contact: admin@example.com`
473
+ // unchanged. A username such as `John Smith` does not start with a host,
474
+ // so `https://John Smith: secret@example.org` is still userinfo. A
475
+ // single-label intranet host also requires a label word that does not start
476
+ // with a lowercase letter, so a lowercase username such as
477
+ // `https://admin user:secret@example.org` stays userinfo while
478
+ // `https://x Contact: admin@example.org` and a caseless label such as
479
+ // `https://x 联系: admin@example.org` are prose. A colon that follows a
480
+ // host directly is prose only when a label follows it, so
481
+ // `https://example.org: Contact admin@example.com` stays unchanged and
482
+ // `https://example.org:secret@example.org` is still userinfo. A
483
+ // punctuated host word is prose only when the label colon is followed by
484
+ // whitespace too, so `https://myhost, contact: admin@example.org` stays
485
+ // unchanged while `https://user, name:secret@example.org` is userinfo.
486
+ const firstWordEnd = Math.min(scan.nextSpaces[start], firstUserinfoEnd);
487
+ const firstWord = text.slice(start, firstWordEnd);
488
+ const beforeColon = text.slice(start, colonIndex);
489
+ const beforeColonWord = firstWordOf(beforeColon);
490
+ const rawBeforeColonWord = beforeColon.trim().split(/\s+/, 1)[0];
491
+ const hasPunctuatedHost = beforeColonWord !== rawBeforeColonWord
492
+ && /^[\p{L}\p{N}_-]+$/u.test(beforeColonWord);
493
+ const label = text.slice(scan.previousSpaces[colonIndex] + 1, colonIndex);
494
+ const hasLabelWord = label.length > 0 && !/^\p{Ll}/u.test(label);
495
+ const colonFollowedByWhitespace = /\s/.test(text[colonIndex + 1]);
496
+ const afterColonStart = scan.nextNonWhitespace[colonIndex + 1];
497
+ const afterColonWord = text.slice(
498
+ afterColonStart,
499
+ Math.min(scan.nextSpaces[afterColonStart], firstUserinfoEnd),
500
+ );
501
+ const hasLabelAfterColon = afterColonWord.length > 0 && !/^\p{Ll}/u.test(afterColonWord);
502
+ const isProseHost = startsWithHost(beforeColon)
503
+ || (hasPunctuatedHost && colonFollowedByWhitespace && /^\p{Ll}/u.test(label))
504
+ || (hasLabelWord && startsWithIntranetHost(firstWord))
505
+ || (colonFollowedByWhitespace
506
+ && hasLabelAfterColon
507
+ && startsWithIntranetHost(beforeColon));
508
+
509
+ if (isProseHost && (firstWordEnd < colonIndex || colonFollowedByWhitespace)) {
510
+ return false;
511
+ }
512
+
513
+ // Prose after the URL starts a terminator after the first `@`. The text
514
+ // between the first `@` and that terminator must not contain another `@`:
515
+ // a second `@` means the first one was not the end of the userinfo. The
516
+ // host cannot contain a path, query, or fragment, so the check stops at
517
+ // the authority end. This keeps a rendered document such as
518
+ // `{"homepage":"https://example.org","exclusions":["@@||good.example^"]}`
519
+ // unchanged.
520
+ const terminator = Math.min(scan.nextTerminators[firstUserinfoEnd + 1], authorityEnd);
521
+ // When the text after the first `@` runs straight into a path, query, or
522
+ // fragment, the whole prefix is a URL and the password can contain a
523
+ // literal `@`. The last `@` before the authority end separates the
524
+ // userinfo from the host, so an intermediate `@` is not prose.
525
+ const runsIntoPath = authorityEnd < text.length
526
+ && scan.nextTerminators[firstUserinfoEnd + 1] >= authorityEnd;
527
+
528
+ if (terminator < text.length) {
529
+ const host = text.slice(firstUserinfoEnd + 1, terminator);
530
+
531
+ if (host.length === 0 || (host.includes('@') && !runsIntoPath)) {
532
+ return false;
533
+ }
534
+ }
535
+
536
+ return true;
537
+ }
538
+
539
+ /**
540
+ * Reads a URL start at the given position.
541
+ *
542
+ * @param {String} text - the string to scan.
543
+ * @param {Number} index - position to check.
544
+ * @returns {{ prefixEnd: Number|null, nextIndex: Number }|null} the index
545
+ * after the URL prefix, the index to resume scanning from when the position
546
+ * starts a scheme that is not a URL, or null when no URL starts here.
547
+ */
548
+ function readUrlStartAt(text, index) {
549
+ const previousChar = index > 0 ? text[index - 1] : undefined;
550
+
551
+ if (isSchemeBoundary(previousChar)) {
552
+ URL_SCHEME_REGEX.lastIndex = index;
553
+ const schemeMatch = URL_SCHEME_REGEX.exec(text);
554
+
555
+ if (schemeMatch) {
556
+ const schemeEnd = index + schemeMatch[0].length;
557
+ URL_SLASHES_REGEX.lastIndex = schemeEnd;
558
+ const slashesMatch = URL_SLASHES_REGEX.exec(text);
559
+ const slashes = slashesMatch ? slashesMatch[0] : '';
560
+ const slashCount = slashes.length;
561
+ const scheme = schemeMatch[0].slice(0, -1).toLowerCase();
562
+ const isHttp = scheme === 'http' || scheme === 'https';
563
+ const hasBackslash = slashes.includes('\\');
564
+ // A single-letter scheme is a Windows drive letter in a local
565
+ // path such as `C:/user@corp/list.txt`, not a URL. No registered
566
+ // URL scheme is a single letter.
567
+ const isWindowsDriveLetter = scheme.length === 1;
568
+
569
+ // A scheme without slashes is only a URL for HTTP(S): the URL
570
+ // parser normalizes `https:user:pass@host` to a URL with
571
+ // authority. For other schemes, a colon is not enough to tell a
572
+ // URL from text such as `note:value`. Backslashes are accepted
573
+ // only for HTTP(S), so a Windows path such as `C:\dir` is not
574
+ // treated as a URL.
575
+ if (!isWindowsDriveLetter
576
+ && ((slashCount > 0 && (isHttp || !hasBackslash)) || (slashCount === 0 && isHttp))) {
577
+ return { prefixEnd: schemeEnd + slashCount, nextIndex: schemeEnd + slashCount };
578
+ }
579
+
580
+ return { prefixEnd: null, nextIndex: schemeEnd };
581
+ }
582
+ }
583
+
584
+ if (text[index] === '/' && text[index + 1] === '/') {
585
+ let end = index + 2;
586
+ while (text[end] === '/') {
587
+ end += 1;
588
+ }
589
+
590
+ if (isProtocolRelativeBoundary(previousChar)) {
591
+ return { prefixEnd: end, nextIndex: end };
592
+ }
593
+
594
+ return { prefixEnd: null, nextIndex: end };
595
+ }
596
+
597
+ return null;
598
+ }
599
+
600
+ /**
601
+ * Finds the next URL start in a string and returns the index right after its
602
+ * scheme and slashes.
603
+ *
604
+ * @param {String} text - the string to scan.
605
+ * @param {Number} from - index to start scanning from.
606
+ * @returns {Number|null} the index after the URL prefix, or null if there is
607
+ * no URL start.
608
+ */
609
+ function findUrlStart(text, from) {
610
+ let i = from;
611
+
612
+ while (i < text.length) {
613
+ const start = readUrlStartAt(text, i);
614
+
615
+ if (start === null) {
616
+ i += 1;
617
+ } else if (start.prefixEnd === null) {
618
+ i = start.nextIndex;
619
+ } else {
620
+ return start.prefixEnd;
621
+ }
622
+ }
623
+
624
+ return null;
625
+ }
626
+
627
+ /**
628
+ * Builds the lookup tables used by the authority scan, so a long string with
629
+ * many URL starts is scanned in linear time. `authorityEnds[i]` is the first
630
+ * `/`, `\`, `?`, or `#` at or after `i`, `atPositions` holds the sorted
631
+ * positions of every `@`, `nextTerminators[i]` is the first whitespace, quote,
632
+ * or angle bracket at or after `i`, `nextNonWhitespace[i]` is the first
633
+ * non-whitespace character at or after `i`, `nextSpaces[i]` is the first
634
+ * whitespace character at or after `i`, `nextPathSeparators[i]` is the first
635
+ * `/` or `\` at or after `i`, and `previousSpaces[i]` is the last whitespace
636
+ * character before `i`.
637
+ *
638
+ * @param {String} text - the string being scanned.
639
+ * @returns {{ authorityEnds: Array<Number>, atPositions: Array<Number>,
640
+ * nextTerminators: Array<Number>, nextNonWhitespace: Array<Number>,
641
+ * nextSpaces: Array<Number>, nextPathSeparators: Array<Number>,
642
+ * previousSpaces: Array<Number> }} the lookup tables.
643
+ */
644
+ function buildAuthorityScan(text) {
645
+ const authorityEnds = new Array(text.length + 1);
646
+ const nextTerminators = new Array(text.length + 1);
647
+ const nextNonWhitespace = new Array(text.length + 1);
648
+ const nextSpaces = new Array(text.length + 1);
649
+ const nextPathSeparators = new Array(text.length + 1);
650
+ const previousSpaces = new Array(text.length + 1);
651
+ authorityEnds[text.length] = text.length;
652
+ nextTerminators[text.length] = text.length;
653
+ nextNonWhitespace[text.length] = text.length;
654
+ nextSpaces[text.length] = text.length;
655
+ nextPathSeparators[text.length] = text.length;
656
+ previousSpaces[0] = -1;
657
+
658
+ for (let i = text.length - 1; i >= 0; i -= 1) {
659
+ const char = text[i];
660
+ authorityEnds[i] = endsUrlAuthority(char) ? i : authorityEnds[i + 1];
661
+ nextTerminators[i] = endsUrlAuthorityBeforeUserinfo(char) ? i : nextTerminators[i + 1];
662
+ nextNonWhitespace[i] = /\s/.test(char) ? nextNonWhitespace[i + 1] : i;
663
+ nextSpaces[i] = /\s/.test(char) ? i : nextSpaces[i + 1];
664
+ nextPathSeparators[i] = char === '/' || char === '\\' ? i : nextPathSeparators[i + 1];
665
+ }
666
+
667
+ for (let i = 0; i < text.length; i += 1) {
668
+ previousSpaces[i + 1] = /\s/.test(text[i]) ? i : previousSpaces[i];
669
+ }
670
+
671
+ const atPositions = [];
672
+ for (let i = 0; i < text.length; i += 1) {
673
+ if (text[i] === '@') {
674
+ atPositions.push(i);
675
+ }
676
+ }
677
+
678
+ return {
679
+ authorityEnds,
680
+ atPositions,
681
+ nextTerminators,
682
+ nextNonWhitespace,
683
+ nextSpaces,
684
+ nextPathSeparators,
685
+ previousSpaces,
686
+ };
687
+ }
688
+
689
+ /**
690
+ * Checks whether a host token looks like a host. The token can carry a port,
691
+ * for example `example.org:8080`, and a trailing colon or comma that belongs
692
+ * to prose, for example `example.org:` in `https://example.org: Contact ...`.
693
+ *
694
+ * @param {String} token - the host token.
695
+ * @returns {Boolean} true if the token looks like a host.
696
+ */
697
+ function looksLikeHostToken(token) {
698
+ const hostName = firstWordOf(token).replace(/:\d+$/, '');
699
+
700
+ return looksLikeHost(hostName);
701
+ }
702
+
703
+ /**
704
+ * Finds the `@` that separates the userinfo from the host in value mode.
705
+ *
706
+ * The URL parser and the downloader accept a literal `@` inside a password, so
707
+ * the first `@` is not always the separator. The function keeps the first `@`
708
+ * when the token after it looks like a host. Otherwise it scans the later `@`
709
+ * positions for the first host-like token, so
710
+ * `https://user:p@ss word@example.org` becomes `https://example.org` instead
711
+ * of the partial `https://ss word@example.org`.
712
+ *
713
+ * A single-label token after the first `@` can be either the rest of a
714
+ * password or an intranet host followed by prose. Trailing punctuation or a
715
+ * label colon after the token shows prose, so the first `@` stays the
716
+ * separator. This keeps `https://user:secret@intranet Contact: admin@...`
717
+ * unchanged.
718
+ *
719
+ * @param {String} text - the string being scanned.
720
+ * @param {Number} firstUserinfoEnd - index of the first `@` in the authority.
721
+ * @param {Number} authorityEnd - index where the authority ends.
722
+ * @param {{ atPositions: Array<Number>, nextTerminators: Array<Number> }} scan -
723
+ * lookup tables built by `buildAuthorityScan()`.
724
+ * @returns {Number} the index of the `@` that separates userinfo from the
725
+ * host.
726
+ */
727
+ function findValueUserinfoEnd(text, firstUserinfoEnd, authorityEnd, scan) {
728
+ const firstAtIndex = _.sortedIndex(scan.atPositions, firstUserinfoEnd);
729
+ const atEndIndex = _.sortedIndex(scan.atPositions, authorityEnd);
730
+ let fallback = firstUserinfoEnd;
731
+
732
+ for (let i = firstAtIndex; i < atEndIndex; i += 1) {
733
+ const at = scan.atPositions[i];
734
+ const terminator = Math.min(scan.nextTerminators[at + 1], authorityEnd);
735
+ const host = text.slice(at + 1, terminator);
736
+
737
+ if (looksLikeHostToken(host)) {
738
+ return at;
739
+ }
740
+
741
+ // A token with `@` sits inside the password, so the host still
742
+ // follows. An empty token means two adjacent `@` characters.
743
+ if (host.length > 0 && !host.includes('@')) {
744
+ const nextAt = i + 1 < atEndIndex ? scan.atPositions[i + 1] : authorityEnd;
745
+ const between = text.slice(terminator, nextAt);
746
+ const hasProseLabel = /[.,;:]$/u.test(host) || /\s+\S+:/u.test(between);
747
+
748
+ if (hasProseLabel) {
749
+ return firstUserinfoEnd;
750
+ }
751
+
752
+ fallback = at;
753
+ }
754
+ }
755
+
756
+ return fallback;
757
+ }
758
+
759
+ /**
760
+ * Checks that the text after an at sign looks like the host and the rest of
761
+ * a URL. The check tells malformed userinfo from an at sign in a query or a
762
+ * fragment.
763
+ *
764
+ * The host can be followed by a path, a query, a fragment, or the end of the
765
+ * value. A single-label host in these forms is ambiguous with an at sign in
766
+ * the query or fragment of a valid URL whose host has no dot, for example
767
+ * `https://fileserver:?token=a@b`. A credential leak in a published list is
768
+ * worse than a wrong host in a display message, so the ambiguous form is
769
+ * treated as userinfo.
770
+ *
771
+ * @param {String} text - the string being scanned.
772
+ * @param {Number} at - index of the at sign.
773
+ * @param {{ nextTerminators: Array<Number> }} scan - lookup tables built by
774
+ * `buildAuthorityScan()`.
775
+ * @returns {Boolean} true if the text after the at sign looks like a URL.
776
+ */
777
+ function hasUrlTailAfterAt(text, at, scan) {
778
+ const tail = text.slice(at + 1, scan.nextTerminators[at + 1]);
779
+
780
+ return /^[^/\\?#:]*[.:]/i.test(tail)
781
+ || /^localhost(?=[/\\?#])/i.test(tail)
782
+ || /^[^/\\?#:.]+(?:[/\\?#]|$)/.test(tail);
783
+ }
784
+
785
+ /**
786
+ * Checks that the text before an at sign is malformed userinfo whose password
787
+ * contains `?` or `#`.
788
+ *
789
+ * A dotted part before the colon is a host with a port, and `localhost` or an
790
+ * IPv6 literal is a host, so these values are not userinfo. The password colon
791
+ * must come before the first `?`, `#`, or whitespace.
792
+ *
793
+ * @param {String} userinfo - the text between the URL prefix and the at sign.
794
+ * @returns {Boolean} true if the text looks like malformed userinfo.
795
+ */
796
+ function isMalformedStrictUserinfo(userinfo) {
797
+ const separatorIndex = indexOfPasswordColon(userinfo);
798
+
799
+ if (separatorIndex === -1) {
800
+ return false;
801
+ }
802
+
803
+ const ambiguousIndex = userinfo.search(/[?#\s]/);
804
+
805
+ if (ambiguousIndex !== -1 && separatorIndex > ambiguousIndex) {
806
+ return false;
807
+ }
808
+
809
+ const username = userinfo.slice(0, separatorIndex);
810
+
811
+ return !username.includes('.')
812
+ && username.toLowerCase() !== 'localhost'
813
+ && !username.startsWith('[');
814
+ }
815
+
816
+ /**
817
+ * Finds the at sign that separates malformed userinfo from the host when the
818
+ * password contains `?` or `#`.
819
+ *
820
+ * The authority ends at the first `?` or `#`, before the at sign, so the
821
+ * normal strict scan does not see the at sign. The downloader recognizes such
822
+ * values for display redaction. Accept the form only when the password colon
823
+ * comes before the first `?`, `#`, or whitespace, the part before the colon is
824
+ * not a host, and the text after the at sign looks like the rest of a URL.
825
+ * A port followed by a path keeps a valid URL with an at sign in the query
826
+ * unchanged, for example `https://myhost:8080/path?email=user@example.com`.
827
+ *
828
+ * The search is bounded by the first value terminator and the first path
829
+ * separator after the authority end and starts from that bound, so a long
830
+ * string with many at signs is scanned once per value instead of once per at
831
+ * sign. The path bound matches the library parser, which stops the malformed
832
+ * userinfo match at a `/` or `\`.
833
+ *
834
+ * @param {String} text - the string being scanned.
835
+ * @param {Number} start - index right after the URL prefix.
836
+ * @param {{ atPositions: Array<Number>, nextTerminators: Array<Number>,
837
+ * nextPathSeparators: Array<Number> }} scan - lookup tables built by
838
+ * `buildAuthorityScan()`.
839
+ * @param {Number} authorityEnd - index where the authority ends.
840
+ * @returns {Number} the index of the at sign, or -1.
841
+ */
842
+ function findMalformedStrictUserinfoEnd(text, start, scan, authorityEnd) {
843
+ // Bound the search by the first value terminator and by the first path
844
+ // separator after the authority end. The library's malformed-userinfo
845
+ // match stops at a `/` or `\`, so an at sign in a later path is not a
846
+ // separator. With the single-label tail rule below, an at sign in the
847
+ // path would otherwise be taken as the separator.
848
+ const limit = Math.min(
849
+ scan.nextTerminators[authorityEnd],
850
+ scan.nextPathSeparators[authorityEnd],
851
+ text.length,
852
+ );
853
+ const lastAt = _.sortedLastIndex(scan.atPositions, limit - 1) - 1;
854
+
855
+ for (let i = lastAt; i >= 0; i -= 1) {
856
+ const at = scan.atPositions[i];
857
+
858
+ if (at < start) {
859
+ break;
860
+ }
861
+
862
+ if (hasUrlTailAfterAt(text, at, scan)
863
+ && isMalformedStrictUserinfo(text.slice(start, at))) {
864
+ return at;
865
+ }
866
+ }
867
+
868
+ return -1;
869
+ }
870
+
871
+ /**
872
+ * Reads the authority component of a URL and splits it into the sanitized
873
+ * value and the end of the URL.
874
+ *
875
+ * @param {String} text - the string being scanned.
876
+ * @param {Number} start - index right after the URL prefix.
877
+ * @param {String} mode - one of `URL_AUTHORITY_MODE`.
878
+ * @param {{ authorityEnds: Array<Number>, atPositions: Array<Number>,
879
+ * nextTerminators: Array<Number>, nextNonWhitespace: Array<Number>,
880
+ * nextSpaces: Array<Number>, nextPathSeparators: Array<Number>,
881
+ * previousSpaces: Array<Number> }} scan -
882
+ * lookup tables built by `buildAuthorityScan()`.
883
+ * @returns {{ value: String, rawEnd: Number }} the sanitized authority and the
884
+ * index where it ends.
885
+ */
886
+ function readUrlAuthority(text, start, mode, scan) {
887
+ const authorityEnd = scan.authorityEnds[start];
888
+ let userinfoEnd = -1;
889
+
890
+ if (mode === URL_AUTHORITY_MODE.STRICT) {
891
+ // The downloader takes everything before the last `@` in the
892
+ // authority as userinfo. Use this only for values that must be a URL,
893
+ // because prose after a URL can also contain an `@`.
894
+ const lastAt = _.sortedLastIndex(scan.atPositions, authorityEnd - 1) - 1;
895
+ userinfoEnd = lastAt >= 0 && scan.atPositions[lastAt] >= start
896
+ ? scan.atPositions[lastAt]
897
+ : -1;
898
+
899
+ if (userinfoEnd === -1) {
900
+ userinfoEnd = findMalformedStrictUserinfoEnd(text, start, scan, authorityEnd);
901
+ }
902
+ } else {
903
+ // The first `@` shows where userinfo could end. `looksLikeUserinfo()`
904
+ // rejects prose that follows a URL and contains an email address.
905
+ const firstAt = _.sortedIndex(scan.atPositions, start);
906
+ const firstUserinfoEnd = firstAt < scan.atPositions.length
907
+ && scan.atPositions[firstAt] < authorityEnd
908
+ ? scan.atPositions[firstAt]
909
+ : -1;
910
+
911
+ if (firstUserinfoEnd !== -1) {
912
+ const firstTerminator = Math.min(scan.nextTerminators[start], firstUserinfoEnd);
913
+ const isValueUserinfo = mode === URL_AUTHORITY_MODE.VALUE
914
+ && looksLikeValueUserinfo(text, start, firstUserinfoEnd, authorityEnd, scan);
915
+ const isUserinfo = looksLikeUserinfo(
916
+ text,
917
+ start,
918
+ firstUserinfoEnd,
919
+ firstTerminator,
920
+ ) || isValueUserinfo;
921
+
922
+ if (isUserinfo) {
923
+ // Userinfo can contain quote, whitespace, and `@` characters,
924
+ // but the host cannot. `findValueUserinfoEnd()` keeps a later
925
+ // email in prose such as `https://host and a@b` intact and
926
+ // also handles a literal `@` in a password.
927
+ userinfoEnd = findValueUserinfoEnd(
928
+ text,
929
+ firstUserinfoEnd,
930
+ authorityEnd,
931
+ scan,
932
+ );
933
+ }
934
+ }
935
+ }
936
+
937
+ const authorityStart = userinfoEnd === -1 ? start : userinfoEnd + 1;
938
+ const rawEnd = Math.min(
939
+ scan.nextTerminators[authorityStart],
940
+ scan.authorityEnds[authorityStart],
941
+ );
942
+
943
+ const value = text.slice(authorityStart, rawEnd);
944
+
945
+ return { value, rawEnd };
946
+ }
947
+
948
+ /**
949
+ * Removes the userinfo component from every URL in a string, using the given
950
+ * authority parser mode.
951
+ *
952
+ * The function does string surgery on the authority component, so the rest of
953
+ * the URL stays byte-identical. It does not parse and re-serialize the URL
954
+ * with `URL`, because that normalizes the URL. It recognizes an optional
955
+ * scheme followed by any number of slashes or backslashes, a scheme with no
956
+ * slashes for HTTP(S), and scheme-relative URLs that start with two or more
957
+ * slashes.
958
+ *
959
+ * @param {String} url - string that may contain URLs with credentials.
960
+ * @param {String} mode - one of `URL_AUTHORITY_MODE`.
961
+ * @returns {String} the string without the userinfo components, or the input
962
+ * unchanged when it has no URL with userinfo.
963
+ */
964
+ function stripUrlCredentialsWithMode(url, mode) {
965
+ if (!_.isString(url)) {
966
+ return url;
967
+ }
968
+
969
+ const scan = buildAuthorityScan(url);
970
+ let result = '';
971
+ let copiedUntil = 0;
972
+ let searchFrom = 0;
973
+
974
+ while (searchFrom < url.length) {
975
+ const prefixEnd = findUrlStart(url, searchFrom);
976
+
977
+ if (prefixEnd === null) {
978
+ break;
979
+ }
980
+
981
+ const authority = readUrlAuthority(url, prefixEnd, mode, scan);
982
+ result += url.slice(copiedUntil, prefixEnd);
983
+ result += authority.value;
984
+ // Resume the scan at the end of the authority, not at the end of the
985
+ // URL. A URL can sit inside the path, query, or fragment of another
986
+ // URL, for example a proxy URL with the target in its query. The rest
987
+ // of the outer URL is copied by the next iteration or by the final
988
+ // slice, so nested credentials are not copied verbatim.
989
+ copiedUntil = authority.rawEnd;
990
+ searchFrom = authority.rawEnd;
991
+ }
992
+
993
+ result += url.slice(copiedUntil);
994
+ return result;
995
+ }
996
+
997
+ /**
998
+ * Removes the userinfo component (the `user:password@` part) from every URL in
999
+ * a string. Use this for free text and for values of an unknown shape: the
1000
+ * function keeps prose that contains an email address unchanged.
1001
+ *
1002
+ * Do not scan a rendered document with this function, for example
1003
+ * `JSON.stringify(configuration)`. A path-less URL can make the text up to a
1004
+ * later `@` look like userinfo and corrupt the document. Use
1005
+ * `sanitizeConfigurationForLog()` for a configuration dump or a validation
1006
+ * error.
1007
+ *
1008
+ * The parser is a heuristic: it keeps text that reads as prose. Two shapes are
1009
+ * lexically ambiguous and stay unresolved. The value
1010
+ * `https://john Smith: secret@example.org` is kept as prose, so the password
1011
+ * can survive. The value `https://user:secret@intranet by admin@example.com`
1012
+ * is read as a credential, so the prose after the intranet host is removed.
1013
+ * Use `stripKnownUrlCredentials()` for values that must be a URL.
1014
+ *
1015
+ * @param {String} url - string that may contain URLs with credentials.
1016
+ * @returns {String} the string without the userinfo components, or the input
1017
+ * unchanged when it has no URL with userinfo.
1018
+ */
1019
+ function stripUrlCredentials(url) {
1020
+ return stripUrlCredentialsWithMode(url, URL_AUTHORITY_MODE.VALUE);
1021
+ }
1022
+
1023
+ /**
1024
+ * Removes the userinfo component from every URL in a value that must be a URL,
1025
+ * such as `source` or `homepage`.
1026
+ *
1027
+ * Unlike `stripUrlCredentials()`, this function has no prose guard. It treats
1028
+ * everything before the last `@` in the authority as userinfo, like the
1029
+ * downloader does. It also recognizes malformed values where the password
1030
+ * contains `?` or `#`: the authority ends before the at sign there, so the
1031
+ * normal strict scan cannot see it. Use this function for known URL values,
1032
+ * where a quote or a space in the username must not hide the password.
1033
+ *
1034
+ * A single-label host with a port and an at sign in the query but no path is
1035
+ * lexically identical to a digit-first password. The password form wins, so
1036
+ * such a value loses its host to avoid a leak. A path after the port tells the
1037
+ * forms apart, so `https://myhost:8080/path?email=user@example.com` stays
1038
+ * unchanged.
1039
+ *
1040
+ * @param {String} url - URL value.
1041
+ * @returns {String} the URL without the userinfo component, or the input
1042
+ * unchanged when it has no userinfo.
1043
+ */
1044
+ function stripKnownUrlCredentials(url) {
1045
+ return stripUrlCredentialsWithMode(url, URL_AUTHORITY_MODE.STRICT);
1046
+ }
1047
+
1048
+ /**
1049
+ * Removes the userinfo component from a string value whose shape is not known
1050
+ * in advance.
1051
+ *
1052
+ * The function first tries to parse the whole value as a URL. When the URL has
1053
+ * a path, query, or fragment, a quote or a space in the username does not hide
1054
+ * the password. A port followed by prose, such as
1055
+ * `https://example.org:8080 by admin@example.com`, stays unchanged.
1056
+ *
1057
+ * Leading whitespace is kept, so a value such as
1058
+ * ` https://user:password@example.org/` is sanitized without changing its
1059
+ * prefix. When the value does not start with a URL, or when the text before
1060
+ * the first `@` looks like prose, the function falls back to the scanner. The
1061
+ * scanner also covers any text after the first URL, so prefixed values such as
1062
+ * `See https://user:password@example.org/` are sanitized too.
1063
+ *
1064
+ * Use `stripKnownUrlCredentials()` for values that must be a URL, because that
1065
+ * function has no prose guard and strips exactly what the downloader treats as
1066
+ * userinfo.
1067
+ *
1068
+ * @param {String} url - string to sanitize.
1069
+ * @returns {String} the string without the userinfo component, or the input
1070
+ * unchanged when it has no userinfo.
1071
+ */
1072
+ function stripUrlValueCredentials(url) {
1073
+ if (!_.isString(url)) {
1074
+ return url;
1075
+ }
1076
+
1077
+ const valueStart = url.search(/\S/);
1078
+ if (valueStart === -1) {
1079
+ return url;
1080
+ }
1081
+
1082
+ const start = readUrlStartAt(url, valueStart);
1083
+ if (start === null || start.prefixEnd === null) {
1084
+ return stripUrlCredentials(url);
1085
+ }
1086
+
1087
+ const { prefixEnd } = start;
1088
+ const authority = readUrlAuthority(
1089
+ url,
1090
+ prefixEnd,
1091
+ URL_AUTHORITY_MODE.VALUE,
1092
+ buildAuthorityScan(url),
1093
+ );
1094
+
1095
+ const sanitized = url.slice(0, prefixEnd)
1096
+ + authority.value
1097
+ + url.slice(authority.rawEnd);
1098
+
1099
+ // The value can have more text after the first URL, and the first URL can
1100
+ // contain another URL in its path, query, or fragment. The scanner covers
1101
+ // both.
1102
+ return stripUrlCredentials(sanitized);
1103
+ }
1104
+
1105
+ /**
1106
+ * Replaces known raw values in a string and then removes URL credentials.
1107
+ *
1108
+ * The replacement runs first because a password can contain a literal `@`
1109
+ * followed by whitespace. The scanner uses the first `@` with a prose guard, so
1110
+ * it can leave the rest of such a password in the text. A caller that knows the
1111
+ * exact source URL can sanitize it with the strict parser before the scanner
1112
+ * runs.
1113
+ *
1114
+ * @param {String} value - the string to sanitize.
1115
+ * @param {Array<Array<String>>} replacements - pairs of a raw value and its
1116
+ * sanitized replacement.
1117
+ * @returns {String} the sanitized string.
1118
+ */
1119
+ function sanitizeErrorString(value, replacements) {
1120
+ let result = value;
1121
+
1122
+ replacements.forEach(([raw, sanitized]) => {
1123
+ if (_.isString(raw) && raw !== sanitized) {
1124
+ result = result.split(raw).join(sanitized);
1125
+ }
1126
+ });
1127
+
1128
+ return stripUrlCredentials(result);
1129
+ }
1130
+
1131
+ /**
1132
+ * Removes URL credentials from an error and its `cause` and `errors` chains.
1133
+ * The error is modified in place, so its type and stack location are
1134
+ * preserved.
1135
+ *
1136
+ * @param {*} error - the error to sanitize.
1137
+ * @param {Array<Array<String>>} [replacements] - pairs of a raw value and its
1138
+ * sanitized replacement, applied before the generic scan.
1139
+ * @param {WeakSet<Object>} [seen] - objects that have already been visited.
1140
+ * @returns {*} the sanitized error.
1141
+ */
1142
+ function stripErrorCredentials(error, replacements = [], seen = new WeakSet()) {
1143
+ if (!_.isObject(error) || seen.has(error)) {
1144
+ return error;
1145
+ }
1146
+ seen.add(error);
1147
+
1148
+ if (_.isString(error.message)) {
1149
+ Object.assign(error, { message: sanitizeErrorString(error.message, replacements) });
1150
+ }
1151
+ if (_.isString(error.stack)) {
1152
+ Object.assign(error, { stack: sanitizeErrorString(error.stack, replacements) });
1153
+ }
1154
+ if (_.isString(error.path)) {
1155
+ Object.assign(error, { path: sanitizeErrorString(error.path, replacements) });
1156
+ }
1157
+
1158
+ if (Array.isArray(error.errors)) {
1159
+ Object.assign(error, {
1160
+ errors: error.errors.map((cause) => (
1161
+ _.isString(cause)
1162
+ ? sanitizeErrorString(cause, replacements)
1163
+ : stripErrorCredentials(cause, replacements, seen)
1164
+ )),
1165
+ });
1166
+ }
1167
+
1168
+ if (_.isString(error.cause)) {
1169
+ Object.assign(error, { cause: sanitizeErrorString(error.cause, replacements) });
1170
+ } else {
1171
+ stripErrorCredentials(error.cause, replacements, seen);
1172
+ }
1173
+
1174
+ return error;
1175
+ }
1176
+
1177
+ /**
1178
+ * Sanitizes URL-like object keys in place, because `cloneDeepWith` visits
1179
+ * values only.
1180
+ *
1181
+ * @param {*} value - configuration value to walk.
1182
+ * @param {WeakSet<Object>} seen - objects that have already been visited.
1183
+ */
1184
+ function sanitizeConfigurationKeys(value, seen) {
1185
+ if (!_.isObject(value) || seen.has(value)) {
1186
+ return;
1187
+ }
1188
+ seen.add(value);
1189
+
1190
+ Object.entries(value).forEach(([key, child]) => {
1191
+ // Keys are never prose, so use the strict parser: a URL-like key can
1192
+ // carry whitespace in the username and in the password, which the
1193
+ // prose-tolerant parser would leave untouched.
1194
+ const sanitizedKey = stripKnownUrlCredentials(key);
1195
+
1196
+ if (sanitizedKey !== key) {
1197
+ Reflect.deleteProperty(value, key);
1198
+ Object.defineProperty(value, sanitizedKey, {
1199
+ value: child,
1200
+ enumerable: true,
1201
+ configurable: true,
1202
+ writable: true,
1203
+ });
1204
+ }
1205
+
1206
+ sanitizeConfigurationKeys(child, seen);
1207
+ });
1208
+ }
1209
+
1210
+ /**
1211
+ * Configuration fields that must contain a URL. A test derives the set from
1212
+ * the `x-credential-url` markers in `src/schemas/configuration.schema.js`, so
1213
+ * a marked schema property without an entry here fails the test.
1214
+ */
1215
+ const URL_VALUE_FIELDS = new Set([
1216
+ 'homepage',
1217
+ 'source',
1218
+ ]);
1219
+
1220
+ /**
1221
+ * Configuration fields that must contain a URL or a list of URLs. A test
1222
+ * derives the set from the `x-credential-url` markers in
1223
+ * `src/schemas/configuration.schema.js`.
1224
+ */
1225
+ const URL_LIST_FIELDS = new Set([
1226
+ 'sources',
1227
+ 'exclusions_sources',
1228
+ 'inclusions_sources',
1229
+ ]);
1230
+
1231
+ /**
1232
+ * Sanitizes a known URL field value or a list of URL field values with the
1233
+ * strict parser. Other values are left for the recursive walk.
1234
+ *
1235
+ * @param {*} value - field value.
1236
+ * @returns {*} sanitized field value.
1237
+ */
1238
+ function sanitizeKnownUrlField(value) {
1239
+ if (_.isString(value)) {
1240
+ return stripKnownUrlCredentials(value);
1241
+ }
1242
+
1243
+ if (Array.isArray(value)) {
1244
+ return value.map((item) => (
1245
+ _.isString(item) ? stripKnownUrlCredentials(item) : item
1246
+ ));
1247
+ }
1248
+
1249
+ return value;
1250
+ }
1251
+
1252
+ /**
1253
+ * Sanitizes known URL fields in place with the strict parser. The generic copy
1254
+ * uses a prose guard, so a quote or a space in the username of a `source` or
1255
+ * `homepage` value would survive it. The downloader treats such a value as
1256
+ * userinfo, so the strict parser runs on these fields.
1257
+ *
1258
+ * @param {*} value - configuration value to walk.
1259
+ * @param {WeakSet<Object>} seen - objects that have already been visited.
1260
+ */
1261
+ function sanitizeKnownUrlFields(value, seen) {
1262
+ if (!_.isObject(value) || seen.has(value)) {
1263
+ return;
1264
+ }
1265
+ seen.add(value);
1266
+
1267
+ Object.entries(value).forEach(([key, child]) => {
1268
+ if (URL_VALUE_FIELDS.has(key) || URL_LIST_FIELDS.has(key)) {
1269
+ Object.assign(value, { [key]: sanitizeKnownUrlField(child) });
1270
+ }
1271
+
1272
+ sanitizeKnownUrlFields(value[key], seen);
1273
+ });
1274
+ }
1275
+
1276
+ /**
1277
+ * Configuration keys allowed at the root level.
1278
+ */
1279
+ const CONFIGURATION_KEYS = new Set([
1280
+ 'name',
1281
+ 'description',
1282
+ 'homepage',
1283
+ 'license',
1284
+ 'version',
1285
+ 'sources',
1286
+ 'transformations',
1287
+ 'exclusions',
1288
+ 'exclusions_sources',
1289
+ 'inclusions',
1290
+ 'inclusions_sources',
1291
+ ]);
1292
+
1293
+ /**
1294
+ * Configuration keys allowed in a source object.
1295
+ */
1296
+ const SOURCE_KEYS = new Set([
1297
+ 'name',
1298
+ 'source',
1299
+ 'type',
1300
+ 'transformations',
1301
+ 'exclusions',
1302
+ 'exclusions_sources',
1303
+ 'inclusions',
1304
+ 'inclusions_sources',
1305
+ ]);
1306
+
1307
+ /**
1308
+ * Sanitizes every string under a value with the strict parser. Unknown
1309
+ * configuration keys are rejected by the schema, so their values are more
1310
+ * likely misplaced URLs than prose.
1311
+ *
1312
+ * @param {*} value - value under an unknown key.
1313
+ * @param {WeakSet<Object>} seen - objects that have already been visited.
1314
+ * @returns {*} sanitized value.
1315
+ */
1316
+ function sanitizeUnknownValue(value, seen) {
1317
+ if (_.isString(value)) {
1318
+ return stripKnownUrlCredentials(value);
1319
+ }
1320
+
1321
+ if (Array.isArray(value)) {
1322
+ value.forEach((item, index) => {
1323
+ Object.assign(value, { [index]: sanitizeUnknownValue(item, seen) });
1324
+ });
1325
+ return value;
1326
+ }
1327
+
1328
+ if (_.isObject(value) && !seen.has(value)) {
1329
+ seen.add(value);
1330
+ Object.entries(value).forEach(([key, child]) => {
1331
+ Object.assign(value, { [key]: sanitizeUnknownValue(child, seen) });
1332
+ });
1333
+ }
1334
+
1335
+ return value;
1336
+ }
1337
+
1338
+ /**
1339
+ * Applies the strict parser to strings under keys that the configuration
1340
+ * schema does not allow at the current level.
1341
+ *
1342
+ * @param {*} value - configuration value to walk.
1343
+ * @param {Set<String>} allowedKeys - keys allowed at this level.
1344
+ * @param {WeakSet<Object>} seen - objects that have already been visited.
1345
+ */
1346
+ function sanitizeUnknownKeyValues(value, allowedKeys, seen) {
1347
+ if (Array.isArray(value)) {
1348
+ value.forEach((item) => sanitizeUnknownKeyValues(item, allowedKeys, seen));
1349
+ return;
1350
+ }
1351
+
1352
+ if (!_.isObject(value) || seen.has(value)) {
1353
+ return;
1354
+ }
1355
+ seen.add(value);
1356
+
1357
+ Object.entries(value).forEach(([key, child]) => {
1358
+ if (!allowedKeys.has(key)) {
1359
+ Object.assign(value, { [key]: sanitizeUnknownValue(child, seen) });
1360
+ return;
1361
+ }
1362
+
1363
+ if (key === 'sources') {
1364
+ if (Array.isArray(child)) {
1365
+ child.forEach((item) => sanitizeUnknownKeyValues(item, SOURCE_KEYS, seen));
1366
+ } else if (_.isObject(child)) {
1367
+ sanitizeUnknownKeyValues(child, SOURCE_KEYS, seen);
1368
+ }
1369
+ return;
1370
+ }
1371
+
1372
+ if (Array.isArray(child)) {
1373
+ child.forEach((item) => sanitizeUnknownKeyValues(item, allowedKeys, seen));
1374
+ } else {
1375
+ sanitizeUnknownKeyValues(child, allowedKeys, seen);
1376
+ }
1377
+ });
1378
+ }
1379
+
1380
+ /**
1381
+ * Returns a deep copy of a configuration with credentials removed from every
1382
+ * string, including strings in malformed configurations and class instances.
1383
+ * The copy runs the generic parser on every string, the strict parser on known
1384
+ * URL fields and on values under unknown keys, and rewrites URL-like object
1385
+ * keys. The original configuration is not modified.
1386
+ *
1387
+ * @param {*} configuration - compilation configuration.
1388
+ * @returns {*} sanitized copy of the configuration.
1389
+ */
1390
+ function stripConfigurationCredentials(configuration) {
1391
+ // Every string goes through the generic parser first: it keeps whitespace
1392
+ // and quotes in userinfo for URL values and falls back to the scanner for
1393
+ // prefixed values and values with more text after the URL.
1394
+ const sanitized = _.cloneDeepWith(configuration, (value) => (
1395
+ _.isString(value) ? stripUrlValueCredentials(value) : undefined
1396
+ ));
1397
+
1398
+ // Known URL fields use the strict parser: the downloader treats a quote or
1399
+ // a space in the username as part of the userinfo.
1400
+ sanitizeKnownUrlFields(sanitized, new WeakSet());
1401
+ // The schema rejects unknown keys, so a value under such a key is more
1402
+ // likely a misplaced URL than prose.
1403
+ sanitizeUnknownKeyValues(sanitized, CONFIGURATION_KEYS, new WeakSet());
1404
+ sanitizeConfigurationKeys(sanitized, new WeakSet());
1405
+
1406
+ return sanitized;
1407
+ }
1408
+
1409
+ /**
1410
+ * Returns a JSON-safe copy of a configuration with credentials removed from
1411
+ * every string and object key. Use the copy for the configuration log, the
1412
+ * verbose CLI dump, and validation errors that render the configuration.
1413
+ *
1414
+ * `stripConfigurationCredentials()` works on the object graph, so it cannot see
1415
+ * the strings that a `toJSON()` method or a `URL` object renders, and
1416
+ * `JSON.stringify()` renders object keys without passing them to a replacer.
1417
+ * This function serializes the sanitized copy with a replacer, parses the
1418
+ * result back into plain data, and sanitizes the parsed copy too. The output is
1419
+ * plain JSON data, so a caller can render it without a second scan of the
1420
+ * whole document. A second scan would treat the text between a path-less URL
1421
+ * and a later `@` as userinfo and corrupt the document.
1422
+ *
1423
+ * @param {*} configuration - compilation configuration.
1424
+ * @returns {*} JSON-safe sanitized copy of the configuration.
1425
+ */
1426
+ function sanitizeConfigurationForLog(configuration) {
1427
+ const sanitized = stripConfigurationCredentials(configuration);
1428
+ // A replacer sees each rendered string after `toJSON()`, so it also covers
1429
+ // `URL` objects and class instances. Object keys do not reach the replacer.
1430
+ const json = JSON.stringify(sanitized, (key, value) => (
1431
+ _.isString(value) ? stripUrlValueCredentials(value) : value
1432
+ ));
1433
+
1434
+ if (json === undefined) {
1435
+ return sanitized;
1436
+ }
1437
+
1438
+ // The parsed copy is plain data. Run the sanitizer again to reach keys and
1439
+ // known URL fields that `toJSON()` introduced.
1440
+ return stripConfigurationCredentials(JSON.parse(json));
1441
+ }
1442
+
172
1443
  /**
173
1444
  * Wildcard is used by the exclusions transformation.
174
1445
  */
@@ -277,4 +1548,17 @@ module.exports = {
277
1548
  Wildcard,
278
1549
  splitByDelimiterWithEscapeCharacter,
279
1550
  substringBetween,
1551
+ stripUrlCredentials,
1552
+ stripUrlValueCredentials,
1553
+ stripKnownUrlCredentials,
1554
+ stripConfigurationCredentials,
1555
+ sanitizeConfigurationForLog,
1556
+ stripErrorCredentials,
1557
+ // Exported so a test can compare them with the `x-credential-url` markers
1558
+ // in the configuration schema. A marked schema property without a matching
1559
+ // entry would silently change which strings use the strict parser.
1560
+ URL_VALUE_FIELDS,
1561
+ URL_LIST_FIELDS,
1562
+ CONFIGURATION_KEYS,
1563
+ SOURCE_KEYS,
280
1564
  };