@adguard/hostlist-compiler 2.1.1 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/prepare-release.yml +5 -0
- package/.markdownlint.json +29 -0
- package/.markdownlintignore +1 -0
- package/AGENTS.md +310 -98
- package/CHANGELOG.md +62 -8
- package/DEPLOYMENT.md +175 -0
- package/DEVELOPMENT.md +171 -72
- package/README.md +238 -143
- package/examples/china/README.md +2 -2
- package/examples/energized/README.md +4 -2
- package/examples/sdn/README.md +2 -2
- package/examples/whitelist/README.md +2 -2
- package/package.json +24 -18
- package/src/cli.js +53 -31
- package/src/compile-source.js +4 -3
- package/src/configuration.js +132 -2
- package/src/download.js +27 -0
- package/src/filter.js +2 -2
- package/src/index.d.ts +2 -0
- package/src/index.js +16 -50
- package/src/{transformations/ip-normalize.js → ip-normalize.js} +2 -2
- package/src/schemas/configuration.schema.js +137 -0
- package/src/source-types.js +17 -0
- package/src/transformations/enum.js +49 -0
- package/src/transformations/include.js +1 -1
- package/src/transformations/transform.js +40 -74
- package/src/transformations/validate-allow-ip-and-public-suffix.js +1 -1
- package/src/transformations/validate-allow-ip.js +1 -1
- package/src/utils.js +1284 -0
- package/src/validation-conflicts.js +106 -0
- package/test/cli-error-output.test.js +305 -0
- package/test/cli.test.js +47 -0
- package/test/compile-source.test.js +167 -7
- package/test/configuration.test.js +438 -0
- package/test/download.test.js +260 -0
- package/test/helpers/http-mock.js +94 -0
- package/test/index.test.js +497 -4
- package/test/{transformations/ip-normalize.test.js → ip-normalize.test.js} +1 -1
- package/test/resolve-include.test.js +8 -8
- package/test/transformations/covert-to-ascii.test.js +1 -1
- package/test/transformations/exclude.test.js +2 -2
- package/test/transformations/include.test.js +9 -3
- package/test/transformations/ip-alignment.test.js +1 -1
- package/test/transformations/transform.test.js +66 -1
- package/test/utils.test.js +1559 -0
- package/test/validation-conflicts.test.js +230 -0
- package/src/schemas/configuration.schema.json +0 -369
package/src/utils.js
CHANGED
|
@@ -15,6 +15,59 @@ const MODIFIER_REGEX = /^[a-zA-Z0-9,=~_.-]+$/;
|
|
|
15
15
|
*/
|
|
16
16
|
const OCTET_REGEX = /^\d{1,3}$/;
|
|
17
17
|
|
|
18
|
+
/**
|
|
19
|
+
* Sticky regex to match a URL scheme at the current position.
|
|
20
|
+
*/
|
|
21
|
+
const URL_SCHEME_REGEX = /[a-z][a-z0-9+.-]*:/iy;
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Sticky regex to match the slashes or backslashes that follow a URL scheme.
|
|
25
|
+
*/
|
|
26
|
+
const URL_SLASHES_REGEX = /[/\\]+/y;
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Regex to match a character that ends the authority component of a URL.
|
|
30
|
+
*/
|
|
31
|
+
const URL_AUTHORITY_END_REGEX = /[/\\?#]/;
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Regex to match a character that ends the authority component when it is not
|
|
35
|
+
* part of userinfo. A quote or whitespace inside userinfo is part of the URL,
|
|
36
|
+
* so the scanner decides whether it ends the URL or not.
|
|
37
|
+
*/
|
|
38
|
+
const URL_AUTHORITY_TERMINATOR_REGEX = /[\s"'<>]/;
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Regex to match a `token:digits` pair that can be a port in the authority
|
|
42
|
+
* text before the first `@`. The lookahead accepts a terminator or punctuation
|
|
43
|
+
* after the digits. A bracketed IPv6 literal is matched as one token.
|
|
44
|
+
*
|
|
45
|
+
* The lookbehind requires the token to start at a boundary. Without it, the
|
|
46
|
+
* regex retries from every position inside a long token, for example a long
|
|
47
|
+
* password, and the scan becomes quadratic.
|
|
48
|
+
*/
|
|
49
|
+
const URL_PORT_REGEX = /(?<![^\s"'<>/:])(?:\[[0-9a-f:.]+\]|[^\s"'<>/:]+):\d+(?=[\s"'<>,.;:!?()[\]{}])/gi;
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Modes for the authority parser.
|
|
53
|
+
*
|
|
54
|
+
* `VALUE` treats the text before the first `@` as userinfo only when it looks
|
|
55
|
+
* like userinfo, so it does not mangle prose that contains an email address.
|
|
56
|
+
* It also accepts userinfo when the URL has a path, query, or fragment, or when
|
|
57
|
+
* the value is a whole URL. The path or value shape shows that the text is a
|
|
58
|
+
* URL, not prose, so a quote or a space in the username does not hide the
|
|
59
|
+
* password.
|
|
60
|
+
*
|
|
61
|
+
* `STRICT` treats everything before the last `@` in the authority as userinfo,
|
|
62
|
+
* like the downloader does. It also recognizes malformed values where the
|
|
63
|
+
* password contains `?` or `#`, because the authority ends before the at sign
|
|
64
|
+
* there. Use it only for values that must be a URL.
|
|
65
|
+
*/
|
|
66
|
+
const URL_AUTHORITY_MODE = {
|
|
67
|
+
VALUE: 'value',
|
|
68
|
+
STRICT: 'strict',
|
|
69
|
+
};
|
|
70
|
+
|
|
18
71
|
/**
|
|
19
72
|
* Checks if a string is a valid IPv4 octet (0-255).
|
|
20
73
|
*
|
|
@@ -169,6 +222,1224 @@ function splitByDelimiterWithEscapeCharacter(
|
|
|
169
222
|
return parts;
|
|
170
223
|
}
|
|
171
224
|
|
|
225
|
+
/**
|
|
226
|
+
* Checks whether a URL scheme can start after the given character. A scheme
|
|
227
|
+
* cannot start in the middle of a word or another URL, but it can start after
|
|
228
|
+
* a slash, for example in a path such as `/https://example.org`.
|
|
229
|
+
*
|
|
230
|
+
* @param {String|undefined} char - the character before the scheme.
|
|
231
|
+
* @returns {Boolean} true if a scheme can start after the character.
|
|
232
|
+
*/
|
|
233
|
+
function isSchemeBoundary(char) {
|
|
234
|
+
return char === undefined || !/[a-z0-9_+.-]/i.test(char);
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* Checks whether a scheme-relative URL (`//host`) can start after the given
|
|
239
|
+
* character. A double slash inside a path is not a new URL, so the previous
|
|
240
|
+
* character must not be a path or scheme character.
|
|
241
|
+
*
|
|
242
|
+
* @param {String|undefined} char - the character before the double slash.
|
|
243
|
+
* @returns {Boolean} true if a scheme-relative URL can start after it.
|
|
244
|
+
*/
|
|
245
|
+
function isProtocolRelativeBoundary(char) {
|
|
246
|
+
return char === undefined || !/[a-z0-9_+.:/\\-]/i.test(char);
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/**
|
|
250
|
+
* Checks whether the character ends the authority component of a URL.
|
|
251
|
+
*
|
|
252
|
+
* @param {String} char - character to check.
|
|
253
|
+
* @returns {Boolean} true if the character ends the authority.
|
|
254
|
+
*/
|
|
255
|
+
function endsUrlAuthority(char) {
|
|
256
|
+
return URL_AUTHORITY_END_REGEX.test(char);
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/**
|
|
260
|
+
* Checks whether the character ends the authority component when it is not
|
|
261
|
+
* part of userinfo.
|
|
262
|
+
*
|
|
263
|
+
* @param {String} char - character to check.
|
|
264
|
+
* @returns {Boolean} true if the character ends the authority.
|
|
265
|
+
*/
|
|
266
|
+
function endsUrlAuthorityBeforeUserinfo(char) {
|
|
267
|
+
return URL_AUTHORITY_TERMINATOR_REGEX.test(char);
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* Checks whether a token looks like a host: a dotted name, `localhost`, or an
|
|
272
|
+
* IP literal. The port heuristic uses this to tell a port from a password.
|
|
273
|
+
*
|
|
274
|
+
* @param {String} value - token before a colon.
|
|
275
|
+
* @returns {Boolean} true if the token looks like a host.
|
|
276
|
+
*/
|
|
277
|
+
function looksLikeHost(value) {
|
|
278
|
+
if (value.startsWith('[') && value.endsWith(']')) {
|
|
279
|
+
const inner = value.slice(1, -1);
|
|
280
|
+
return inner.includes(':') && /^[0-9a-f:.]+$/i.test(inner);
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
if (value.toLowerCase() === 'localhost') {
|
|
284
|
+
return true;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
const octets = value.split('.');
|
|
288
|
+
if (octets.length === 4 && octets.every(isValidOctet)) {
|
|
289
|
+
return true;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
// Hostnames can be internationalized and can contain underscores, for
|
|
293
|
+
// example an intranet host such as `my_host.org` or `фильтры.рф`.
|
|
294
|
+
return /^(?:[\p{L}\p{N}_-]+\.)+[\p{L}\p{N}_-]+$/iu.test(value);
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/**
|
|
298
|
+
* Checks whether the text before the first `@` in an authority contains a
|
|
299
|
+
* port. A port looks like a password, so a URL followed by prose such as
|
|
300
|
+
* `https://example.org:8080 by admin@example.com` must stay unchanged. The
|
|
301
|
+
* port can be followed by a terminator or by punctuation, and the text before
|
|
302
|
+
* the colon must look like a host. Otherwise a numeric password followed by
|
|
303
|
+
* punctuation, such as `https://user:1234, secret@example.org`, would be
|
|
304
|
+
* mistaken for a port.
|
|
305
|
+
*
|
|
306
|
+
* @param {String} value - the authority text before the first `@`.
|
|
307
|
+
* @returns {Boolean} true if the text contains a port followed by a terminator
|
|
308
|
+
* or punctuation.
|
|
309
|
+
*/
|
|
310
|
+
function looksLikePort(value) {
|
|
311
|
+
const ports = value.match(URL_PORT_REGEX) || [];
|
|
312
|
+
|
|
313
|
+
return ports.some((port) => looksLikeHost(port.slice(0, port.lastIndexOf(':'))));
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
/**
|
|
317
|
+
* Finds the colon that can start a password in the authority text before the
|
|
318
|
+
* first `@`. A colon inside a bracketed IPv6 literal is part of the host, not
|
|
319
|
+
* a password.
|
|
320
|
+
*
|
|
321
|
+
* @param {String} text - the string to scan.
|
|
322
|
+
* @param {Number} [start] - the index to start scanning from.
|
|
323
|
+
* @param {Number} [end] - the index to stop scanning at.
|
|
324
|
+
* @returns {Number} the index of the password colon, or -1.
|
|
325
|
+
*/
|
|
326
|
+
function indexOfPasswordColon(text, start = 0, end = text.length) {
|
|
327
|
+
let inBrackets = false;
|
|
328
|
+
|
|
329
|
+
for (let i = start; i < end; i += 1) {
|
|
330
|
+
const char = text[i];
|
|
331
|
+
|
|
332
|
+
if (char === '[') {
|
|
333
|
+
inBrackets = true;
|
|
334
|
+
} else if (char === ']') {
|
|
335
|
+
inBrackets = false;
|
|
336
|
+
} else if (char === ':' && !inBrackets) {
|
|
337
|
+
return i;
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
return -1;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
/**
|
|
345
|
+
* Checks whether the text before the first `@` in an authority looks like
|
|
346
|
+
* userinfo rather than prose that follows a URL. Prose can contain whitespace
|
|
347
|
+
* and an email-like `@`, so a colon that starts a password is the signal.
|
|
348
|
+
*
|
|
349
|
+
* `user:p s s` is userinfo, and `https://example.org is maintained by
|
|
350
|
+
* user@example.com` is prose. A colon that is followed by whitespace starts a
|
|
351
|
+
* label in prose, not a password. A colon followed by digits and then a
|
|
352
|
+
* terminator or punctuation is a port, not a password. This is the fast path;
|
|
353
|
+
* `looksLikeValueUserinfo()` handles a password with internal whitespace.
|
|
354
|
+
*
|
|
355
|
+
* @param {String} text - the string being scanned.
|
|
356
|
+
* @param {Number} start - index right after the URL prefix.
|
|
357
|
+
* @param {Number} end - index of the first `@`.
|
|
358
|
+
* @param {Number} firstTerminator - index of the first whitespace, quote, or
|
|
359
|
+
* angle bracket after `start`, bounded by `end`.
|
|
360
|
+
* @returns {Boolean} true if the text looks like userinfo.
|
|
361
|
+
*/
|
|
362
|
+
function looksLikeUserinfo(text, start, end, firstTerminator) {
|
|
363
|
+
if (firstTerminator >= end) {
|
|
364
|
+
return true;
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
const colonIndex = indexOfPasswordColon(text, start, end);
|
|
368
|
+
|
|
369
|
+
if (colonIndex === -1 || colonIndex > firstTerminator) {
|
|
370
|
+
return false;
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
if (looksLikePort(text.slice(start, firstTerminator + 1))) {
|
|
374
|
+
return false;
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
return !/\s/.test(text[colonIndex + 1]);
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
/**
|
|
381
|
+
* Extracts the first word of a value for host checks. Punctuation after a
|
|
382
|
+
* host, for example at the end of a sentence, is ignored. A bracketed IPv6
|
|
383
|
+
* literal is kept intact, because its closing `]` belongs to the host.
|
|
384
|
+
*
|
|
385
|
+
* @param {String} value - the text to check.
|
|
386
|
+
* @returns {String} the first word without trailing punctuation.
|
|
387
|
+
*/
|
|
388
|
+
function firstWordOf(value) {
|
|
389
|
+
const firstWordMatch = value.trim().split(/\s+/, 1)[0];
|
|
390
|
+
const bracketedHostEnd = firstWordMatch.startsWith('[') ? firstWordMatch.indexOf(']') : -1;
|
|
391
|
+
|
|
392
|
+
return bracketedHostEnd === -1
|
|
393
|
+
? firstWordMatch.replace(/[.,;:!?)\]}"'>]+$/, '')
|
|
394
|
+
: firstWordMatch.slice(0, bracketedHostEnd + 1);
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
/**
|
|
398
|
+
* Checks whether the first word of a value looks like the host of a URL:
|
|
399
|
+
* a dotted name, `localhost`, or an IP literal.
|
|
400
|
+
*
|
|
401
|
+
* @param {String} value - the text to check.
|
|
402
|
+
* @returns {Boolean} true if the first word looks like a host.
|
|
403
|
+
*/
|
|
404
|
+
function startsWithHost(value) {
|
|
405
|
+
return looksLikeHost(firstWordOf(value));
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/**
|
|
409
|
+
* Checks whether the first word of a value looks like a single-label intranet
|
|
410
|
+
* host. A dotted name, `localhost`, and an IP literal are handled by
|
|
411
|
+
* `startsWithHost()`. The word must be lowercase, because a capitalized single
|
|
412
|
+
* word is a person's name in a username such as `John Smith` or `J. Smith`.
|
|
413
|
+
*
|
|
414
|
+
* @param {String} value - the text to check.
|
|
415
|
+
* @returns {Boolean} true if the first word looks like an intranet host.
|
|
416
|
+
*/
|
|
417
|
+
function startsWithIntranetHost(value) {
|
|
418
|
+
return /^\p{Ll}[\p{L}\p{N}_-]*$/u.test(firstWordOf(value));
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
/**
|
|
422
|
+
* Checks whether the text before the first `@` is a URL with userinfo rather
|
|
423
|
+
* than prose. The text must contain a password colon with a value after it. A
|
|
424
|
+
* port, and a host-like first word followed by a label, are prose. A password
|
|
425
|
+
* can contain whitespace, so prose is recognized by the text before the colon
|
|
426
|
+
* rather than by the password. A URL can be followed by prose, so a terminator
|
|
427
|
+
* after the host does not make the text before the `@` prose.
|
|
428
|
+
*
|
|
429
|
+
* This covers `https://o'brien:secret@example.org`,
|
|
430
|
+
* `https://John Smith:secret@example.org`,
|
|
431
|
+
* `https://user: secret@example.org`, a password with internal whitespace such
|
|
432
|
+
* as `https://o'brien:pass phrase@example.org`, and the same values followed by
|
|
433
|
+
* text such as `https://John Smith:secret@example.org by admin@example.com`.
|
|
434
|
+
* Prose such as `https://example.org team: contact@example.com`,
|
|
435
|
+
* `https://example.org or mailto:admin@example.com for support`, and
|
|
436
|
+
* `Homepage "https://example.org". Contact: admin@example.com` stays unchanged.
|
|
437
|
+
*
|
|
438
|
+
* @param {String} text - the string being scanned.
|
|
439
|
+
* @param {Number} start - index right after the URL prefix.
|
|
440
|
+
* @param {Number} firstUserinfoEnd - index of the first `@`.
|
|
441
|
+
* @param {Number} authorityEnd - index where the authority ends.
|
|
442
|
+
* @param {{ nextTerminators: Array<Number>, nextNonWhitespace: Array<Number>,
|
|
443
|
+
* nextSpaces: Array<Number>, previousSpaces: Array<Number> }} scan - lookup
|
|
444
|
+
* tables built by `buildAuthorityScan()`.
|
|
445
|
+
* @returns {Boolean} true if the text looks like a URL with userinfo.
|
|
446
|
+
*/
|
|
447
|
+
function looksLikeValueUserinfo(text, start, firstUserinfoEnd, authorityEnd, scan) {
|
|
448
|
+
// A port can only be in the host token, which ends at the first
|
|
449
|
+
// terminator. Checking the whole prefix would scan the rest of the string
|
|
450
|
+
// for every URL start.
|
|
451
|
+
const firstTerminator = Math.min(scan.nextTerminators[start], firstUserinfoEnd);
|
|
452
|
+
// Include the terminator so the port lookahead can see it.
|
|
453
|
+
const portCandidateEnd = Math.min(firstTerminator + 1, firstUserinfoEnd);
|
|
454
|
+
|
|
455
|
+
if (looksLikePort(text.slice(start, portCandidateEnd))) {
|
|
456
|
+
return false;
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
const colonIndex = indexOfPasswordColon(text, start, firstUserinfoEnd);
|
|
460
|
+
|
|
461
|
+
if (colonIndex === -1) {
|
|
462
|
+
return false;
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
if (scan.nextNonWhitespace[colonIndex + 1] >= firstUserinfoEnd) {
|
|
466
|
+
return false;
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
// A host-like first word followed by whitespace before the colon makes the
|
|
470
|
+
// colon a label in prose, whatever follows it. The value starts with the
|
|
471
|
+
// host, so this also keeps `https://example.org or mailto:admin@example.com`
|
|
472
|
+
// and `Homepage "https://example.org". Contact: admin@example.com`
|
|
473
|
+
// unchanged. A username such as `John Smith` does not start with a host,
|
|
474
|
+
// so `https://John Smith: secret@example.org` is still userinfo. A
|
|
475
|
+
// single-label intranet host also requires a label word that does not start
|
|
476
|
+
// with a lowercase letter, so a lowercase username such as
|
|
477
|
+
// `https://admin user:secret@example.org` stays userinfo while
|
|
478
|
+
// `https://x Contact: admin@example.org` and a caseless label such as
|
|
479
|
+
// `https://x 联系: admin@example.org` are prose. A colon that follows a
|
|
480
|
+
// host directly is prose only when a label follows it, so
|
|
481
|
+
// `https://example.org: Contact admin@example.com` stays unchanged and
|
|
482
|
+
// `https://example.org:secret@example.org` is still userinfo. A
|
|
483
|
+
// punctuated host word is prose only when the label colon is followed by
|
|
484
|
+
// whitespace too, so `https://myhost, contact: admin@example.org` stays
|
|
485
|
+
// unchanged while `https://user, name:secret@example.org` is userinfo.
|
|
486
|
+
const firstWordEnd = Math.min(scan.nextSpaces[start], firstUserinfoEnd);
|
|
487
|
+
const firstWord = text.slice(start, firstWordEnd);
|
|
488
|
+
const beforeColon = text.slice(start, colonIndex);
|
|
489
|
+
const beforeColonWord = firstWordOf(beforeColon);
|
|
490
|
+
const rawBeforeColonWord = beforeColon.trim().split(/\s+/, 1)[0];
|
|
491
|
+
const hasPunctuatedHost = beforeColonWord !== rawBeforeColonWord
|
|
492
|
+
&& /^[\p{L}\p{N}_-]+$/u.test(beforeColonWord);
|
|
493
|
+
const label = text.slice(scan.previousSpaces[colonIndex] + 1, colonIndex);
|
|
494
|
+
const hasLabelWord = label.length > 0 && !/^\p{Ll}/u.test(label);
|
|
495
|
+
const colonFollowedByWhitespace = /\s/.test(text[colonIndex + 1]);
|
|
496
|
+
const afterColonStart = scan.nextNonWhitespace[colonIndex + 1];
|
|
497
|
+
const afterColonWord = text.slice(
|
|
498
|
+
afterColonStart,
|
|
499
|
+
Math.min(scan.nextSpaces[afterColonStart], firstUserinfoEnd),
|
|
500
|
+
);
|
|
501
|
+
const hasLabelAfterColon = afterColonWord.length > 0 && !/^\p{Ll}/u.test(afterColonWord);
|
|
502
|
+
const isProseHost = startsWithHost(beforeColon)
|
|
503
|
+
|| (hasPunctuatedHost && colonFollowedByWhitespace && /^\p{Ll}/u.test(label))
|
|
504
|
+
|| (hasLabelWord && startsWithIntranetHost(firstWord))
|
|
505
|
+
|| (colonFollowedByWhitespace
|
|
506
|
+
&& hasLabelAfterColon
|
|
507
|
+
&& startsWithIntranetHost(beforeColon));
|
|
508
|
+
|
|
509
|
+
if (isProseHost && (firstWordEnd < colonIndex || colonFollowedByWhitespace)) {
|
|
510
|
+
return false;
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
// Prose after the URL starts a terminator after the first `@`. The text
|
|
514
|
+
// between the first `@` and that terminator must not contain another `@`:
|
|
515
|
+
// a second `@` means the first one was not the end of the userinfo. The
|
|
516
|
+
// host cannot contain a path, query, or fragment, so the check stops at
|
|
517
|
+
// the authority end. This keeps a rendered document such as
|
|
518
|
+
// `{"homepage":"https://example.org","exclusions":["@@||good.example^"]}`
|
|
519
|
+
// unchanged.
|
|
520
|
+
const terminator = Math.min(scan.nextTerminators[firstUserinfoEnd + 1], authorityEnd);
|
|
521
|
+
// When the text after the first `@` runs straight into a path, query, or
|
|
522
|
+
// fragment, the whole prefix is a URL and the password can contain a
|
|
523
|
+
// literal `@`. The last `@` before the authority end separates the
|
|
524
|
+
// userinfo from the host, so an intermediate `@` is not prose.
|
|
525
|
+
const runsIntoPath = authorityEnd < text.length
|
|
526
|
+
&& scan.nextTerminators[firstUserinfoEnd + 1] >= authorityEnd;
|
|
527
|
+
|
|
528
|
+
if (terminator < text.length) {
|
|
529
|
+
const host = text.slice(firstUserinfoEnd + 1, terminator);
|
|
530
|
+
|
|
531
|
+
if (host.length === 0 || (host.includes('@') && !runsIntoPath)) {
|
|
532
|
+
return false;
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
return true;
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
/**
|
|
540
|
+
* Reads a URL start at the given position.
|
|
541
|
+
*
|
|
542
|
+
* @param {String} text - the string to scan.
|
|
543
|
+
* @param {Number} index - position to check.
|
|
544
|
+
* @returns {{ prefixEnd: Number|null, nextIndex: Number }|null} the index
|
|
545
|
+
* after the URL prefix, the index to resume scanning from when the position
|
|
546
|
+
* starts a scheme that is not a URL, or null when no URL starts here.
|
|
547
|
+
*/
|
|
548
|
+
function readUrlStartAt(text, index) {
|
|
549
|
+
const previousChar = index > 0 ? text[index - 1] : undefined;
|
|
550
|
+
|
|
551
|
+
if (isSchemeBoundary(previousChar)) {
|
|
552
|
+
URL_SCHEME_REGEX.lastIndex = index;
|
|
553
|
+
const schemeMatch = URL_SCHEME_REGEX.exec(text);
|
|
554
|
+
|
|
555
|
+
if (schemeMatch) {
|
|
556
|
+
const schemeEnd = index + schemeMatch[0].length;
|
|
557
|
+
URL_SLASHES_REGEX.lastIndex = schemeEnd;
|
|
558
|
+
const slashesMatch = URL_SLASHES_REGEX.exec(text);
|
|
559
|
+
const slashes = slashesMatch ? slashesMatch[0] : '';
|
|
560
|
+
const slashCount = slashes.length;
|
|
561
|
+
const scheme = schemeMatch[0].slice(0, -1).toLowerCase();
|
|
562
|
+
const isHttp = scheme === 'http' || scheme === 'https';
|
|
563
|
+
const hasBackslash = slashes.includes('\\');
|
|
564
|
+
// A single-letter scheme is a Windows drive letter in a local
|
|
565
|
+
// path such as `C:/user@corp/list.txt`, not a URL. No registered
|
|
566
|
+
// URL scheme is a single letter.
|
|
567
|
+
const isWindowsDriveLetter = scheme.length === 1;
|
|
568
|
+
|
|
569
|
+
// A scheme without slashes is only a URL for HTTP(S): the URL
|
|
570
|
+
// parser normalizes `https:user:pass@host` to a URL with
|
|
571
|
+
// authority. For other schemes, a colon is not enough to tell a
|
|
572
|
+
// URL from text such as `note:value`. Backslashes are accepted
|
|
573
|
+
// only for HTTP(S), so a Windows path such as `C:\dir` is not
|
|
574
|
+
// treated as a URL.
|
|
575
|
+
if (!isWindowsDriveLetter
|
|
576
|
+
&& ((slashCount > 0 && (isHttp || !hasBackslash)) || (slashCount === 0 && isHttp))) {
|
|
577
|
+
return { prefixEnd: schemeEnd + slashCount, nextIndex: schemeEnd + slashCount };
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
return { prefixEnd: null, nextIndex: schemeEnd };
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
if (text[index] === '/' && text[index + 1] === '/') {
|
|
585
|
+
let end = index + 2;
|
|
586
|
+
while (text[end] === '/') {
|
|
587
|
+
end += 1;
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
if (isProtocolRelativeBoundary(previousChar)) {
|
|
591
|
+
return { prefixEnd: end, nextIndex: end };
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
return { prefixEnd: null, nextIndex: end };
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
return null;
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
/**
|
|
601
|
+
* Finds the next URL start in a string and returns the index right after its
|
|
602
|
+
* scheme and slashes.
|
|
603
|
+
*
|
|
604
|
+
* @param {String} text - the string to scan.
|
|
605
|
+
* @param {Number} from - index to start scanning from.
|
|
606
|
+
* @returns {Number|null} the index after the URL prefix, or null if there is
|
|
607
|
+
* no URL start.
|
|
608
|
+
*/
|
|
609
|
+
function findUrlStart(text, from) {
|
|
610
|
+
let i = from;
|
|
611
|
+
|
|
612
|
+
while (i < text.length) {
|
|
613
|
+
const start = readUrlStartAt(text, i);
|
|
614
|
+
|
|
615
|
+
if (start === null) {
|
|
616
|
+
i += 1;
|
|
617
|
+
} else if (start.prefixEnd === null) {
|
|
618
|
+
i = start.nextIndex;
|
|
619
|
+
} else {
|
|
620
|
+
return start.prefixEnd;
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
return null;
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
/**
|
|
628
|
+
* Builds the lookup tables used by the authority scan, so a long string with
|
|
629
|
+
* many URL starts is scanned in linear time. `authorityEnds[i]` is the first
|
|
630
|
+
* `/`, `\`, `?`, or `#` at or after `i`, `atPositions` holds the sorted
|
|
631
|
+
* positions of every `@`, `nextTerminators[i]` is the first whitespace, quote,
|
|
632
|
+
* or angle bracket at or after `i`, `nextNonWhitespace[i]` is the first
|
|
633
|
+
* non-whitespace character at or after `i`, `nextSpaces[i]` is the first
|
|
634
|
+
* whitespace character at or after `i`, `nextPathSeparators[i]` is the first
|
|
635
|
+
* `/` or `\` at or after `i`, and `previousSpaces[i]` is the last whitespace
|
|
636
|
+
* character before `i`.
|
|
637
|
+
*
|
|
638
|
+
* @param {String} text - the string being scanned.
|
|
639
|
+
* @returns {{ authorityEnds: Array<Number>, atPositions: Array<Number>,
|
|
640
|
+
* nextTerminators: Array<Number>, nextNonWhitespace: Array<Number>,
|
|
641
|
+
* nextSpaces: Array<Number>, nextPathSeparators: Array<Number>,
|
|
642
|
+
* previousSpaces: Array<Number> }} the lookup tables.
|
|
643
|
+
*/
|
|
644
|
+
function buildAuthorityScan(text) {
|
|
645
|
+
const authorityEnds = new Array(text.length + 1);
|
|
646
|
+
const nextTerminators = new Array(text.length + 1);
|
|
647
|
+
const nextNonWhitespace = new Array(text.length + 1);
|
|
648
|
+
const nextSpaces = new Array(text.length + 1);
|
|
649
|
+
const nextPathSeparators = new Array(text.length + 1);
|
|
650
|
+
const previousSpaces = new Array(text.length + 1);
|
|
651
|
+
authorityEnds[text.length] = text.length;
|
|
652
|
+
nextTerminators[text.length] = text.length;
|
|
653
|
+
nextNonWhitespace[text.length] = text.length;
|
|
654
|
+
nextSpaces[text.length] = text.length;
|
|
655
|
+
nextPathSeparators[text.length] = text.length;
|
|
656
|
+
previousSpaces[0] = -1;
|
|
657
|
+
|
|
658
|
+
for (let i = text.length - 1; i >= 0; i -= 1) {
|
|
659
|
+
const char = text[i];
|
|
660
|
+
authorityEnds[i] = endsUrlAuthority(char) ? i : authorityEnds[i + 1];
|
|
661
|
+
nextTerminators[i] = endsUrlAuthorityBeforeUserinfo(char) ? i : nextTerminators[i + 1];
|
|
662
|
+
nextNonWhitespace[i] = /\s/.test(char) ? nextNonWhitespace[i + 1] : i;
|
|
663
|
+
nextSpaces[i] = /\s/.test(char) ? i : nextSpaces[i + 1];
|
|
664
|
+
nextPathSeparators[i] = char === '/' || char === '\\' ? i : nextPathSeparators[i + 1];
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
for (let i = 0; i < text.length; i += 1) {
|
|
668
|
+
previousSpaces[i + 1] = /\s/.test(text[i]) ? i : previousSpaces[i];
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
const atPositions = [];
|
|
672
|
+
for (let i = 0; i < text.length; i += 1) {
|
|
673
|
+
if (text[i] === '@') {
|
|
674
|
+
atPositions.push(i);
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
return {
|
|
679
|
+
authorityEnds,
|
|
680
|
+
atPositions,
|
|
681
|
+
nextTerminators,
|
|
682
|
+
nextNonWhitespace,
|
|
683
|
+
nextSpaces,
|
|
684
|
+
nextPathSeparators,
|
|
685
|
+
previousSpaces,
|
|
686
|
+
};
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
/**
|
|
690
|
+
* Checks whether a host token looks like a host. The token can carry a port,
|
|
691
|
+
* for example `example.org:8080`, and a trailing colon or comma that belongs
|
|
692
|
+
* to prose, for example `example.org:` in `https://example.org: Contact ...`.
|
|
693
|
+
*
|
|
694
|
+
* @param {String} token - the host token.
|
|
695
|
+
* @returns {Boolean} true if the token looks like a host.
|
|
696
|
+
*/
|
|
697
|
+
function looksLikeHostToken(token) {
|
|
698
|
+
const hostName = firstWordOf(token).replace(/:\d+$/, '');
|
|
699
|
+
|
|
700
|
+
return looksLikeHost(hostName);
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
/**
|
|
704
|
+
* Finds the `@` that separates the userinfo from the host in value mode.
|
|
705
|
+
*
|
|
706
|
+
* The URL parser and the downloader accept a literal `@` inside a password, so
|
|
707
|
+
* the first `@` is not always the separator. The function keeps the first `@`
|
|
708
|
+
* when the token after it looks like a host. Otherwise it scans the later `@`
|
|
709
|
+
* positions for the first host-like token, so
|
|
710
|
+
* `https://user:p@ss word@example.org` becomes `https://example.org` instead
|
|
711
|
+
* of the partial `https://ss word@example.org`.
|
|
712
|
+
*
|
|
713
|
+
* A single-label token after the first `@` can be either the rest of a
|
|
714
|
+
* password or an intranet host followed by prose. Trailing punctuation or a
|
|
715
|
+
* label colon after the token shows prose, so the first `@` stays the
|
|
716
|
+
* separator. This keeps `https://user:secret@intranet Contact: admin@...`
|
|
717
|
+
* unchanged.
|
|
718
|
+
*
|
|
719
|
+
* @param {String} text - the string being scanned.
|
|
720
|
+
* @param {Number} firstUserinfoEnd - index of the first `@` in the authority.
|
|
721
|
+
* @param {Number} authorityEnd - index where the authority ends.
|
|
722
|
+
* @param {{ atPositions: Array<Number>, nextTerminators: Array<Number> }} scan -
|
|
723
|
+
* lookup tables built by `buildAuthorityScan()`.
|
|
724
|
+
* @returns {Number} the index of the `@` that separates userinfo from the
|
|
725
|
+
* host.
|
|
726
|
+
*/
|
|
727
|
+
function findValueUserinfoEnd(text, firstUserinfoEnd, authorityEnd, scan) {
|
|
728
|
+
const firstAtIndex = _.sortedIndex(scan.atPositions, firstUserinfoEnd);
|
|
729
|
+
const atEndIndex = _.sortedIndex(scan.atPositions, authorityEnd);
|
|
730
|
+
let fallback = firstUserinfoEnd;
|
|
731
|
+
|
|
732
|
+
for (let i = firstAtIndex; i < atEndIndex; i += 1) {
|
|
733
|
+
const at = scan.atPositions[i];
|
|
734
|
+
const terminator = Math.min(scan.nextTerminators[at + 1], authorityEnd);
|
|
735
|
+
const host = text.slice(at + 1, terminator);
|
|
736
|
+
|
|
737
|
+
if (looksLikeHostToken(host)) {
|
|
738
|
+
return at;
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
// A token with `@` sits inside the password, so the host still
|
|
742
|
+
// follows. An empty token means two adjacent `@` characters.
|
|
743
|
+
if (host.length > 0 && !host.includes('@')) {
|
|
744
|
+
const nextAt = i + 1 < atEndIndex ? scan.atPositions[i + 1] : authorityEnd;
|
|
745
|
+
const between = text.slice(terminator, nextAt);
|
|
746
|
+
const hasProseLabel = /[.,;:]$/u.test(host) || /\s+\S+:/u.test(between);
|
|
747
|
+
|
|
748
|
+
if (hasProseLabel) {
|
|
749
|
+
return firstUserinfoEnd;
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
fallback = at;
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
return fallback;
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
/**
|
|
760
|
+
* Checks that the text after an at sign looks like the host and the rest of
|
|
761
|
+
* a URL. The check tells malformed userinfo from an at sign in a query or a
|
|
762
|
+
* fragment.
|
|
763
|
+
*
|
|
764
|
+
* The host can be followed by a path, a query, a fragment, or the end of the
|
|
765
|
+
* value. A single-label host in these forms is ambiguous with an at sign in
|
|
766
|
+
* the query or fragment of a valid URL whose host has no dot, for example
|
|
767
|
+
* `https://fileserver:?token=a@b`. A credential leak in a published list is
|
|
768
|
+
* worse than a wrong host in a display message, so the ambiguous form is
|
|
769
|
+
* treated as userinfo.
|
|
770
|
+
*
|
|
771
|
+
* @param {String} text - the string being scanned.
|
|
772
|
+
* @param {Number} at - index of the at sign.
|
|
773
|
+
* @param {{ nextTerminators: Array<Number> }} scan - lookup tables built by
|
|
774
|
+
* `buildAuthorityScan()`.
|
|
775
|
+
* @returns {Boolean} true if the text after the at sign looks like a URL.
|
|
776
|
+
*/
|
|
777
|
+
function hasUrlTailAfterAt(text, at, scan) {
|
|
778
|
+
const tail = text.slice(at + 1, scan.nextTerminators[at + 1]);
|
|
779
|
+
|
|
780
|
+
return /^[^/\\?#:]*[.:]/i.test(tail)
|
|
781
|
+
|| /^localhost(?=[/\\?#])/i.test(tail)
|
|
782
|
+
|| /^[^/\\?#:.]+(?:[/\\?#]|$)/.test(tail);
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
/**
|
|
786
|
+
* Checks that the text before an at sign is malformed userinfo whose password
|
|
787
|
+
* contains `?` or `#`.
|
|
788
|
+
*
|
|
789
|
+
* A dotted part before the colon is a host with a port, and `localhost` or an
|
|
790
|
+
* IPv6 literal is a host, so these values are not userinfo. The password colon
|
|
791
|
+
* must come before the first `?`, `#`, or whitespace.
|
|
792
|
+
*
|
|
793
|
+
* @param {String} userinfo - the text between the URL prefix and the at sign.
|
|
794
|
+
* @returns {Boolean} true if the text looks like malformed userinfo.
|
|
795
|
+
*/
|
|
796
|
+
function isMalformedStrictUserinfo(userinfo) {
|
|
797
|
+
const separatorIndex = indexOfPasswordColon(userinfo);
|
|
798
|
+
|
|
799
|
+
if (separatorIndex === -1) {
|
|
800
|
+
return false;
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
const ambiguousIndex = userinfo.search(/[?#\s]/);
|
|
804
|
+
|
|
805
|
+
if (ambiguousIndex !== -1 && separatorIndex > ambiguousIndex) {
|
|
806
|
+
return false;
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
const username = userinfo.slice(0, separatorIndex);
|
|
810
|
+
|
|
811
|
+
return !username.includes('.')
|
|
812
|
+
&& username.toLowerCase() !== 'localhost'
|
|
813
|
+
&& !username.startsWith('[');
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
/**
|
|
817
|
+
* Finds the at sign that separates malformed userinfo from the host when the
|
|
818
|
+
* password contains `?` or `#`.
|
|
819
|
+
*
|
|
820
|
+
* The authority ends at the first `?` or `#`, before the at sign, so the
|
|
821
|
+
* normal strict scan does not see the at sign. The downloader recognizes such
|
|
822
|
+
* values for display redaction. Accept the form only when the password colon
|
|
823
|
+
* comes before the first `?`, `#`, or whitespace, the part before the colon is
|
|
824
|
+
* not a host, and the text after the at sign looks like the rest of a URL.
|
|
825
|
+
* A port followed by a path keeps a valid URL with an at sign in the query
|
|
826
|
+
* unchanged, for example `https://myhost:8080/path?email=user@example.com`.
|
|
827
|
+
*
|
|
828
|
+
* The search is bounded by the first value terminator and the first path
|
|
829
|
+
* separator after the authority end and starts from that bound, so a long
|
|
830
|
+
* string with many at signs is scanned once per value instead of once per at
|
|
831
|
+
* sign. The path bound matches the library parser, which stops the malformed
|
|
832
|
+
* userinfo match at a `/` or `\`.
|
|
833
|
+
*
|
|
834
|
+
* @param {String} text - the string being scanned.
|
|
835
|
+
* @param {Number} start - index right after the URL prefix.
|
|
836
|
+
* @param {{ atPositions: Array<Number>, nextTerminators: Array<Number>,
|
|
837
|
+
* nextPathSeparators: Array<Number> }} scan - lookup tables built by
|
|
838
|
+
* `buildAuthorityScan()`.
|
|
839
|
+
* @param {Number} authorityEnd - index where the authority ends.
|
|
840
|
+
* @returns {Number} the index of the at sign, or -1.
|
|
841
|
+
*/
|
|
842
|
+
function findMalformedStrictUserinfoEnd(text, start, scan, authorityEnd) {
|
|
843
|
+
// Bound the search by the first value terminator and by the first path
|
|
844
|
+
// separator after the authority end. The library's malformed-userinfo
|
|
845
|
+
// match stops at a `/` or `\`, so an at sign in a later path is not a
|
|
846
|
+
// separator. With the single-label tail rule below, an at sign in the
|
|
847
|
+
// path would otherwise be taken as the separator.
|
|
848
|
+
const limit = Math.min(
|
|
849
|
+
scan.nextTerminators[authorityEnd],
|
|
850
|
+
scan.nextPathSeparators[authorityEnd],
|
|
851
|
+
text.length,
|
|
852
|
+
);
|
|
853
|
+
const lastAt = _.sortedLastIndex(scan.atPositions, limit - 1) - 1;
|
|
854
|
+
|
|
855
|
+
for (let i = lastAt; i >= 0; i -= 1) {
|
|
856
|
+
const at = scan.atPositions[i];
|
|
857
|
+
|
|
858
|
+
if (at < start) {
|
|
859
|
+
break;
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
if (hasUrlTailAfterAt(text, at, scan)
|
|
863
|
+
&& isMalformedStrictUserinfo(text.slice(start, at))) {
|
|
864
|
+
return at;
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
return -1;
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
/**
|
|
872
|
+
* Reads the authority component of a URL and splits it into the sanitized
|
|
873
|
+
* value and the end of the URL.
|
|
874
|
+
*
|
|
875
|
+
* @param {String} text - the string being scanned.
|
|
876
|
+
* @param {Number} start - index right after the URL prefix.
|
|
877
|
+
* @param {String} mode - one of `URL_AUTHORITY_MODE`.
|
|
878
|
+
* @param {{ authorityEnds: Array<Number>, atPositions: Array<Number>,
|
|
879
|
+
* nextTerminators: Array<Number>, nextNonWhitespace: Array<Number>,
|
|
880
|
+
* nextSpaces: Array<Number>, nextPathSeparators: Array<Number>,
|
|
881
|
+
* previousSpaces: Array<Number> }} scan -
|
|
882
|
+
* lookup tables built by `buildAuthorityScan()`.
|
|
883
|
+
* @returns {{ value: String, rawEnd: Number }} the sanitized authority and the
|
|
884
|
+
* index where it ends.
|
|
885
|
+
*/
|
|
886
|
+
function readUrlAuthority(text, start, mode, scan) {
|
|
887
|
+
const authorityEnd = scan.authorityEnds[start];
|
|
888
|
+
let userinfoEnd = -1;
|
|
889
|
+
|
|
890
|
+
if (mode === URL_AUTHORITY_MODE.STRICT) {
|
|
891
|
+
// The downloader takes everything before the last `@` in the
|
|
892
|
+
// authority as userinfo. Use this only for values that must be a URL,
|
|
893
|
+
// because prose after a URL can also contain an `@`.
|
|
894
|
+
const lastAt = _.sortedLastIndex(scan.atPositions, authorityEnd - 1) - 1;
|
|
895
|
+
userinfoEnd = lastAt >= 0 && scan.atPositions[lastAt] >= start
|
|
896
|
+
? scan.atPositions[lastAt]
|
|
897
|
+
: -1;
|
|
898
|
+
|
|
899
|
+
if (userinfoEnd === -1) {
|
|
900
|
+
userinfoEnd = findMalformedStrictUserinfoEnd(text, start, scan, authorityEnd);
|
|
901
|
+
}
|
|
902
|
+
} else {
|
|
903
|
+
// The first `@` shows where userinfo could end. `looksLikeUserinfo()`
|
|
904
|
+
// rejects prose that follows a URL and contains an email address.
|
|
905
|
+
const firstAt = _.sortedIndex(scan.atPositions, start);
|
|
906
|
+
const firstUserinfoEnd = firstAt < scan.atPositions.length
|
|
907
|
+
&& scan.atPositions[firstAt] < authorityEnd
|
|
908
|
+
? scan.atPositions[firstAt]
|
|
909
|
+
: -1;
|
|
910
|
+
|
|
911
|
+
if (firstUserinfoEnd !== -1) {
|
|
912
|
+
const firstTerminator = Math.min(scan.nextTerminators[start], firstUserinfoEnd);
|
|
913
|
+
const isValueUserinfo = mode === URL_AUTHORITY_MODE.VALUE
|
|
914
|
+
&& looksLikeValueUserinfo(text, start, firstUserinfoEnd, authorityEnd, scan);
|
|
915
|
+
const isUserinfo = looksLikeUserinfo(
|
|
916
|
+
text,
|
|
917
|
+
start,
|
|
918
|
+
firstUserinfoEnd,
|
|
919
|
+
firstTerminator,
|
|
920
|
+
) || isValueUserinfo;
|
|
921
|
+
|
|
922
|
+
if (isUserinfo) {
|
|
923
|
+
// Userinfo can contain quote, whitespace, and `@` characters,
|
|
924
|
+
// but the host cannot. `findValueUserinfoEnd()` keeps a later
|
|
925
|
+
// email in prose such as `https://host and a@b` intact and
|
|
926
|
+
// also handles a literal `@` in a password.
|
|
927
|
+
userinfoEnd = findValueUserinfoEnd(
|
|
928
|
+
text,
|
|
929
|
+
firstUserinfoEnd,
|
|
930
|
+
authorityEnd,
|
|
931
|
+
scan,
|
|
932
|
+
);
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
}
|
|
936
|
+
|
|
937
|
+
const authorityStart = userinfoEnd === -1 ? start : userinfoEnd + 1;
|
|
938
|
+
const rawEnd = Math.min(
|
|
939
|
+
scan.nextTerminators[authorityStart],
|
|
940
|
+
scan.authorityEnds[authorityStart],
|
|
941
|
+
);
|
|
942
|
+
|
|
943
|
+
const value = text.slice(authorityStart, rawEnd);
|
|
944
|
+
|
|
945
|
+
return { value, rawEnd };
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
/**
|
|
949
|
+
* Removes the userinfo component from every URL in a string, using the given
|
|
950
|
+
* authority parser mode.
|
|
951
|
+
*
|
|
952
|
+
* The function does string surgery on the authority component, so the rest of
|
|
953
|
+
* the URL stays byte-identical. It does not parse and re-serialize the URL
|
|
954
|
+
* with `URL`, because that normalizes the URL. It recognizes an optional
|
|
955
|
+
* scheme followed by any number of slashes or backslashes, a scheme with no
|
|
956
|
+
* slashes for HTTP(S), and scheme-relative URLs that start with two or more
|
|
957
|
+
* slashes.
|
|
958
|
+
*
|
|
959
|
+
* @param {String} url - string that may contain URLs with credentials.
|
|
960
|
+
* @param {String} mode - one of `URL_AUTHORITY_MODE`.
|
|
961
|
+
* @returns {String} the string without the userinfo components, or the input
|
|
962
|
+
* unchanged when it has no URL with userinfo.
|
|
963
|
+
*/
|
|
964
|
+
function stripUrlCredentialsWithMode(url, mode) {
|
|
965
|
+
if (!_.isString(url)) {
|
|
966
|
+
return url;
|
|
967
|
+
}
|
|
968
|
+
|
|
969
|
+
const scan = buildAuthorityScan(url);
|
|
970
|
+
let result = '';
|
|
971
|
+
let copiedUntil = 0;
|
|
972
|
+
let searchFrom = 0;
|
|
973
|
+
|
|
974
|
+
while (searchFrom < url.length) {
|
|
975
|
+
const prefixEnd = findUrlStart(url, searchFrom);
|
|
976
|
+
|
|
977
|
+
if (prefixEnd === null) {
|
|
978
|
+
break;
|
|
979
|
+
}
|
|
980
|
+
|
|
981
|
+
const authority = readUrlAuthority(url, prefixEnd, mode, scan);
|
|
982
|
+
result += url.slice(copiedUntil, prefixEnd);
|
|
983
|
+
result += authority.value;
|
|
984
|
+
// Resume the scan at the end of the authority, not at the end of the
|
|
985
|
+
// URL. A URL can sit inside the path, query, or fragment of another
|
|
986
|
+
// URL, for example a proxy URL with the target in its query. The rest
|
|
987
|
+
// of the outer URL is copied by the next iteration or by the final
|
|
988
|
+
// slice, so nested credentials are not copied verbatim.
|
|
989
|
+
copiedUntil = authority.rawEnd;
|
|
990
|
+
searchFrom = authority.rawEnd;
|
|
991
|
+
}
|
|
992
|
+
|
|
993
|
+
result += url.slice(copiedUntil);
|
|
994
|
+
return result;
|
|
995
|
+
}
|
|
996
|
+
|
|
997
|
+
/**
|
|
998
|
+
* Removes the userinfo component (the `user:password@` part) from every URL in
|
|
999
|
+
* a string. Use this for free text and for values of an unknown shape: the
|
|
1000
|
+
* function keeps prose that contains an email address unchanged.
|
|
1001
|
+
*
|
|
1002
|
+
* Do not scan a rendered document with this function, for example
|
|
1003
|
+
* `JSON.stringify(configuration)`. A path-less URL can make the text up to a
|
|
1004
|
+
* later `@` look like userinfo and corrupt the document. Use
|
|
1005
|
+
* `sanitizeConfigurationForLog()` for a configuration dump or a validation
|
|
1006
|
+
* error.
|
|
1007
|
+
*
|
|
1008
|
+
* The parser is a heuristic: it keeps text that reads as prose. Two shapes are
|
|
1009
|
+
* lexically ambiguous and stay unresolved. The value
|
|
1010
|
+
* `https://john Smith: secret@example.org` is kept as prose, so the password
|
|
1011
|
+
* can survive. The value `https://user:secret@intranet by admin@example.com`
|
|
1012
|
+
* is read as a credential, so the prose after the intranet host is removed.
|
|
1013
|
+
* Use `stripKnownUrlCredentials()` for values that must be a URL.
|
|
1014
|
+
*
|
|
1015
|
+
* @param {String} url - string that may contain URLs with credentials.
|
|
1016
|
+
* @returns {String} the string without the userinfo components, or the input
|
|
1017
|
+
* unchanged when it has no URL with userinfo.
|
|
1018
|
+
*/
|
|
1019
|
+
function stripUrlCredentials(url) {
|
|
1020
|
+
return stripUrlCredentialsWithMode(url, URL_AUTHORITY_MODE.VALUE);
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
/**
|
|
1024
|
+
* Removes the userinfo component from every URL in a value that must be a URL,
|
|
1025
|
+
* such as `source` or `homepage`.
|
|
1026
|
+
*
|
|
1027
|
+
* Unlike `stripUrlCredentials()`, this function has no prose guard. It treats
|
|
1028
|
+
* everything before the last `@` in the authority as userinfo, like the
|
|
1029
|
+
* downloader does. It also recognizes malformed values where the password
|
|
1030
|
+
* contains `?` or `#`: the authority ends before the at sign there, so the
|
|
1031
|
+
* normal strict scan cannot see it. Use this function for known URL values,
|
|
1032
|
+
* where a quote or a space in the username must not hide the password.
|
|
1033
|
+
*
|
|
1034
|
+
* A single-label host with a port and an at sign in the query but no path is
|
|
1035
|
+
* lexically identical to a digit-first password. The password form wins, so
|
|
1036
|
+
* such a value loses its host to avoid a leak. A path after the port tells the
|
|
1037
|
+
* forms apart, so `https://myhost:8080/path?email=user@example.com` stays
|
|
1038
|
+
* unchanged.
|
|
1039
|
+
*
|
|
1040
|
+
* @param {String} url - URL value.
|
|
1041
|
+
* @returns {String} the URL without the userinfo component, or the input
|
|
1042
|
+
* unchanged when it has no userinfo.
|
|
1043
|
+
*/
|
|
1044
|
+
function stripKnownUrlCredentials(url) {
|
|
1045
|
+
return stripUrlCredentialsWithMode(url, URL_AUTHORITY_MODE.STRICT);
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
/**
|
|
1049
|
+
* Removes the userinfo component from a string value whose shape is not known
|
|
1050
|
+
* in advance.
|
|
1051
|
+
*
|
|
1052
|
+
* The function first tries to parse the whole value as a URL. When the URL has
|
|
1053
|
+
* a path, query, or fragment, a quote or a space in the username does not hide
|
|
1054
|
+
* the password. A port followed by prose, such as
|
|
1055
|
+
* `https://example.org:8080 by admin@example.com`, stays unchanged.
|
|
1056
|
+
*
|
|
1057
|
+
* Leading whitespace is kept, so a value such as
|
|
1058
|
+
* ` https://user:password@example.org/` is sanitized without changing its
|
|
1059
|
+
* prefix. When the value does not start with a URL, or when the text before
|
|
1060
|
+
* the first `@` looks like prose, the function falls back to the scanner. The
|
|
1061
|
+
* scanner also covers any text after the first URL, so prefixed values such as
|
|
1062
|
+
* `See https://user:password@example.org/` are sanitized too.
|
|
1063
|
+
*
|
|
1064
|
+
* Use `stripKnownUrlCredentials()` for values that must be a URL, because that
|
|
1065
|
+
* function has no prose guard and strips exactly what the downloader treats as
|
|
1066
|
+
* userinfo.
|
|
1067
|
+
*
|
|
1068
|
+
* @param {String} url - string to sanitize.
|
|
1069
|
+
* @returns {String} the string without the userinfo component, or the input
|
|
1070
|
+
* unchanged when it has no userinfo.
|
|
1071
|
+
*/
|
|
1072
|
+
function stripUrlValueCredentials(url) {
|
|
1073
|
+
if (!_.isString(url)) {
|
|
1074
|
+
return url;
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1077
|
+
const valueStart = url.search(/\S/);
|
|
1078
|
+
if (valueStart === -1) {
|
|
1079
|
+
return url;
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1082
|
+
const start = readUrlStartAt(url, valueStart);
|
|
1083
|
+
if (start === null || start.prefixEnd === null) {
|
|
1084
|
+
return stripUrlCredentials(url);
|
|
1085
|
+
}
|
|
1086
|
+
|
|
1087
|
+
const { prefixEnd } = start;
|
|
1088
|
+
const authority = readUrlAuthority(
|
|
1089
|
+
url,
|
|
1090
|
+
prefixEnd,
|
|
1091
|
+
URL_AUTHORITY_MODE.VALUE,
|
|
1092
|
+
buildAuthorityScan(url),
|
|
1093
|
+
);
|
|
1094
|
+
|
|
1095
|
+
const sanitized = url.slice(0, prefixEnd)
|
|
1096
|
+
+ authority.value
|
|
1097
|
+
+ url.slice(authority.rawEnd);
|
|
1098
|
+
|
|
1099
|
+
// The value can have more text after the first URL, and the first URL can
|
|
1100
|
+
// contain another URL in its path, query, or fragment. The scanner covers
|
|
1101
|
+
// both.
|
|
1102
|
+
return stripUrlCredentials(sanitized);
|
|
1103
|
+
}
|
|
1104
|
+
|
|
1105
|
+
/**
|
|
1106
|
+
* Replaces known raw values in a string and then removes URL credentials.
|
|
1107
|
+
*
|
|
1108
|
+
* The replacement runs first because a password can contain a literal `@`
|
|
1109
|
+
* followed by whitespace. The scanner uses the first `@` with a prose guard, so
|
|
1110
|
+
* it can leave the rest of such a password in the text. A caller that knows the
|
|
1111
|
+
* exact source URL can sanitize it with the strict parser before the scanner
|
|
1112
|
+
* runs.
|
|
1113
|
+
*
|
|
1114
|
+
* @param {String} value - the string to sanitize.
|
|
1115
|
+
* @param {Array<Array<String>>} replacements - pairs of a raw value and its
|
|
1116
|
+
* sanitized replacement.
|
|
1117
|
+
* @returns {String} the sanitized string.
|
|
1118
|
+
*/
|
|
1119
|
+
function sanitizeErrorString(value, replacements) {
|
|
1120
|
+
let result = value;
|
|
1121
|
+
|
|
1122
|
+
replacements.forEach(([raw, sanitized]) => {
|
|
1123
|
+
if (_.isString(raw) && raw !== sanitized) {
|
|
1124
|
+
result = result.split(raw).join(sanitized);
|
|
1125
|
+
}
|
|
1126
|
+
});
|
|
1127
|
+
|
|
1128
|
+
return stripUrlCredentials(result);
|
|
1129
|
+
}
|
|
1130
|
+
|
|
1131
|
+
/**
|
|
1132
|
+
* Removes URL credentials from an error and its `cause` and `errors` chains.
|
|
1133
|
+
* The error is modified in place, so its type and stack location are
|
|
1134
|
+
* preserved.
|
|
1135
|
+
*
|
|
1136
|
+
* @param {*} error - the error to sanitize.
|
|
1137
|
+
* @param {Array<Array<String>>} [replacements] - pairs of a raw value and its
|
|
1138
|
+
* sanitized replacement, applied before the generic scan.
|
|
1139
|
+
* @param {WeakSet<Object>} [seen] - objects that have already been visited.
|
|
1140
|
+
* @returns {*} the sanitized error.
|
|
1141
|
+
*/
|
|
1142
|
+
function stripErrorCredentials(error, replacements = [], seen = new WeakSet()) {
|
|
1143
|
+
if (!_.isObject(error) || seen.has(error)) {
|
|
1144
|
+
return error;
|
|
1145
|
+
}
|
|
1146
|
+
seen.add(error);
|
|
1147
|
+
|
|
1148
|
+
if (_.isString(error.message)) {
|
|
1149
|
+
Object.assign(error, { message: sanitizeErrorString(error.message, replacements) });
|
|
1150
|
+
}
|
|
1151
|
+
if (_.isString(error.stack)) {
|
|
1152
|
+
Object.assign(error, { stack: sanitizeErrorString(error.stack, replacements) });
|
|
1153
|
+
}
|
|
1154
|
+
if (_.isString(error.path)) {
|
|
1155
|
+
Object.assign(error, { path: sanitizeErrorString(error.path, replacements) });
|
|
1156
|
+
}
|
|
1157
|
+
|
|
1158
|
+
if (Array.isArray(error.errors)) {
|
|
1159
|
+
Object.assign(error, {
|
|
1160
|
+
errors: error.errors.map((cause) => (
|
|
1161
|
+
_.isString(cause)
|
|
1162
|
+
? sanitizeErrorString(cause, replacements)
|
|
1163
|
+
: stripErrorCredentials(cause, replacements, seen)
|
|
1164
|
+
)),
|
|
1165
|
+
});
|
|
1166
|
+
}
|
|
1167
|
+
|
|
1168
|
+
if (_.isString(error.cause)) {
|
|
1169
|
+
Object.assign(error, { cause: sanitizeErrorString(error.cause, replacements) });
|
|
1170
|
+
} else {
|
|
1171
|
+
stripErrorCredentials(error.cause, replacements, seen);
|
|
1172
|
+
}
|
|
1173
|
+
|
|
1174
|
+
return error;
|
|
1175
|
+
}
|
|
1176
|
+
|
|
1177
|
+
/**
|
|
1178
|
+
* Sanitizes URL-like object keys in place, because `cloneDeepWith` visits
|
|
1179
|
+
* values only.
|
|
1180
|
+
*
|
|
1181
|
+
* @param {*} value - configuration value to walk.
|
|
1182
|
+
* @param {WeakSet<Object>} seen - objects that have already been visited.
|
|
1183
|
+
*/
|
|
1184
|
+
function sanitizeConfigurationKeys(value, seen) {
|
|
1185
|
+
if (!_.isObject(value) || seen.has(value)) {
|
|
1186
|
+
return;
|
|
1187
|
+
}
|
|
1188
|
+
seen.add(value);
|
|
1189
|
+
|
|
1190
|
+
Object.entries(value).forEach(([key, child]) => {
|
|
1191
|
+
// Keys are never prose, so use the strict parser: a URL-like key can
|
|
1192
|
+
// carry whitespace in the username and in the password, which the
|
|
1193
|
+
// prose-tolerant parser would leave untouched.
|
|
1194
|
+
const sanitizedKey = stripKnownUrlCredentials(key);
|
|
1195
|
+
|
|
1196
|
+
if (sanitizedKey !== key) {
|
|
1197
|
+
Reflect.deleteProperty(value, key);
|
|
1198
|
+
Object.defineProperty(value, sanitizedKey, {
|
|
1199
|
+
value: child,
|
|
1200
|
+
enumerable: true,
|
|
1201
|
+
configurable: true,
|
|
1202
|
+
writable: true,
|
|
1203
|
+
});
|
|
1204
|
+
}
|
|
1205
|
+
|
|
1206
|
+
sanitizeConfigurationKeys(child, seen);
|
|
1207
|
+
});
|
|
1208
|
+
}
|
|
1209
|
+
|
|
1210
|
+
/**
|
|
1211
|
+
* Configuration fields that must contain a URL. A test derives the set from
|
|
1212
|
+
* the `x-credential-url` markers in `src/schemas/configuration.schema.js`, so
|
|
1213
|
+
* a marked schema property without an entry here fails the test.
|
|
1214
|
+
*/
|
|
1215
|
+
const URL_VALUE_FIELDS = new Set([
|
|
1216
|
+
'homepage',
|
|
1217
|
+
'source',
|
|
1218
|
+
]);
|
|
1219
|
+
|
|
1220
|
+
/**
|
|
1221
|
+
* Configuration fields that must contain a URL or a list of URLs. A test
|
|
1222
|
+
* derives the set from the `x-credential-url` markers in
|
|
1223
|
+
* `src/schemas/configuration.schema.js`.
|
|
1224
|
+
*/
|
|
1225
|
+
const URL_LIST_FIELDS = new Set([
|
|
1226
|
+
'sources',
|
|
1227
|
+
'exclusions_sources',
|
|
1228
|
+
'inclusions_sources',
|
|
1229
|
+
]);
|
|
1230
|
+
|
|
1231
|
+
/**
|
|
1232
|
+
* Sanitizes a known URL field value or a list of URL field values with the
|
|
1233
|
+
* strict parser. Other values are left for the recursive walk.
|
|
1234
|
+
*
|
|
1235
|
+
* @param {*} value - field value.
|
|
1236
|
+
* @returns {*} sanitized field value.
|
|
1237
|
+
*/
|
|
1238
|
+
function sanitizeKnownUrlField(value) {
|
|
1239
|
+
if (_.isString(value)) {
|
|
1240
|
+
return stripKnownUrlCredentials(value);
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
if (Array.isArray(value)) {
|
|
1244
|
+
return value.map((item) => (
|
|
1245
|
+
_.isString(item) ? stripKnownUrlCredentials(item) : item
|
|
1246
|
+
));
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
return value;
|
|
1250
|
+
}
|
|
1251
|
+
|
|
1252
|
+
/**
|
|
1253
|
+
* Sanitizes known URL fields in place with the strict parser. The generic copy
|
|
1254
|
+
* uses a prose guard, so a quote or a space in the username of a `source` or
|
|
1255
|
+
* `homepage` value would survive it. The downloader treats such a value as
|
|
1256
|
+
* userinfo, so the strict parser runs on these fields.
|
|
1257
|
+
*
|
|
1258
|
+
* @param {*} value - configuration value to walk.
|
|
1259
|
+
* @param {WeakSet<Object>} seen - objects that have already been visited.
|
|
1260
|
+
*/
|
|
1261
|
+
function sanitizeKnownUrlFields(value, seen) {
|
|
1262
|
+
if (!_.isObject(value) || seen.has(value)) {
|
|
1263
|
+
return;
|
|
1264
|
+
}
|
|
1265
|
+
seen.add(value);
|
|
1266
|
+
|
|
1267
|
+
Object.entries(value).forEach(([key, child]) => {
|
|
1268
|
+
if (URL_VALUE_FIELDS.has(key) || URL_LIST_FIELDS.has(key)) {
|
|
1269
|
+
Object.assign(value, { [key]: sanitizeKnownUrlField(child) });
|
|
1270
|
+
}
|
|
1271
|
+
|
|
1272
|
+
sanitizeKnownUrlFields(value[key], seen);
|
|
1273
|
+
});
|
|
1274
|
+
}
|
|
1275
|
+
|
|
1276
|
+
/**
|
|
1277
|
+
* Configuration keys allowed at the root level.
|
|
1278
|
+
*/
|
|
1279
|
+
const CONFIGURATION_KEYS = new Set([
|
|
1280
|
+
'name',
|
|
1281
|
+
'description',
|
|
1282
|
+
'homepage',
|
|
1283
|
+
'license',
|
|
1284
|
+
'version',
|
|
1285
|
+
'sources',
|
|
1286
|
+
'transformations',
|
|
1287
|
+
'exclusions',
|
|
1288
|
+
'exclusions_sources',
|
|
1289
|
+
'inclusions',
|
|
1290
|
+
'inclusions_sources',
|
|
1291
|
+
]);
|
|
1292
|
+
|
|
1293
|
+
/**
|
|
1294
|
+
* Configuration keys allowed in a source object.
|
|
1295
|
+
*/
|
|
1296
|
+
const SOURCE_KEYS = new Set([
|
|
1297
|
+
'name',
|
|
1298
|
+
'source',
|
|
1299
|
+
'type',
|
|
1300
|
+
'transformations',
|
|
1301
|
+
'exclusions',
|
|
1302
|
+
'exclusions_sources',
|
|
1303
|
+
'inclusions',
|
|
1304
|
+
'inclusions_sources',
|
|
1305
|
+
]);
|
|
1306
|
+
|
|
1307
|
+
/**
|
|
1308
|
+
* Sanitizes every string under a value with the strict parser. Unknown
|
|
1309
|
+
* configuration keys are rejected by the schema, so their values are more
|
|
1310
|
+
* likely misplaced URLs than prose.
|
|
1311
|
+
*
|
|
1312
|
+
* @param {*} value - value under an unknown key.
|
|
1313
|
+
* @param {WeakSet<Object>} seen - objects that have already been visited.
|
|
1314
|
+
* @returns {*} sanitized value.
|
|
1315
|
+
*/
|
|
1316
|
+
function sanitizeUnknownValue(value, seen) {
|
|
1317
|
+
if (_.isString(value)) {
|
|
1318
|
+
return stripKnownUrlCredentials(value);
|
|
1319
|
+
}
|
|
1320
|
+
|
|
1321
|
+
if (Array.isArray(value)) {
|
|
1322
|
+
value.forEach((item, index) => {
|
|
1323
|
+
Object.assign(value, { [index]: sanitizeUnknownValue(item, seen) });
|
|
1324
|
+
});
|
|
1325
|
+
return value;
|
|
1326
|
+
}
|
|
1327
|
+
|
|
1328
|
+
if (_.isObject(value) && !seen.has(value)) {
|
|
1329
|
+
seen.add(value);
|
|
1330
|
+
Object.entries(value).forEach(([key, child]) => {
|
|
1331
|
+
Object.assign(value, { [key]: sanitizeUnknownValue(child, seen) });
|
|
1332
|
+
});
|
|
1333
|
+
}
|
|
1334
|
+
|
|
1335
|
+
return value;
|
|
1336
|
+
}
|
|
1337
|
+
|
|
1338
|
+
/**
|
|
1339
|
+
* Applies the strict parser to strings under keys that the configuration
|
|
1340
|
+
* schema does not allow at the current level.
|
|
1341
|
+
*
|
|
1342
|
+
* @param {*} value - configuration value to walk.
|
|
1343
|
+
* @param {Set<String>} allowedKeys - keys allowed at this level.
|
|
1344
|
+
* @param {WeakSet<Object>} seen - objects that have already been visited.
|
|
1345
|
+
*/
|
|
1346
|
+
function sanitizeUnknownKeyValues(value, allowedKeys, seen) {
|
|
1347
|
+
if (Array.isArray(value)) {
|
|
1348
|
+
value.forEach((item) => sanitizeUnknownKeyValues(item, allowedKeys, seen));
|
|
1349
|
+
return;
|
|
1350
|
+
}
|
|
1351
|
+
|
|
1352
|
+
if (!_.isObject(value) || seen.has(value)) {
|
|
1353
|
+
return;
|
|
1354
|
+
}
|
|
1355
|
+
seen.add(value);
|
|
1356
|
+
|
|
1357
|
+
Object.entries(value).forEach(([key, child]) => {
|
|
1358
|
+
if (!allowedKeys.has(key)) {
|
|
1359
|
+
Object.assign(value, { [key]: sanitizeUnknownValue(child, seen) });
|
|
1360
|
+
return;
|
|
1361
|
+
}
|
|
1362
|
+
|
|
1363
|
+
if (key === 'sources') {
|
|
1364
|
+
if (Array.isArray(child)) {
|
|
1365
|
+
child.forEach((item) => sanitizeUnknownKeyValues(item, SOURCE_KEYS, seen));
|
|
1366
|
+
} else if (_.isObject(child)) {
|
|
1367
|
+
sanitizeUnknownKeyValues(child, SOURCE_KEYS, seen);
|
|
1368
|
+
}
|
|
1369
|
+
return;
|
|
1370
|
+
}
|
|
1371
|
+
|
|
1372
|
+
if (Array.isArray(child)) {
|
|
1373
|
+
child.forEach((item) => sanitizeUnknownKeyValues(item, allowedKeys, seen));
|
|
1374
|
+
} else {
|
|
1375
|
+
sanitizeUnknownKeyValues(child, allowedKeys, seen);
|
|
1376
|
+
}
|
|
1377
|
+
});
|
|
1378
|
+
}
|
|
1379
|
+
|
|
1380
|
+
/**
|
|
1381
|
+
* Returns a deep copy of a configuration with credentials removed from every
|
|
1382
|
+
* string, including strings in malformed configurations and class instances.
|
|
1383
|
+
* The copy runs the generic parser on every string, the strict parser on known
|
|
1384
|
+
* URL fields and on values under unknown keys, and rewrites URL-like object
|
|
1385
|
+
* keys. The original configuration is not modified.
|
|
1386
|
+
*
|
|
1387
|
+
* @param {*} configuration - compilation configuration.
|
|
1388
|
+
* @returns {*} sanitized copy of the configuration.
|
|
1389
|
+
*/
|
|
1390
|
+
function stripConfigurationCredentials(configuration) {
|
|
1391
|
+
// Every string goes through the generic parser first: it keeps whitespace
|
|
1392
|
+
// and quotes in userinfo for URL values and falls back to the scanner for
|
|
1393
|
+
// prefixed values and values with more text after the URL.
|
|
1394
|
+
const sanitized = _.cloneDeepWith(configuration, (value) => (
|
|
1395
|
+
_.isString(value) ? stripUrlValueCredentials(value) : undefined
|
|
1396
|
+
));
|
|
1397
|
+
|
|
1398
|
+
// Known URL fields use the strict parser: the downloader treats a quote or
|
|
1399
|
+
// a space in the username as part of the userinfo.
|
|
1400
|
+
sanitizeKnownUrlFields(sanitized, new WeakSet());
|
|
1401
|
+
// The schema rejects unknown keys, so a value under such a key is more
|
|
1402
|
+
// likely a misplaced URL than prose.
|
|
1403
|
+
sanitizeUnknownKeyValues(sanitized, CONFIGURATION_KEYS, new WeakSet());
|
|
1404
|
+
sanitizeConfigurationKeys(sanitized, new WeakSet());
|
|
1405
|
+
|
|
1406
|
+
return sanitized;
|
|
1407
|
+
}
|
|
1408
|
+
|
|
1409
|
+
/**
|
|
1410
|
+
* Returns a JSON-safe copy of a configuration with credentials removed from
|
|
1411
|
+
* every string and object key. Use the copy for the configuration log, the
|
|
1412
|
+
* verbose CLI dump, and validation errors that render the configuration.
|
|
1413
|
+
*
|
|
1414
|
+
* `stripConfigurationCredentials()` works on the object graph, so it cannot see
|
|
1415
|
+
* the strings that a `toJSON()` method or a `URL` object renders, and
|
|
1416
|
+
* `JSON.stringify()` renders object keys without passing them to a replacer.
|
|
1417
|
+
* This function serializes the sanitized copy with a replacer, parses the
|
|
1418
|
+
* result back into plain data, and sanitizes the parsed copy too. The output is
|
|
1419
|
+
* plain JSON data, so a caller can render it without a second scan of the
|
|
1420
|
+
* whole document. A second scan would treat the text between a path-less URL
|
|
1421
|
+
* and a later `@` as userinfo and corrupt the document.
|
|
1422
|
+
*
|
|
1423
|
+
* @param {*} configuration - compilation configuration.
|
|
1424
|
+
* @returns {*} JSON-safe sanitized copy of the configuration.
|
|
1425
|
+
*/
|
|
1426
|
+
function sanitizeConfigurationForLog(configuration) {
|
|
1427
|
+
const sanitized = stripConfigurationCredentials(configuration);
|
|
1428
|
+
// A replacer sees each rendered string after `toJSON()`, so it also covers
|
|
1429
|
+
// `URL` objects and class instances. Object keys do not reach the replacer.
|
|
1430
|
+
const json = JSON.stringify(sanitized, (key, value) => (
|
|
1431
|
+
_.isString(value) ? stripUrlValueCredentials(value) : value
|
|
1432
|
+
));
|
|
1433
|
+
|
|
1434
|
+
if (json === undefined) {
|
|
1435
|
+
return sanitized;
|
|
1436
|
+
}
|
|
1437
|
+
|
|
1438
|
+
// The parsed copy is plain data. Run the sanitizer again to reach keys and
|
|
1439
|
+
// known URL fields that `toJSON()` introduced.
|
|
1440
|
+
return stripConfigurationCredentials(JSON.parse(json));
|
|
1441
|
+
}
|
|
1442
|
+
|
|
172
1443
|
/**
|
|
173
1444
|
* Wildcard is used by the exclusions transformation.
|
|
174
1445
|
*/
|
|
@@ -277,4 +1548,17 @@ module.exports = {
|
|
|
277
1548
|
Wildcard,
|
|
278
1549
|
splitByDelimiterWithEscapeCharacter,
|
|
279
1550
|
substringBetween,
|
|
1551
|
+
stripUrlCredentials,
|
|
1552
|
+
stripUrlValueCredentials,
|
|
1553
|
+
stripKnownUrlCredentials,
|
|
1554
|
+
stripConfigurationCredentials,
|
|
1555
|
+
sanitizeConfigurationForLog,
|
|
1556
|
+
stripErrorCredentials,
|
|
1557
|
+
// Exported so a test can compare them with the `x-credential-url` markers
|
|
1558
|
+
// in the configuration schema. A marked schema property without a matching
|
|
1559
|
+
// entry would silently change which strings use the strict parser.
|
|
1560
|
+
URL_VALUE_FIELDS,
|
|
1561
|
+
URL_LIST_FIELDS,
|
|
1562
|
+
CONFIGURATION_KEYS,
|
|
1563
|
+
SOURCE_KEYS,
|
|
280
1564
|
};
|