@maschinenlesbar.org/ckan-cli 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.js +1 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.d.ts +26 -4
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.js +152 -45
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.d.ts +52 -6
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.js +155 -17
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/http.d.ts +2 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/http.js +33 -6
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.d.ts +8 -4
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.js +21 -4
- package/dist/src/cli/run.d.ts +0 -7
- package/dist/src/cli/run.js +8 -15
- package/package.json +1 -1
|
@@ -208,7 +208,7 @@ export class CkanClient {
|
|
|
208
208
|
// A success:false envelope can come with HTTP 200, so it never passes the
|
|
209
209
|
// engine's error-detail sanitising: strip terminal controls here too.
|
|
210
210
|
const type = env.error?.__type;
|
|
211
|
-
throw new CkanActionError(`${request}: CKAN action "${cutForMessage(name)}" failed: ${cutForMessage(sanitizeServerText(this.#engine.scrub(describeCkanError(env.error))))}`, { action: name, errorType: typeof type === "string" ? cutForMessage(sanitizeServerText(type)) : undefined, url });
|
|
211
|
+
throw new CkanActionError(`${request}: CKAN action "${cutForMessage(name)}" failed: ${cutForMessage(sanitizeServerText(this.#engine.scrub(describeCkanError(env.error))))}`, { action: name, errorType: typeof type === "string" ? cutForMessage(sanitizeServerText(this.#engine.scrub(type))) : undefined, url });
|
|
212
212
|
}
|
|
213
213
|
// `{"success": true}` without a result would print nothing useful (and the CLI
|
|
214
214
|
// would crash rendering `undefined`); CKAN always sends one, `null` included.
|
|
@@ -131,19 +131,25 @@ export declare function isBidiControl(code: number): boolean;
|
|
|
131
131
|
* `detail`, a CKAN `success:false` error, the echoed Content-Type — safe to print
|
|
132
132
|
* into an error message on stderr:
|
|
133
133
|
*
|
|
134
|
-
* -
|
|
134
|
+
* - Whole escape sequences (CSI, OSC and other control strings) are dropped first,
|
|
135
|
+
* then any remaining C0 and C1 controls and DEL. A JSON error body can encode an escape
|
|
135
136
|
* (U+001B) that JSON.parse turns into a real control byte; printed raw, a hostile
|
|
136
137
|
* or MITM'd endpoint could drive ANSI/OSC sequences into the terminal (display
|
|
137
138
|
* spoofing, title changes, OSC 52 clipboard writes).
|
|
138
|
-
* -
|
|
139
|
-
* reorder the visible message
|
|
139
|
+
* - Unicode format characters (General Category Cf) are dropped: the bidi controls
|
|
140
|
+
* (isBidiControl), so server text cannot reorder the visible message, and the
|
|
141
|
+
* invisible ones — zero-width space/joiners, word joiner, soft hyphen, BOM — so
|
|
142
|
+
* two messages that look identical are identical.
|
|
140
143
|
* - Every run of whitespace — newlines, tabs, U+2028/U+2029 included — becomes one
|
|
141
144
|
* space and the ends are trimmed, so the text stays on one line and a server
|
|
142
145
|
* cannot forge an `Error:` line of its own.
|
|
146
|
+
* - Only the first MAX_SERVER_TEXT (16 384) units are read, and every step is a linear
|
|
147
|
+
* regex replace: a hostile error body costs neither seconds nor the heap. A caller that
|
|
148
|
+
* scrubs credentials does so first, so a cut never splits one.
|
|
143
149
|
*
|
|
144
150
|
* The CLI's JSON output is escaped separately (`escapeControlChars` in
|
|
145
151
|
* cli/shared.ts): `JSON.stringify` alone leaves DEL, C1 and bidi characters raw.
|
|
146
|
-
* Written
|
|
152
|
+
* Written with escapes so no raw control byte appears in this source.
|
|
147
153
|
*/
|
|
148
154
|
export declare function sanitizeServerText(text: string): string;
|
|
149
155
|
/**
|
|
@@ -200,6 +206,15 @@ export declare class RequestEngine {
|
|
|
200
206
|
* messages built from server text.
|
|
201
207
|
*/
|
|
202
208
|
scrub(text: string): string;
|
|
209
|
+
/**
|
|
210
|
+
* An error body, already scrubbed as sent, as `CkanApiError.body` keeps it: when it is JSON
|
|
211
|
+
* with an escape in it, checked decoded too. An escape (`\"`, `\u00e4`) hides an echoed
|
|
212
|
+
* credential from the scrub of the text as sent, while a caller who parses the body sees it. Such
|
|
213
|
+
* a body is kept as the decoded JSON written again (`JSON.stringify`, one spelling per character)
|
|
214
|
+
* with the credential scrubbed in that spelling. A body that doesn't parse is text nobody
|
|
215
|
+
* decodes, and stays as sent; one too deeply nested to write again is left out.
|
|
216
|
+
*/
|
|
217
|
+
private scrubBody;
|
|
203
218
|
/**
|
|
204
219
|
* A failure as the `cause` of the error the engine raises: the original when its
|
|
205
220
|
* text carries no credentials, otherwise a copy with them scrubbed (message, `code`
|
|
@@ -223,6 +238,13 @@ export declare class RequestEngine {
|
|
|
223
238
|
* becomes a rejection.
|
|
224
239
|
*/
|
|
225
240
|
private callTransport;
|
|
241
|
+
/**
|
|
242
|
+
* The URL a redirect leads to as messages and errors show it: the Location with the base URL's
|
|
243
|
+
* credentials scrubbed as sent, before resolving percent-encodes what is left of them in a new
|
|
244
|
+
* spelling (`'` as `%27` in a query), then resolved against the URL shown so far, its userinfo
|
|
245
|
+
* left out. A Location the scrub made unparseable (a credential in its host) is shown as scrubbed.
|
|
246
|
+
*/
|
|
247
|
+
private shownTarget;
|
|
226
248
|
/** Perform a request with Accept negotiation and transient-error retries. */
|
|
227
249
|
request(method: string, path: string, options?: {
|
|
228
250
|
query?: QueryParams;
|
|
@@ -5,7 +5,7 @@ import { constants } from "node:buffer";
|
|
|
5
5
|
import { TextDecoder } from "node:util";
|
|
6
6
|
import { MAX_TIMEOUT_MS, nodeHttpTransport, sizeLimitMessage } from "./http.js";
|
|
7
7
|
import { buildQueryString } from "./query.js";
|
|
8
|
-
import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, describeCkanError, echoedCredentialForms, redactCredentials, redactSecrets, redactUrl, } from "./errors.js";
|
|
8
|
+
import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, cutText, describeCkanError, echoedCredentialForms, redactCredentials, redactSecrets, redactUrl, TextBuilder, } from "./errors.js";
|
|
9
9
|
import { assertValid, baseUrlProblem, headerValueProblem, intRangeProblem } from "./validate.js";
|
|
10
10
|
// quoteValue lives with the other message helpers (validate.ts quotes refused values with it).
|
|
11
11
|
export { quoteValue } from "./errors.js";
|
|
@@ -80,35 +80,57 @@ export function isBidiControl(code) {
|
|
|
80
80
|
(code >= 0x202a && code <= 0x202e) ||
|
|
81
81
|
(code >= 0x2066 && code <= 0x2069));
|
|
82
82
|
}
|
|
83
|
+
/**
|
|
84
|
+
* The most server text (in UTF-16 units) `sanitizeServerText` reads; a longer text is cut first.
|
|
85
|
+
* Every message keeps far less of it (MAX_MESSAGE_VALUE_LENGTH, 500 characters), and cleaning all
|
|
86
|
+
* of a 16 MB error detail, one string node per character, exhausted a small heap.
|
|
87
|
+
*/
|
|
88
|
+
const MAX_SERVER_TEXT = 16 * 1024;
|
|
89
|
+
/**
|
|
90
|
+
* A whole terminal escape sequence: a CSI (`ESC [` or the 8-bit U+009B, parameters,
|
|
91
|
+
* intermediates, final byte), a terminated control string (OSC/DCS/SOS/PM/APC, `ESC ]` …
|
|
92
|
+
* up to BEL or ST, or their 8-bit forms) or a short `ESC` sequence. Dropping only the
|
|
93
|
+
* control bytes left the printable rest behind (`ESC ] 0 ; title BEL` became `]0;title`).
|
|
94
|
+
* An unterminated control string is left to the per-character filter, so it cannot
|
|
95
|
+
* swallow the rest of the message. A control string's body ends at an ESC or at another
|
|
96
|
+
* introducer: with the introducers allowed inside it, an unterminated run of them was matched
|
|
97
|
+
* from each one to the end of the text, quadratic (100 000 × U+009D took 16 s). Written with
|
|
98
|
+
* escapes, so the source holds no control bytes.
|
|
99
|
+
*/
|
|
100
|
+
const ESCAPE_SEQUENCE = /(?:\u001b\[|\u009b)[0-?]*[ -/]*[@-~]|(?:\u001b[\]PX^_]|[\u0090\u0098\u009d-\u009f])[^\u0007\u001b\u0090\u0098\u009c-\u009f]*(?:\u0007|\u001b\\|\u009c)|\u001b[ -/]*[0-~]/g;
|
|
101
|
+
/**
|
|
102
|
+
* What `sanitizeServerText` drops once the escape sequences are gone: the C0 controls but the
|
|
103
|
+
* whitespace ones (TAB to CR), DEL, the C1 controls, and every format character (General
|
|
104
|
+
* Category Cf: bidi controls, zero-width characters, U+00AD, U+FEFF, …).
|
|
105
|
+
*/
|
|
106
|
+
const DROPPED_CHAR = /[\u0000-\u0008\u000e-\u001f\u007f-\u009f\p{Cf}]/gu;
|
|
83
107
|
/**
|
|
84
108
|
* Make a string that originates in an attacker-controlled response — the error
|
|
85
109
|
* `detail`, a CKAN `success:false` error, the echoed Content-Type — safe to print
|
|
86
110
|
* into an error message on stderr:
|
|
87
111
|
*
|
|
88
|
-
* -
|
|
112
|
+
* - Whole escape sequences (CSI, OSC and other control strings) are dropped first,
|
|
113
|
+
* then any remaining C0 and C1 controls and DEL. A JSON error body can encode an escape
|
|
89
114
|
* (U+001B) that JSON.parse turns into a real control byte; printed raw, a hostile
|
|
90
115
|
* or MITM'd endpoint could drive ANSI/OSC sequences into the terminal (display
|
|
91
116
|
* spoofing, title changes, OSC 52 clipboard writes).
|
|
92
|
-
* -
|
|
93
|
-
* reorder the visible message
|
|
117
|
+
* - Unicode format characters (General Category Cf) are dropped: the bidi controls
|
|
118
|
+
* (isBidiControl), so server text cannot reorder the visible message, and the
|
|
119
|
+
* invisible ones — zero-width space/joiners, word joiner, soft hyphen, BOM — so
|
|
120
|
+
* two messages that look identical are identical.
|
|
94
121
|
* - Every run of whitespace — newlines, tabs, U+2028/U+2029 included — becomes one
|
|
95
122
|
* space and the ends are trimmed, so the text stays on one line and a server
|
|
96
123
|
* cannot forge an `Error:` line of its own.
|
|
124
|
+
* - Only the first MAX_SERVER_TEXT (16 384) units are read, and every step is a linear
|
|
125
|
+
* regex replace: a hostile error body costs neither seconds nor the heap. A caller that
|
|
126
|
+
* scrubs credentials does so first, so a cut never splits one.
|
|
97
127
|
*
|
|
98
128
|
* The CLI's JSON output is escaped separately (`escapeControlChars` in
|
|
99
129
|
* cli/shared.ts): `JSON.stringify` alone leaves DEL, C1 and bidi characters raw.
|
|
100
|
-
* Written
|
|
130
|
+
* Written with escapes so no raw control byte appears in this source.
|
|
101
131
|
*/
|
|
102
132
|
export function sanitizeServerText(text) {
|
|
103
|
-
|
|
104
|
-
for (const ch of text) {
|
|
105
|
-
const n = ch.codePointAt(0) ?? 0;
|
|
106
|
-
const whitespaceControl = n >= 0x09 && n <= 0x0d;
|
|
107
|
-
if (!whitespaceControl && (n <= 0x1f || (n >= 0x7f && n <= 0x9f) || isBidiControl(n)))
|
|
108
|
-
continue;
|
|
109
|
-
out += ch;
|
|
110
|
-
}
|
|
111
|
-
return out.replace(/\s+/g, " ").trim();
|
|
133
|
+
return cutText(text, MAX_SERVER_TEXT).replace(ESCAPE_SEQUENCE, "").replace(DROPPED_CHAR, "").replace(/\s+/g, " ").trim();
|
|
112
134
|
}
|
|
113
135
|
/**
|
|
114
136
|
* Check a base URL (baseUrlProblem: not blank, no whitespace or control
|
|
@@ -246,6 +268,8 @@ export class RequestEngine {
|
|
|
246
268
|
// JSON.stringify of a client never show it, so a password in the base URL can't be
|
|
247
269
|
// logged by accident. Messages use redactUrl.
|
|
248
270
|
#baseUrl;
|
|
271
|
+
/** `#baseUrl` without its userinfo: what every request URL starts with. */
|
|
272
|
+
#requestBase;
|
|
249
273
|
/** The base URL's userinfo, raw and percent-decoded, for scrubbing server and transport text. */
|
|
250
274
|
#credentials;
|
|
251
275
|
/**
|
|
@@ -272,8 +296,18 @@ export class RequestEngine {
|
|
|
272
296
|
// consumer that injects a custom transport would otherwise get no gating at
|
|
273
297
|
// all, and could be steered to a non-http(s) scheme. Only undefined selects
|
|
274
298
|
// the default.
|
|
275
|
-
|
|
276
|
-
|
|
299
|
+
const baseUrl = validateBaseUrl(options.baseUrl ?? DEFAULT_BASE_URL);
|
|
300
|
+
// Requests go to the base URL as WHATWG parses it, the reading every check of it makes (the
|
|
301
|
+
// host, the cleartext warning, the Authorization header), never to a text-based reading of
|
|
302
|
+
// the string: a transport that reads the URL as text reaches the same host.
|
|
303
|
+
const parsed = new URL(baseUrl);
|
|
304
|
+
this.#baseUrl = parsed.href.replace(/\/+$/, "");
|
|
305
|
+
parsed.username = "";
|
|
306
|
+
parsed.password = "";
|
|
307
|
+
this.#requestBase = parsed.href.replace(/\/+$/, "");
|
|
308
|
+
// The userinfo as typed and as sent (WHATWG percent-encodes some of its characters).
|
|
309
|
+
const userinfos = [...new Set([...credentialsIn(baseUrl), ...credentialsIn(this.#baseUrl)])];
|
|
310
|
+
this.#credentials = userinfos.flatMap((raw) => {
|
|
277
311
|
try {
|
|
278
312
|
return [raw, decodeURIComponent(raw)];
|
|
279
313
|
}
|
|
@@ -281,8 +315,7 @@ export class RequestEngine {
|
|
|
281
315
|
return [raw];
|
|
282
316
|
}
|
|
283
317
|
});
|
|
284
|
-
this.#echoed =
|
|
285
|
-
.flatMap(echoedCredentialForms)
|
|
318
|
+
this.#echoed = [...new Set(userinfos.flatMap(echoedCredentialForms).flatMap(urlSpellings))]
|
|
286
319
|
.sort((a, b) => b.length - a.length);
|
|
287
320
|
this.transport = functionOption("transport", options.transport, nodeHttpTransport);
|
|
288
321
|
// Only undefined selects the default; a blank or unsendable value is refused.
|
|
@@ -338,6 +371,34 @@ export class RequestEngine {
|
|
|
338
371
|
scrub(text) {
|
|
339
372
|
return this.#credentials.length === 0 ? text : redactSecrets(redactCredentials(text, this.#credentials), this.#echoed);
|
|
340
373
|
}
|
|
374
|
+
/**
|
|
375
|
+
* An error body, already scrubbed as sent, as `CkanApiError.body` keeps it: when it is JSON
|
|
376
|
+
* with an escape in it, checked decoded too. An escape (`\"`, `\u00e4`) hides an echoed
|
|
377
|
+
* credential from the scrub of the text as sent, while a caller who parses the body sees it. Such
|
|
378
|
+
* a body is kept as the decoded JSON written again (`JSON.stringify`, one spelling per character)
|
|
379
|
+
* with the credential scrubbed in that spelling. A body that doesn't parse is text nobody
|
|
380
|
+
* decodes, and stays as sent; one too deeply nested to write again is left out.
|
|
381
|
+
*/
|
|
382
|
+
scrubBody(scrubbed) {
|
|
383
|
+
if (this.#echoed.length === 0 || !scrubbed.includes("\\"))
|
|
384
|
+
return scrubbed;
|
|
385
|
+
let decoded;
|
|
386
|
+
try {
|
|
387
|
+
decoded = JSON.parse(scrubbed);
|
|
388
|
+
}
|
|
389
|
+
catch {
|
|
390
|
+
return scrubbed;
|
|
391
|
+
}
|
|
392
|
+
let rewritten;
|
|
393
|
+
try {
|
|
394
|
+
rewritten = JSON.stringify(decoded);
|
|
395
|
+
}
|
|
396
|
+
catch {
|
|
397
|
+
return "";
|
|
398
|
+
}
|
|
399
|
+
const spelled = this.#echoed.map((form) => JSON.stringify(form).slice(1, -1));
|
|
400
|
+
return spelled.some((form) => rewritten.includes(form)) ? redactSecrets(rewritten, spelled) : scrubbed;
|
|
401
|
+
}
|
|
341
402
|
/**
|
|
342
403
|
* A failure as the `cause` of the error the engine raises: the original when its
|
|
343
404
|
* text carries no credentials, otherwise a copy with them scrubbed (message, `code`
|
|
@@ -375,7 +436,7 @@ export class RequestEngine {
|
|
|
375
436
|
composeUrl(path, query, withUserinfo) {
|
|
376
437
|
const normalizedPath = path.startsWith("/") ? path : `/${path}`;
|
|
377
438
|
const qs = query ? buildQueryString(query) : "";
|
|
378
|
-
const base = withUserinfo ? this.#baseUrl :
|
|
439
|
+
const base = withUserinfo ? this.#baseUrl : this.#requestBase;
|
|
379
440
|
return `${base}${normalizedPath}${qs ? `?${qs}` : ""}`;
|
|
380
441
|
}
|
|
381
442
|
/**
|
|
@@ -405,6 +466,24 @@ export class RequestEngine {
|
|
|
405
466
|
clearTimeout(timer);
|
|
406
467
|
}
|
|
407
468
|
}
|
|
469
|
+
/**
|
|
470
|
+
* The URL a redirect leads to as messages and errors show it: the Location with the base URL's
|
|
471
|
+
* credentials scrubbed as sent, before resolving percent-encodes what is left of them in a new
|
|
472
|
+
* spelling (`'` as `%27` in a query), then resolved against the URL shown so far, its userinfo
|
|
473
|
+
* left out. A Location the scrub made unparseable (a credential in its host) is shown as scrubbed.
|
|
474
|
+
*/
|
|
475
|
+
shownTarget(location, shown) {
|
|
476
|
+
const sent = this.scrub(location);
|
|
477
|
+
try {
|
|
478
|
+
const target = new URL(sent, shown);
|
|
479
|
+
target.username = "";
|
|
480
|
+
target.password = "";
|
|
481
|
+
return target.href;
|
|
482
|
+
}
|
|
483
|
+
catch {
|
|
484
|
+
return sent;
|
|
485
|
+
}
|
|
486
|
+
}
|
|
408
487
|
/** Perform a request with Accept negotiation and transient-error retries. */
|
|
409
488
|
async request(method, path, options = { accept: "application/json" }) {
|
|
410
489
|
// The transport never sees the base URL's userinfo: the engine sends it as an
|
|
@@ -412,6 +491,9 @@ export class RequestEngine {
|
|
|
412
491
|
// absolute) keeps it and one to another origin or scheme drops it. A transport
|
|
413
492
|
// such as fetch also refuses a URL with credentials outright.
|
|
414
493
|
let url = this.composeUrl(path, options.query, false);
|
|
494
|
+
// The URL as messages, errors and `RawResponse.url` show it: after a redirect, the Location
|
|
495
|
+
// with the base URL's credentials scrubbed (shownTarget); `url` is what is requested.
|
|
496
|
+
let shown = url;
|
|
415
497
|
let headers = {
|
|
416
498
|
Accept: options.accept,
|
|
417
499
|
"User-Agent": this.userAgent,
|
|
@@ -445,7 +527,7 @@ export class RequestEngine {
|
|
|
445
527
|
if (cause instanceof CkanError && !(cause instanceof CkanNetworkError))
|
|
446
528
|
throw cause;
|
|
447
529
|
const reason = cause instanceof Error ? cause.message : String(cause);
|
|
448
|
-
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(
|
|
530
|
+
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(shown))} failed: ${sanitizeServerText(this.scrub(reason))}`, {
|
|
449
531
|
cause: this.scrubCause(cause),
|
|
450
532
|
});
|
|
451
533
|
}
|
|
@@ -453,14 +535,14 @@ export class RequestEngine {
|
|
|
453
535
|
// would otherwise surface below as a raw TypeError, outside the CkanError contract.
|
|
454
536
|
const invalid = responseProblem(response);
|
|
455
537
|
if (invalid !== undefined) {
|
|
456
|
-
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(
|
|
538
|
+
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(shown))} failed: the transport returned an invalid response (${invalid}).`);
|
|
457
539
|
}
|
|
458
540
|
// A transport must not follow redirects itself (`redirect: "manual"`): one that
|
|
459
541
|
// did (fetch's default) may have carried the Authorization header to another
|
|
460
542
|
// host, and the answer is not the one asked for. Reject it when it says so (`url`).
|
|
461
543
|
const finalUrl = response.url;
|
|
462
544
|
if (typeof finalUrl === "string" && finalUrl !== "" && originOf(finalUrl) !== originOf(url)) {
|
|
463
|
-
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(
|
|
545
|
+
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(shown))} failed: the transport followed a redirect to another origin ` +
|
|
464
546
|
`(${cutForMessage(sanitizeServerText(redactUrl(this.scrub(finalUrl))))}); a transport must not follow redirects ` +
|
|
465
547
|
`(HttpRequest.redirect is "manual").`);
|
|
466
548
|
}
|
|
@@ -471,7 +553,7 @@ export class RequestEngine {
|
|
|
471
553
|
// The size cap holds whatever the transport did: the default one aborts early, a
|
|
472
554
|
// custom one may have read everything.
|
|
473
555
|
if (this.maxResponseBytes > 0 && body.byteLength > this.maxResponseBytes) {
|
|
474
|
-
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(
|
|
556
|
+
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(shown))} failed: ${sizeLimitMessage(this.maxResponseBytes)}`);
|
|
475
557
|
}
|
|
476
558
|
const retryable = status === 429 || status === 503;
|
|
477
559
|
// A Retry-After beyond MAX_RETRY_AFTER_MS is not retried: the error below
|
|
@@ -484,7 +566,7 @@ export class RequestEngine {
|
|
|
484
566
|
// for less: `Retry-After: 0` or a date in the past turned the retries into a
|
|
485
567
|
// zero-delay burst against a server that had just asked for less load.
|
|
486
568
|
const backoff = this.retryDelayMs * attempt;
|
|
487
|
-
await this.backOff(attempt, retryAfter === undefined ? backoff : Math.max(retryAfter, backoff),
|
|
569
|
+
await this.backOff(attempt, retryAfter === undefined ? backoff : Math.max(retryAfter, backoff), shown, status);
|
|
488
570
|
continue;
|
|
489
571
|
}
|
|
490
572
|
// Follow redirects, resolving the Location relative to the current URL.
|
|
@@ -500,7 +582,7 @@ export class RequestEngine {
|
|
|
500
582
|
// never reaches a transport (a custom one may not check the scheme).
|
|
501
583
|
if (nextUrl.protocol !== "http:" && nextUrl.protocol !== "https:") {
|
|
502
584
|
throw new CkanNetworkError(`Refusing to follow redirect to unsupported protocol "${cutForMessage(sanitizeServerText(nextUrl.protocol))}" ` +
|
|
503
|
-
`for ${method} ${cutForMessage(redactUrl(
|
|
585
|
+
`for ${method} ${cutForMessage(redactUrl(shown))}`);
|
|
504
586
|
}
|
|
505
587
|
// Credential-strip guard: if the redirect crosses origin (scheme + host +
|
|
506
588
|
// port, so an https->http downgrade counts), keep only the engine's own
|
|
@@ -521,6 +603,7 @@ export class RequestEngine {
|
|
|
521
603
|
headers = engineHeadersOnly(headers);
|
|
522
604
|
}
|
|
523
605
|
url = nextUrl.toString();
|
|
606
|
+
shown = this.shownTarget(location, shown);
|
|
524
607
|
redirects += 1;
|
|
525
608
|
continue;
|
|
526
609
|
}
|
|
@@ -535,16 +618,16 @@ export class RequestEngine {
|
|
|
535
618
|
const detail = target !== undefined
|
|
536
619
|
? `stopped after ${this.maxRedirects} redirects (a redirect loop?)`
|
|
537
620
|
: location
|
|
538
|
-
? `redirect to ${cutForMessage(sanitizeServerText(location))} not followed`
|
|
621
|
+
? `redirect to ${cutForMessage(sanitizeServerText(this.scrub(location)))} not followed`
|
|
539
622
|
: "redirect not followed (no Location header)";
|
|
540
|
-
throw new CkanApiError({ status, url, method, body: this.scrub(errorBodyText(body)), detail });
|
|
623
|
+
throw new CkanApiError({ status, url: shown, method, body: this.scrubBody(this.scrub(errorBodyText(body))), detail });
|
|
541
624
|
}
|
|
542
|
-
throw this.toApiError(method,
|
|
625
|
+
throw this.toApiError(method, shown, status, body, status === 401 || status === 403 ? dropped : undefined, {
|
|
543
626
|
retries: attempt,
|
|
544
627
|
...(tooLong ? { retryAfterMs: retryAfter } : {}),
|
|
545
628
|
});
|
|
546
629
|
}
|
|
547
|
-
return { data: body, contentType, status, url };
|
|
630
|
+
return { data: body, contentType, status, url: shown };
|
|
548
631
|
}
|
|
549
632
|
}
|
|
550
633
|
/** Perform a GET expecting JSON and parse it into `T` (see {@link getAnswer}). */
|
|
@@ -627,28 +710,24 @@ export class RequestEngine {
|
|
|
627
710
|
catch {
|
|
628
711
|
// Non-JSON error body; leave detail undefined.
|
|
629
712
|
}
|
|
630
|
-
// `detail` came from the response body
|
|
713
|
+
// `detail` came from the response body, decoded: scrubbed again (a JSON escape hid the
|
|
714
|
+
// credential from the scrub above), then stripped of control characters so a hostile
|
|
631
715
|
// endpoint cannot inject terminal escape sequences via the stderr error message.
|
|
632
716
|
if (detail !== undefined)
|
|
633
|
-
detail = sanitizeServerText(detail);
|
|
717
|
+
detail = sanitizeServerText(this.scrub(detail));
|
|
634
718
|
if (hint !== undefined)
|
|
635
719
|
detail = detail === undefined ? hint : `${detail}; ${hint}`;
|
|
636
720
|
return new CkanApiError({
|
|
637
721
|
status,
|
|
638
722
|
url,
|
|
639
723
|
method,
|
|
640
|
-
body: text,
|
|
724
|
+
body: this.scrubBody(text),
|
|
641
725
|
detail,
|
|
642
726
|
retries: retry.retries,
|
|
643
727
|
...(retry.retryAfterMs === undefined ? {} : { retryAfterMs: retry.retryAfterMs, maxRetryAfterMs: MAX_RETRY_AFTER_MS }),
|
|
644
728
|
});
|
|
645
729
|
}
|
|
646
730
|
}
|
|
647
|
-
/** `url` without its userinfo (`https://user:pw@host/x` → `https://host/x`), as written otherwise. */
|
|
648
|
-
function withoutUserinfo(url) {
|
|
649
|
-
const [userinfo] = credentialsIn(url);
|
|
650
|
-
return userinfo === undefined ? url : url.replace(`://${userinfo}@`, "://");
|
|
651
|
-
}
|
|
652
731
|
/**
|
|
653
732
|
* The `Authorization` header for a URL's userinfo (`Basic base64(user:password)`,
|
|
654
733
|
* both percent-decoded, as Node's own http client builds it), or undefined without
|
|
@@ -770,7 +849,7 @@ function escapeAt(json, at) {
|
|
|
770
849
|
function wellFormedEscapes(json) {
|
|
771
850
|
if (!SURROGATE_ESCAPE.test(json))
|
|
772
851
|
return { json, replaced: 0 };
|
|
773
|
-
|
|
852
|
+
const out = new TextBuilder();
|
|
774
853
|
let copied = 0;
|
|
775
854
|
let replaced = 0;
|
|
776
855
|
for (let i = 0; i < json.length; i++) {
|
|
@@ -786,16 +865,32 @@ function wellFormedEscapes(json) {
|
|
|
786
865
|
i += 11;
|
|
787
866
|
continue;
|
|
788
867
|
}
|
|
789
|
-
out
|
|
868
|
+
out.push(json.slice(copied, i), "\\ufffd");
|
|
790
869
|
copied = i + 6;
|
|
791
870
|
replaced++;
|
|
792
871
|
i += 5;
|
|
793
872
|
}
|
|
794
|
-
|
|
873
|
+
if (replaced === 0)
|
|
874
|
+
return { json, replaced };
|
|
875
|
+
out.push(json.slice(copied));
|
|
876
|
+
return { json: out.toString(), replaced };
|
|
795
877
|
}
|
|
796
878
|
function isLowSurrogate(code) {
|
|
797
879
|
return code >= 0xdc00 && code <= 0xdfff;
|
|
798
880
|
}
|
|
881
|
+
/**
|
|
882
|
+
* `form` and its spelling in a URL (`encodeURIComponent`): a server that puts a credential into
|
|
883
|
+
* a URL, a redirect's Location, percent-encodes it. A form that has no such spelling (a lone
|
|
884
|
+
* surrogate) is kept as it is.
|
|
885
|
+
*/
|
|
886
|
+
function urlSpellings(form) {
|
|
887
|
+
try {
|
|
888
|
+
return [form, encodeURIComponent(form)];
|
|
889
|
+
}
|
|
890
|
+
catch {
|
|
891
|
+
return [form];
|
|
892
|
+
}
|
|
893
|
+
}
|
|
799
894
|
/** Resolve a Location header against the current URL; undefined if missing or malformed. */
|
|
800
895
|
function resolveLocation(location, base) {
|
|
801
896
|
if (location === undefined || location === "")
|
|
@@ -833,9 +928,17 @@ function exactAtTheEdges(literal, value) {
|
|
|
833
928
|
/** A decimal literal as sign, significant digits (no leading or trailing zeros) and exponent: digits × 10^exponent. */
|
|
834
929
|
function decimalOf(text) {
|
|
835
930
|
const [, sign = "", whole = "", fraction = "", exp = "0"] = /^(-?)(\d+)(?:\.(\d+))?(?:[eE]([+-]?\d+))?$/.exec(text) ?? [];
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
931
|
+
// The zeros are counted with index loops: `/0+$/` tried a match at every zero of a run that
|
|
932
|
+
// doesn't end the text, quadratic in the run, and one `1.<64 000 zeros>1e300` in an answer held
|
|
933
|
+
// the CLI for seconds, past --timeout (the scan runs once the answer is in).
|
|
934
|
+
const all = whole + fraction;
|
|
935
|
+
let start = 0;
|
|
936
|
+
while (start < all.length && all.charCodeAt(start) === 0x30)
|
|
937
|
+
start++;
|
|
938
|
+
let end = all.length;
|
|
939
|
+
while (end > start && all.charCodeAt(end - 1) === 0x30)
|
|
940
|
+
end--;
|
|
941
|
+
return { negative: sign === "-", digits: all.slice(start, end), exponent: Number(exp) - fraction.length + (all.length - end) };
|
|
839
942
|
}
|
|
840
943
|
/** A JSON number token as the grammar allows it, matched at a given position (sticky). */
|
|
841
944
|
const STRICT_JSON_NUMBER = /-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/y;
|
|
@@ -902,7 +1005,8 @@ export function scanNumbers(json, bigIntegerKeys = NO_KEYS) {
|
|
|
902
1005
|
const literal = STRICT_JSON_NUMBER.exec(json)?.[0];
|
|
903
1006
|
if (literal === undefined)
|
|
904
1007
|
continue;
|
|
905
|
-
out
|
|
1008
|
+
out ??= new TextBuilder();
|
|
1009
|
+
out.push(json.slice(copied, at), '"', literal, '"');
|
|
906
1010
|
copied = at + literal.length;
|
|
907
1011
|
i = copied - 1;
|
|
908
1012
|
}
|
|
@@ -920,7 +1024,10 @@ export function scanNumbers(json, bigIntegerKeys = NO_KEYS) {
|
|
|
920
1024
|
i += literal.length - 1;
|
|
921
1025
|
}
|
|
922
1026
|
}
|
|
923
|
-
|
|
1027
|
+
if (out === undefined)
|
|
1028
|
+
return { json, inexact };
|
|
1029
|
+
out.push(json.slice(copied));
|
|
1030
|
+
return { json: out.toString(), inexact };
|
|
924
1031
|
}
|
|
925
1032
|
/** A member name written with an escape, decoded; a name JSON.parse can't read matches nothing. */
|
|
926
1033
|
function decodedName(quoted) {
|
|
@@ -9,20 +9,60 @@ export declare function redactUrl(url: string): string;
|
|
|
9
9
|
/**
|
|
10
10
|
* The userinfo a URL carries, exactly as written — `["alice:pa#ss"]` for
|
|
11
11
|
* `https://alice:pa#ss@host` — or `[]` when it carries none. Only a value that starts
|
|
12
|
-
* with a scheme (`^[A-Za-z][A-Za-z0-9+.-]
|
|
12
|
+
* with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`, or a special one such as `http:` with any
|
|
13
|
+
* slashes or backslashes after it, as WHATWG reads it) counts: a bare `a:b@c` is a dataset id, a
|
|
13
14
|
* search text or a User-Agent as often as a credential, and the base URL always has a
|
|
14
15
|
* scheme. It works on URLs that don't parse too: the userinfo is everything between
|
|
15
16
|
* `://` and the last `@` before the host. Used to redact those exact strings from text
|
|
16
17
|
* that echoes the value (usage errors, help), whatever characters the password contains.
|
|
17
18
|
*/
|
|
18
19
|
export declare function credentialsIn(value: string): string[];
|
|
20
|
+
/**
|
|
21
|
+
* The userinfo of a value meant as a URL (a base URL, a request URL), however its scheme is
|
|
22
|
+
* typed: as `credentialsIn` reads it when the value has a scheme; else as if it had `http://`
|
|
23
|
+
* (`user:password@host`, `TOKEN:@host`); else, for a scheme mistyped before slashes
|
|
24
|
+
* (`http//user:pw@host` with the colon left out, `http/…`, `htp:/…`), everything between the
|
|
25
|
+
* first run of slashes or backslashes and the last `@`, as `credentialsIn` reads a URL that
|
|
26
|
+
* doesn't parse. Such a value is refused, but the refusal (commander's, the transport's) quotes
|
|
27
|
+
* it, so its userinfo has to be found to be redacted. Only for a value known to be a URL: in
|
|
28
|
+
* other text a bare `a/b@c` is no credential.
|
|
29
|
+
*/
|
|
30
|
+
export declare function typedUrlCredentials(value: string): string[];
|
|
31
|
+
/**
|
|
32
|
+
* `text` with the userinfo of every URL in it replaced by `***` (`https://user:secret@host`
|
|
33
|
+
* becomes `https://***@host`): the pattern backstop for text no exact secret matches. Text-based,
|
|
34
|
+
* so it also covers a URL that does not parse; the userinfo ends at the last `@` before the first
|
|
35
|
+
* whitespace, `/`, `?`, `#` or `'` after `://`. Linear in `text`, however many URLs it holds: each
|
|
36
|
+
* `://` is found with `indexOf`, its scheme by walking back no further than the previous `://`,
|
|
37
|
+
* and the search goes on right after each `://`, not after the authority it read: a URL glued
|
|
38
|
+
* onto the one before it (`https://host://u:p@h`, `https://u:p@x://v:q@w`) starts inside that
|
|
39
|
+
* authority, which ends at its `/`. An authority holds no `/`, so no `://` lies wholly inside one,
|
|
40
|
+
* and every character is read a bounded number of times: by the authority loop and `indexOf`
|
|
41
|
+
* forwards, by `schemeStart` backwards. The regex it replaces (a scheme tried at every
|
|
42
|
+
* word boundary) was quadratic: an 80 KB answer of `a.a.…` took about 5 s to print.
|
|
43
|
+
*/
|
|
44
|
+
export declare function redactUrlUserinfo(text: string): string;
|
|
45
|
+
/**
|
|
46
|
+
* A text put together from many pieces (slices of a server answer and their replacements),
|
|
47
|
+
* joined in batches of BUILDER_BATCH into flat strings, so memory stays proportional to the text,
|
|
48
|
+
* not to the number of pieces. One `+=` per piece made a rope node each, and one array of every
|
|
49
|
+
* piece a slot each: millions of lone-surrogate repairs in a large answer exhausted a small heap
|
|
50
|
+
* either way.
|
|
51
|
+
*/
|
|
52
|
+
export declare class TextBuilder {
|
|
53
|
+
#private;
|
|
54
|
+
push(...pieces: string[]): void;
|
|
55
|
+
toString(): string;
|
|
56
|
+
}
|
|
19
57
|
/**
|
|
20
58
|
* The forms in which a server may echo the credentials of a userinfo (`user:password`,
|
|
21
|
-
* as `credentialsIn` returns it) back in an error body
|
|
22
|
-
* (base64 of the decoded `user:password`, UTF-8 as the engine
|
|
23
|
-
* `user:password` itself, and the
|
|
24
|
-
*
|
|
25
|
-
*
|
|
59
|
+
* as `credentialsIn` returns it) back in an error body, in this order: the
|
|
60
|
+
* `Authorization: Basic` value (base64 of the decoded `user:password`, UTF-8 as the engine
|
|
61
|
+
* sends it), the decoded `user:password` itself, and the secret alone when it is at least 4
|
|
62
|
+
* characters long. The secret is the password; in a userinfo without one (`TOKEN@host`,
|
|
63
|
+
* `TOKEN:@host`) it is the user name, a token, which the engine sends as Basic `TOKEN:`. `[]`
|
|
64
|
+
* for an empty userinfo. None of them has an `@` to anchor on, so they are replaced as exact
|
|
65
|
+
* strings (`redactSecrets`).
|
|
26
66
|
*/
|
|
27
67
|
export declare function echoedCredentialForms(userinfo: string): string[];
|
|
28
68
|
/** `text` with every occurrence of each of `secrets` (exact strings, no anchor) replaced by `***`. */
|
|
@@ -83,6 +123,12 @@ export declare class CkanApiError extends CkanError {
|
|
|
83
123
|
readonly detail: string | undefined;
|
|
84
124
|
readonly url: string;
|
|
85
125
|
readonly method: string;
|
|
126
|
+
/**
|
|
127
|
+
* The answer's body as text, the base URL's credentials scrubbed. As sent, unless it is JSON
|
|
128
|
+
* whose decoded text still holds an echoed credential behind an escape: then the decoded JSON
|
|
129
|
+
* written again (other escapes, no insignificant whitespace) with it replaced by `***`, and `""`
|
|
130
|
+
* for such a body nested too deeply to write again.
|
|
131
|
+
*/
|
|
86
132
|
readonly body: string;
|
|
87
133
|
/** How many times the engine retried the request before giving up (0 when it did not). */
|
|
88
134
|
readonly retries: number;
|
|
@@ -27,14 +27,17 @@ export function redactUrl(url) {
|
|
|
27
27
|
/**
|
|
28
28
|
* The userinfo a URL carries, exactly as written — `["alice:pa#ss"]` for
|
|
29
29
|
* `https://alice:pa#ss@host` — or `[]` when it carries none. Only a value that starts
|
|
30
|
-
* with a scheme (`^[A-Za-z][A-Za-z0-9+.-]
|
|
30
|
+
* with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`, or a special one such as `http:` with any
|
|
31
|
+
* slashes or backslashes after it, as WHATWG reads it) counts: a bare `a:b@c` is a dataset id, a
|
|
31
32
|
* search text or a User-Agent as often as a credential, and the base URL always has a
|
|
32
33
|
* scheme. It works on URLs that don't parse too: the userinfo is everything between
|
|
33
34
|
* `://` and the last `@` before the host. Used to redact those exact strings from text
|
|
34
35
|
* that echoes the value (usage errors, help), whatever characters the password contains.
|
|
35
36
|
*/
|
|
36
37
|
export function credentialsIn(value) {
|
|
37
|
-
|
|
38
|
+
// WHATWG skips any number of "/" and "\" after a special scheme (`http:///u:pw@h`,
|
|
39
|
+
// `https:\\u:pw@h` and `http:u:pw@h` all send `u:pw`), so its userinfo starts after them.
|
|
40
|
+
const scheme = SPECIAL_SCHEME_START.exec(value) ?? /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.exec(value);
|
|
38
41
|
if (scheme === null)
|
|
39
42
|
return [];
|
|
40
43
|
const rest = value.slice(scheme[0].length);
|
|
@@ -47,23 +50,138 @@ export function credentialsIn(value) {
|
|
|
47
50
|
// Doesn't parse: the password may hold "/", "?", "#" or spaces.
|
|
48
51
|
}
|
|
49
52
|
// In a URL that parses, the userinfo ends at the last "@" of the authority (before the
|
|
50
|
-
// first "/", "?" or "#"
|
|
51
|
-
|
|
53
|
+
// first "/", "?" or "#", and for a special scheme such as http(s) a "\", where WHATWG ends
|
|
54
|
+
// it too: `https://u:pw@a.example\@b.example` sends `u:pw` to a.example); in one that doesn't,
|
|
55
|
+
// at the last "@" of the value.
|
|
56
|
+
const stop = SPECIAL_SCHEME.test(scheme[0]) ? /[/?#\\]|$/ : /[/?#]|$/;
|
|
57
|
+
const authority = parses ? rest.slice(0, rest.search(stop)) : rest;
|
|
52
58
|
const end = authority.lastIndexOf("@");
|
|
53
59
|
return end > 0 ? [rest.slice(0, end)] : [];
|
|
54
60
|
}
|
|
61
|
+
/**
|
|
62
|
+
* The userinfo of a value meant as a URL (a base URL, a request URL), however its scheme is
|
|
63
|
+
* typed: as `credentialsIn` reads it when the value has a scheme; else as if it had `http://`
|
|
64
|
+
* (`user:password@host`, `TOKEN:@host`); else, for a scheme mistyped before slashes
|
|
65
|
+
* (`http//user:pw@host` with the colon left out, `http/…`, `htp:/…`), everything between the
|
|
66
|
+
* first run of slashes or backslashes and the last `@`, as `credentialsIn` reads a URL that
|
|
67
|
+
* doesn't parse. Such a value is refused, but the refusal (commander's, the transport's) quotes
|
|
68
|
+
* it, so its userinfo has to be found to be redacted. Only for a value known to be a URL: in
|
|
69
|
+
* other text a bare `a/b@c` is no credential.
|
|
70
|
+
*/
|
|
71
|
+
export function typedUrlCredentials(value) {
|
|
72
|
+
if (value === "" || credentialsIn(value).length > 0 || /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(value))
|
|
73
|
+
return credentialsIn(value);
|
|
74
|
+
const prefixed = credentialsIn(`http://${value}`);
|
|
75
|
+
if (prefixed.length > 0)
|
|
76
|
+
return prefixed;
|
|
77
|
+
const slashes = /[/\\]+/.exec(value);
|
|
78
|
+
const at = value.lastIndexOf("@");
|
|
79
|
+
if (slashes === null || slashes.index + slashes[0].length >= at)
|
|
80
|
+
return [];
|
|
81
|
+
return [value.slice(slashes.index + slashes[0].length, at)];
|
|
82
|
+
}
|
|
83
|
+
/** The schemes WHATWG parses as special (`ftp`, `file`, `http(s)`, `ws(s)`): their authority also ends at a backslash. */
|
|
84
|
+
const SPECIAL_SCHEME = /^(?:ftp|file|https?|wss?):[/\\]*$/i;
|
|
85
|
+
/** A special scheme at the start of a value, with the slashes and backslashes WHATWG skips after it. */
|
|
86
|
+
const SPECIAL_SCHEME_START = /^(?:ftp|file|https?|wss?):[/\\]*/i;
|
|
87
|
+
/** A scheme character (RFC 3986: letters, digits, `+`, `.`, `-`). */
|
|
88
|
+
function isSchemeChar(code) {
|
|
89
|
+
return (code >= 0x61 && code <= 0x7a) || (code >= 0x41 && code <= 0x5a) || (code >= 0x30 && code <= 0x39) || code === 0x2b || code === 0x2e || code === 0x2d;
|
|
90
|
+
}
|
|
91
|
+
/** An ASCII letter, with which a scheme starts. */
|
|
92
|
+
function isLetter(code) {
|
|
93
|
+
return (code >= 0x61 && code <= 0x7a) || (code >= 0x41 && code <= 0x5a);
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Where the scheme of the `://` at `at` starts: the first letter of the run of scheme characters
|
|
97
|
+
* before it, looking back no further than `from`; -1 when there is none (`1://`, ` ://`). A run
|
|
98
|
+
* led by other scheme characters (`-https://`) starts at its first letter.
|
|
99
|
+
*/
|
|
100
|
+
function schemeStart(text, at, from) {
|
|
101
|
+
let start = at;
|
|
102
|
+
while (start > from && isSchemeChar(text.charCodeAt(start - 1)))
|
|
103
|
+
start--;
|
|
104
|
+
while (start < at && !isLetter(text.charCodeAt(start)))
|
|
105
|
+
start++;
|
|
106
|
+
return start < at ? start : -1;
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* `text` with the userinfo of every URL in it replaced by `***` (`https://user:secret@host`
|
|
110
|
+
* becomes `https://***@host`): the pattern backstop for text no exact secret matches. Text-based,
|
|
111
|
+
* so it also covers a URL that does not parse; the userinfo ends at the last `@` before the first
|
|
112
|
+
* whitespace, `/`, `?`, `#` or `'` after `://`. Linear in `text`, however many URLs it holds: each
|
|
113
|
+
* `://` is found with `indexOf`, its scheme by walking back no further than the previous `://`,
|
|
114
|
+
* and the search goes on right after each `://`, not after the authority it read: a URL glued
|
|
115
|
+
* onto the one before it (`https://host://u:p@h`, `https://u:p@x://v:q@w`) starts inside that
|
|
116
|
+
* authority, which ends at its `/`. An authority holds no `/`, so no `://` lies wholly inside one,
|
|
117
|
+
* and every character is read a bounded number of times: by the authority loop and `indexOf`
|
|
118
|
+
* forwards, by `schemeStart` backwards. The regex it replaces (a scheme tried at every
|
|
119
|
+
* word boundary) was quadratic: an 80 KB answer of `a.a.…` took about 5 s to print.
|
|
120
|
+
*/
|
|
121
|
+
export function redactUrlUserinfo(text) {
|
|
122
|
+
const parts = new TextBuilder();
|
|
123
|
+
let copied = 0;
|
|
124
|
+
let from = 0;
|
|
125
|
+
for (let at = text.indexOf("://"); at !== -1; at = text.indexOf("://", from)) {
|
|
126
|
+
const begin = at + 3;
|
|
127
|
+
let end = begin;
|
|
128
|
+
let lastAt = -1;
|
|
129
|
+
for (; end < text.length; end++) {
|
|
130
|
+
const c = text.charCodeAt(end);
|
|
131
|
+
if (c === 0x2f || c === 0x3f || c === 0x23 || c === 0x27 || /\s/.test(text[end]))
|
|
132
|
+
break;
|
|
133
|
+
if (c === 0x40)
|
|
134
|
+
lastAt = end;
|
|
135
|
+
}
|
|
136
|
+
if (lastAt !== -1 && schemeStart(text, at, from) !== -1) {
|
|
137
|
+
parts.push(text.slice(copied, begin), "***");
|
|
138
|
+
copied = lastAt;
|
|
139
|
+
}
|
|
140
|
+
from = begin;
|
|
141
|
+
}
|
|
142
|
+
if (copied === 0)
|
|
143
|
+
return text;
|
|
144
|
+
parts.push(text.slice(copied));
|
|
145
|
+
return parts.toString();
|
|
146
|
+
}
|
|
147
|
+
/** How many pieces a TextBuilder joins at a time. */
|
|
148
|
+
const BUILDER_BATCH = 16_384;
|
|
149
|
+
/**
|
|
150
|
+
* A text put together from many pieces (slices of a server answer and their replacements),
|
|
151
|
+
* joined in batches of BUILDER_BATCH into flat strings, so memory stays proportional to the text,
|
|
152
|
+
* not to the number of pieces. One `+=` per piece made a rope node each, and one array of every
|
|
153
|
+
* piece a slot each: millions of lone-surrogate repairs in a large answer exhausted a small heap
|
|
154
|
+
* either way.
|
|
155
|
+
*/
|
|
156
|
+
export class TextBuilder {
|
|
157
|
+
#done = [];
|
|
158
|
+
#batch = [];
|
|
159
|
+
push(...pieces) {
|
|
160
|
+
for (const piece of pieces)
|
|
161
|
+
this.#batch.push(piece);
|
|
162
|
+
if (this.#batch.length >= BUILDER_BATCH) {
|
|
163
|
+
this.#done.push(this.#batch.join(""));
|
|
164
|
+
this.#batch = [];
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
toString() {
|
|
168
|
+
this.#done.push(this.#batch.join(""));
|
|
169
|
+
this.#batch = [];
|
|
170
|
+
return this.#done.join("");
|
|
171
|
+
}
|
|
172
|
+
}
|
|
55
173
|
/**
|
|
56
174
|
* The forms in which a server may echo the credentials of a userinfo (`user:password`,
|
|
57
|
-
* as `credentialsIn` returns it) back in an error body
|
|
58
|
-
* (base64 of the decoded `user:password`, UTF-8 as the engine
|
|
59
|
-
* `user:password` itself, and the
|
|
60
|
-
*
|
|
61
|
-
*
|
|
175
|
+
* as `credentialsIn` returns it) back in an error body, in this order: the
|
|
176
|
+
* `Authorization: Basic` value (base64 of the decoded `user:password`, UTF-8 as the engine
|
|
177
|
+
* sends it), the decoded `user:password` itself, and the secret alone when it is at least 4
|
|
178
|
+
* characters long. The secret is the password; in a userinfo without one (`TOKEN@host`,
|
|
179
|
+
* `TOKEN:@host`) it is the user name, a token, which the engine sends as Basic `TOKEN:`. `[]`
|
|
180
|
+
* for an empty userinfo. None of them has an `@` to anchor on, so they are replaced as exact
|
|
181
|
+
* strings (`redactSecrets`).
|
|
62
182
|
*/
|
|
63
183
|
export function echoedCredentialForms(userinfo) {
|
|
64
184
|
const colon = userinfo.indexOf(":");
|
|
65
|
-
if (colon < 0)
|
|
66
|
-
return [];
|
|
67
185
|
const decode = (part) => {
|
|
68
186
|
try {
|
|
69
187
|
return decodeURIComponent(part);
|
|
@@ -72,14 +190,15 @@ export function echoedCredentialForms(userinfo) {
|
|
|
72
190
|
return part;
|
|
73
191
|
}
|
|
74
192
|
};
|
|
75
|
-
const user = decode(userinfo.slice(0, colon));
|
|
76
|
-
const password = decode(userinfo.slice(colon + 1));
|
|
77
|
-
|
|
193
|
+
const user = decode(colon < 0 ? userinfo : userinfo.slice(0, colon));
|
|
194
|
+
const password = colon < 0 ? "" : decode(userinfo.slice(colon + 1));
|
|
195
|
+
const secret = password === "" ? user : password;
|
|
196
|
+
if (secret === "")
|
|
78
197
|
return [];
|
|
79
198
|
const pair = `${user}:${password}`;
|
|
80
199
|
const forms = [Buffer.from(pair, "utf8").toString("base64"), pair];
|
|
81
|
-
if (
|
|
82
|
-
forms.push(
|
|
200
|
+
if (secret.length >= 4)
|
|
201
|
+
forms.push(secret);
|
|
83
202
|
return forms;
|
|
84
203
|
}
|
|
85
204
|
/** `text` with every occurrence of each of `secrets` (exact strings, no anchor) replaced by `***`. */
|
|
@@ -186,6 +305,12 @@ export class CkanApiError extends CkanError {
|
|
|
186
305
|
detail;
|
|
187
306
|
url;
|
|
188
307
|
method;
|
|
308
|
+
/**
|
|
309
|
+
* The answer's body as text, the base URL's credentials scrubbed. As sent, unless it is JSON
|
|
310
|
+
* whose decoded text still holds an echoed credential behind an escape: then the decoded JSON
|
|
311
|
+
* written again (other escapes, no insignificant whitespace) with it replaced by `***`, and `""`
|
|
312
|
+
* for such a body nested too deeply to write again.
|
|
313
|
+
*/
|
|
189
314
|
body;
|
|
190
315
|
/** How many times the engine retried the request before giving up (0 when it did not). */
|
|
191
316
|
retries;
|
|
@@ -291,8 +416,21 @@ export function describeCkanError(error) {
|
|
|
291
416
|
* quotes and newlines escaped several levels deep.
|
|
292
417
|
*/
|
|
293
418
|
function solrReason(message) {
|
|
294
|
-
|
|
419
|
+
// Only a window after the first "[Reason: " is read: the pattern's lazy group, tried against
|
|
420
|
+
// `\\+n` at every step, is quadratic on a run of backslashes, and the message is uncut server
|
|
421
|
+
// text (80 KB took 2.4 s, past --timeout, and it grows with the square). The window is twice what a
|
|
422
|
+
// message shows; a reason that runs past it ends at the window's end.
|
|
423
|
+
const at = message.indexOf(REASON);
|
|
424
|
+
if (at === -1)
|
|
425
|
+
return undefined;
|
|
426
|
+
const end = at + REASON.length + 2 * MAX_MESSAGE_VALUE_LENGTH;
|
|
427
|
+
const window = message.slice(at, end);
|
|
428
|
+
const pattern = end < message.length ? SOLR_REASON_IN_CUT_WINDOW : SOLR_REASON;
|
|
429
|
+
const match = pattern.exec(window);
|
|
295
430
|
if (!match?.[1])
|
|
296
431
|
return undefined;
|
|
297
432
|
return match[1].replace(/\\+(['"])/g, "$1").trim();
|
|
298
433
|
}
|
|
434
|
+
const REASON = "[Reason: ";
|
|
435
|
+
const SOLR_REASON = /^\[Reason: (?:[\w.$]+(?:Exception|Error): )?([\s\S]*?)(?:\\+n|\n|\])/;
|
|
436
|
+
const SOLR_REASON_IN_CUT_WINDOW = /^\[Reason: (?:[\w.$]+(?:Exception|Error): )?([\s\S]*?)(?:\\+n|\n|\]|$)/;
|
|
@@ -48,6 +48,7 @@ export declare const MAX_TIMEOUT_MS = 2147483647;
|
|
|
48
48
|
/**
|
|
49
49
|
* Default transport. Resolves with the raw response (including non-2xx) — status
|
|
50
50
|
* interpretation is the client's job. Rejects only on transport-level failures
|
|
51
|
-
* (connection errors, timeouts, malformed URLs).
|
|
51
|
+
* (connection errors, timeouts, malformed URLs). The body's memory follows its bytes, not the
|
|
52
|
+
* number of chunks a server sends it in: small chunks are joined as they arrive.
|
|
52
53
|
*/
|
|
53
54
|
export declare const nodeHttpTransport: Transport;
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
// `http.createServer` in the test-suite.
|
|
8
8
|
import http from "node:http";
|
|
9
9
|
import https from "node:https";
|
|
10
|
-
import { CkanNetworkError, redactUrl } from "./errors.js";
|
|
10
|
+
import { CkanNetworkError, redactUrl, credentialsIn, redactCredentials, typedUrlCredentials } from "./errors.js";
|
|
11
11
|
/** The message for a body over the size cap, naming the option on both sides. */
|
|
12
12
|
export function sizeLimitMessage(maxBytes) {
|
|
13
13
|
return `Response exceeded the size limit of ${maxBytes} bytes (maxResponseBytes; --max-response-bytes on the CLI)`;
|
|
@@ -17,10 +17,23 @@ export function sizeLimitMessage(maxBytes) {
|
|
|
17
17
|
* prints a TimeoutOverflowWarning and fires after 1 ms, so timeouts are capped here.
|
|
18
18
|
*/
|
|
19
19
|
export const MAX_TIMEOUT_MS = 2_147_483_647;
|
|
20
|
+
/** The most received chunks the default transport keeps unjoined. */
|
|
21
|
+
const COALESCE_CHUNKS = 1024;
|
|
22
|
+
/** The bytes of unjoined chunks at which the default transport joins them into one Buffer. */
|
|
23
|
+
const COALESCE_BYTES = 64 * 1024;
|
|
24
|
+
/**
|
|
25
|
+
* `url` for a message, its userinfo redacted; one typed without its scheme (`user:pw@host`,
|
|
26
|
+
* `TOKEN:@host`) or with a mistyped one (`http//user:pw@host`) is read as a URL all the same
|
|
27
|
+
* (`typedUrlCredentials`), since `redactUrl` finds a userinfo only after a scheme.
|
|
28
|
+
*/
|
|
29
|
+
function shownUrl(url) {
|
|
30
|
+
return credentialsIn(url).length > 0 ? redactUrl(url) : redactCredentials(url, typedUrlCredentials(url));
|
|
31
|
+
}
|
|
20
32
|
/**
|
|
21
33
|
* Default transport. Resolves with the raw response (including non-2xx) — status
|
|
22
34
|
* interpretation is the client's job. Rejects only on transport-level failures
|
|
23
|
-
* (connection errors, timeouts, malformed URLs).
|
|
35
|
+
* (connection errors, timeouts, malformed URLs). The body's memory follows its bytes, not the
|
|
36
|
+
* number of chunks a server sends it in: small chunks are joined as they arrive.
|
|
24
37
|
*/
|
|
25
38
|
export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
|
|
26
39
|
let url;
|
|
@@ -28,14 +41,17 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
|
|
|
28
41
|
url = new URL(request.url);
|
|
29
42
|
}
|
|
30
43
|
catch {
|
|
31
|
-
reject(new CkanNetworkError(`Invalid URL: ${
|
|
44
|
+
reject(new CkanNetworkError(`Invalid URL: ${shownUrl(request.url)}`));
|
|
32
45
|
return;
|
|
33
46
|
}
|
|
34
47
|
// Only http/https are supported. Reject anything else up front with a clear,
|
|
35
48
|
// typed error instead of letting Node throw an opaque ERR_INVALID_PROTOCOL
|
|
36
49
|
// (and so this never reaches the file:/ftp:/etc. drivers).
|
|
37
50
|
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
38
|
-
|
|
51
|
+
// `user:password@host` without a scheme parses with the user name as the scheme: name a
|
|
52
|
+
// scheme only when the value spells one (`<scheme>://`).
|
|
53
|
+
const named = /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(request.url) ? ` "${url.protocol}"` : "";
|
|
54
|
+
reject(new CkanNetworkError(`Unsupported protocol${named} in URL: ${shownUrl(request.url)}`));
|
|
39
55
|
return;
|
|
40
56
|
}
|
|
41
57
|
const isHttps = url.protocol === "https:";
|
|
@@ -62,7 +78,12 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
|
|
|
62
78
|
method: request.method,
|
|
63
79
|
headers: request.headers,
|
|
64
80
|
}, (res) => {
|
|
81
|
+
// Small chunks are joined as they arrive (`pending`, at most COALESCE_CHUNKS of them
|
|
82
|
+
// or COALESCE_BYTES), so the Buffers kept follow the bytes, not the number of chunks
|
|
83
|
+
// a server chooses to send: a Buffer per one-byte chunk held ~210 bytes of heap each.
|
|
65
84
|
const chunks = [];
|
|
85
|
+
let pending = [];
|
|
86
|
+
let pendingBytes = 0;
|
|
66
87
|
let received = 0;
|
|
67
88
|
let aborted = false;
|
|
68
89
|
res.on("data", (chunk) => {
|
|
@@ -75,7 +96,13 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
|
|
|
75
96
|
settleReject(new CkanNetworkError(sizeLimitMessage(maxBytes)));
|
|
76
97
|
return;
|
|
77
98
|
}
|
|
78
|
-
|
|
99
|
+
pending.push(chunk);
|
|
100
|
+
pendingBytes += chunk.length;
|
|
101
|
+
if (pending.length >= COALESCE_CHUNKS || pendingBytes >= COALESCE_BYTES) {
|
|
102
|
+
chunks.push(pending.length === 1 ? chunk : Buffer.concat(pending, pendingBytes));
|
|
103
|
+
pending = [];
|
|
104
|
+
pendingBytes = 0;
|
|
105
|
+
}
|
|
79
106
|
});
|
|
80
107
|
res.on("end", () => {
|
|
81
108
|
if (aborted)
|
|
@@ -83,7 +110,7 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
|
|
|
83
110
|
settleResolve({
|
|
84
111
|
status: res.statusCode ?? 0,
|
|
85
112
|
headers: res.headers,
|
|
86
|
-
body: Buffer.concat(chunks),
|
|
113
|
+
body: Buffer.concat([...chunks, ...pending]),
|
|
87
114
|
});
|
|
88
115
|
});
|
|
89
116
|
res.on("error", (err) => {
|
|
@@ -117,13 +117,17 @@ export declare function headerValueProblem(value: string): string | undefined;
|
|
|
117
117
|
*
|
|
118
118
|
* - not blank;
|
|
119
119
|
* - no surrounding or embedded whitespace and no control characters: `new URL`
|
|
120
|
-
* trims and drops tab/newline silently,
|
|
121
|
-
*
|
|
122
|
-
* (`/ckan%20/api/3/...`) or reach a custom transport raw;
|
|
120
|
+
* trims and drops tab/newline silently, so the URL requested would not be the one
|
|
121
|
+
* typed (a trailing space would request another path, `/ckan%20/api/3/...`);
|
|
123
122
|
* - an absolute URL with an http(s) scheme (`file:`, `ftp:` … never reach a
|
|
124
|
-
* transport);
|
|
123
|
+
* transport); another scheme is named only when the value spells `<scheme>://`, since
|
|
124
|
+
* `user:password@host` parses with the user name as its scheme;
|
|
125
125
|
* - no query string or fragment: the API path is appended, so it would land in
|
|
126
126
|
* front of `/api/3/action`;
|
|
127
|
+
* - no backslash, and `http://` or `https://` followed by the host (not `http:host`,
|
|
128
|
+
* `http:/host` or `http:///host`): WHATWG ends an http(s) host at a backslash and skips
|
|
129
|
+
* any slashes after the scheme, so text-based code would read another host or no userinfo
|
|
130
|
+
* (`https://u:pw@a.example\@b.example` sent `u:pw` to b.example);
|
|
127
131
|
* - a `%` in the user name or password must start a valid escape (`%25` for a
|
|
128
132
|
* literal one): the engine decodes the userinfo for the Authorization header.
|
|
129
133
|
*
|
|
@@ -278,13 +278,17 @@ export function headerValueProblem(value) {
|
|
|
278
278
|
*
|
|
279
279
|
* - not blank;
|
|
280
280
|
* - no surrounding or embedded whitespace and no control characters: `new URL`
|
|
281
|
-
* trims and drops tab/newline silently,
|
|
282
|
-
*
|
|
283
|
-
* (`/ckan%20/api/3/...`) or reach a custom transport raw;
|
|
281
|
+
* trims and drops tab/newline silently, so the URL requested would not be the one
|
|
282
|
+
* typed (a trailing space would request another path, `/ckan%20/api/3/...`);
|
|
284
283
|
* - an absolute URL with an http(s) scheme (`file:`, `ftp:` … never reach a
|
|
285
|
-
* transport);
|
|
284
|
+
* transport); another scheme is named only when the value spells `<scheme>://`, since
|
|
285
|
+
* `user:password@host` parses with the user name as its scheme;
|
|
286
286
|
* - no query string or fragment: the API path is appended, so it would land in
|
|
287
287
|
* front of `/api/3/action`;
|
|
288
|
+
* - no backslash, and `http://` or `https://` followed by the host (not `http:host`,
|
|
289
|
+
* `http:/host` or `http:///host`): WHATWG ends an http(s) host at a backslash and skips
|
|
290
|
+
* any slashes after the scheme, so text-based code would read another host or no userinfo
|
|
291
|
+
* (`https://u:pw@a.example\@b.example` sent `u:pw` to b.example);
|
|
288
292
|
* - a `%` in the user name or password must start a valid escape (`%25` for a
|
|
289
293
|
* literal one): the engine decodes the userinfo for the Authorization header.
|
|
290
294
|
*
|
|
@@ -311,10 +315,23 @@ export function baseUrlProblem(value) {
|
|
|
311
315
|
return "Expected an absolute http(s) URL.";
|
|
312
316
|
}
|
|
313
317
|
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
318
|
+
// `user:password@host` without a scheme parses as a URL whose scheme is the user name
|
|
319
|
+
// ("t0kens3cret:"), lower-cased, which no exact-string redaction can match: name a scheme
|
|
320
|
+
// only when the value spells one, `<scheme>://`.
|
|
321
|
+
if (!/^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(value))
|
|
322
|
+
return "Expected an absolute http(s) URL.";
|
|
314
323
|
return `Unsupported scheme "${cutForMessage(url.protocol)}". Expected an http(s) URL.`;
|
|
315
324
|
}
|
|
316
325
|
if (/[?#]/.test(value))
|
|
317
326
|
return "Expected a site URL without a query string or fragment.";
|
|
327
|
+
// WHATWG reads a "\" in an http(s) URL as "/", so it ends the host, and it skips any number of
|
|
328
|
+
// slashes after "http:": `http://u:pw@a.example\@b.example` is a.example to the parser and
|
|
329
|
+
// was b.example to text-based code, and `http:u:pw@host` hid the userinfo from the redaction.
|
|
330
|
+
// Every reader of the base URL has to agree where its host is.
|
|
331
|
+
if (value.includes("\\"))
|
|
332
|
+
return 'A base URL cannot contain a backslash (\\); write "/" instead.';
|
|
333
|
+
if (!/^https?:\/\/[^/]/i.test(value))
|
|
334
|
+
return 'Expected an absolute http(s) URL ("https://" and the host).';
|
|
318
335
|
// The engine decodes the userinfo into the Authorization header; a "%" that isn't an
|
|
319
336
|
// escape would fail there ("URI malformed") at request time. Reject it here.
|
|
320
337
|
for (const part of [url.username, url.password]) {
|
package/dist/src/cli/run.d.ts
CHANGED
|
@@ -1,13 +1,6 @@
|
|
|
1
1
|
import { type Command } from "commander";
|
|
2
2
|
import { type CliDeps } from "./io.js";
|
|
3
3
|
import { type Logger } from "./log.js";
|
|
4
|
-
/**
|
|
5
|
-
* Replace the userinfo of every URL in `text` with `***`, the form `redactUrl` gives
|
|
6
|
-
* (`https://user:secret@host` becomes `https://***@host`). Text-based, so it also
|
|
7
|
-
* covers a URL that does not parse; a fallback behind the exact strings of
|
|
8
|
-
* `withRedactedOutput`.
|
|
9
|
-
*/
|
|
10
|
-
export declare function redactUserinfo(text: string): string;
|
|
11
4
|
/** The secrets of a run, and the two ways they are replaced. */
|
|
12
5
|
export interface Redaction {
|
|
13
6
|
/**
|
package/dist/src/cli/run.js
CHANGED
|
@@ -6,7 +6,7 @@ import { buildProgram, defaultDeps } from "./program.js";
|
|
|
6
6
|
import { logOf } from "./io.js";
|
|
7
7
|
import { OutputError } from "./output.js";
|
|
8
8
|
import { createLogger, logFormatFromArgv } from "./log.js";
|
|
9
|
-
import { CkanActionError, CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, echoedCredentialForms, redactCredentials, redactSecrets, } from "@maschinenlesbar.org/ckan-client/internal/errors";
|
|
9
|
+
import { CkanActionError, CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, echoedCredentialForms, redactCredentials, redactSecrets, redactUrlUserinfo, typedUrlCredentials, } from "@maschinenlesbar.org/ckan-client/internal/errors";
|
|
10
10
|
/** The exit code of a usage error: commander's own for a rejected option value. */
|
|
11
11
|
const USAGE_EXIT = 1;
|
|
12
12
|
/**
|
|
@@ -107,15 +107,6 @@ function escapeCommanderError(message) {
|
|
|
107
107
|
return escapeTerminalText(message);
|
|
108
108
|
return escapeTerminalText(message.slice(0, hint.index)) + hint[0];
|
|
109
109
|
}
|
|
110
|
-
/**
|
|
111
|
-
* Replace the userinfo of every URL in `text` with `***`, the form `redactUrl` gives
|
|
112
|
-
* (`https://user:secret@host` becomes `https://***@host`). Text-based, so it also
|
|
113
|
-
* covers a URL that does not parse; a fallback behind the exact strings of
|
|
114
|
-
* `withRedactedOutput`.
|
|
115
|
-
*/
|
|
116
|
-
export function redactUserinfo(text) {
|
|
117
|
-
return text.replace(/\b([a-z][a-z0-9+.-]*:\/\/)[^\s/?#']*@/gi, "$1***@");
|
|
118
|
-
}
|
|
119
110
|
/**
|
|
120
111
|
* The options whose value is a base URL (`--portal` takes a portal's URL as well as its
|
|
121
112
|
* id): a `user:password@host` given there without its scheme is still a credential, like
|
|
@@ -182,14 +173,16 @@ export function redactionFor(argv, env) {
|
|
|
182
173
|
});
|
|
183
174
|
// A base URL typed without its scheme is read as if it had one (anywhere else a bare
|
|
184
175
|
// `a:b@c` is no credential: a dataset id, a search text, a User-Agent).
|
|
185
|
-
|
|
186
|
-
|
|
176
|
+
// Read as a URL however its scheme is typed: missing, special without "//", or mistyped
|
|
177
|
+
// (`http//u:pw@h`, the colon left out): typedUrlCredentials.
|
|
178
|
+
const baseUrls = [...flagValues(argv, BASE_URL_FLAGS), env["CKAN_BASE_URL"] ?? ""];
|
|
179
|
+
const sources = [...values.flatMap(urlsWithin).map(credentialsIn), ...baseUrls.map(typedUrlCredentials)];
|
|
187
180
|
const secrets = new Set();
|
|
188
181
|
const encoded = new Set();
|
|
189
182
|
const echoed = new Set();
|
|
190
183
|
const passwords = new Set();
|
|
191
|
-
for (const
|
|
192
|
-
for (const secret of
|
|
184
|
+
for (const found of sources) {
|
|
185
|
+
for (const secret of found) {
|
|
193
186
|
secrets.add(secret);
|
|
194
187
|
secrets.add(JSON.stringify(secret).slice(1, -1));
|
|
195
188
|
// As commander's echo and the library's quoteValue show it: invisible characters escaped.
|
|
@@ -216,7 +209,7 @@ export function redactionFor(argv, env) {
|
|
|
216
209
|
const echoedList = [...echoed].sort((a, b) => b.length - a.length);
|
|
217
210
|
const passwordList = [...passwords].sort((a, b) => b.length - a.length);
|
|
218
211
|
const out = (text) => {
|
|
219
|
-
let result =
|
|
212
|
+
let result = redactUrlUserinfo(redactCredentials(text, list));
|
|
220
213
|
for (const form of encoded)
|
|
221
214
|
result = result.split(form).join("***%40");
|
|
222
215
|
return redactSecrets(result, echoedList);
|
package/package.json
CHANGED