@maschinenlesbar.org/ckan-cli 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.d.ts +25 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.js +64 -23
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.d.ts +26 -4
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.js +152 -45
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.d.ts +52 -6
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.js +155 -17
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/http.d.ts +2 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/http.js +33 -6
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.d.ts +2 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.js +2 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/operations.d.ts +441 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/operations.js +1211 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.d.ts +2 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.js +2 -2
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.d.ts +8 -4
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.js +21 -4
- package/dist/node_modules/@maschinenlesbar.org/ckan-log/dist/src/index.d.ts +1 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-log/dist/src/index.js +1 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-log/dist/src/log.d.ts +80 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-log/dist/src/log.js +124 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-log/package.json +25 -0
- package/dist/src/cli/index.js +3 -1
- package/dist/src/cli/io.d.ts +17 -0
- package/dist/src/cli/io.js +14 -0
- package/dist/src/cli/limits.d.ts +19 -0
- package/dist/src/cli/limits.js +53 -0
- package/dist/src/cli/program.js +3 -0
- package/dist/src/cli/run.d.ts +0 -7
- package/dist/src/cli/run.js +17 -16
- package/dist/src/cli/shared.d.ts +4 -0
- package/dist/src/cli/shared.js +16 -1
- package/node_modules/commander/Readme.md +6 -32
- package/node_modules/commander/index.js +1 -1
- package/node_modules/commander/lib/command.js +22 -9
- package/node_modules/commander/lib/help.js +6 -2
- package/node_modules/commander/lib/option.js +1 -1
- package/node_modules/commander/package.json +2 -1
- package/node_modules/commander/typings/index.d.ts +6 -3
- package/package.json +2 -2
|
@@ -5,7 +5,7 @@ import { constants } from "node:buffer";
|
|
|
5
5
|
import { TextDecoder } from "node:util";
|
|
6
6
|
import { MAX_TIMEOUT_MS, nodeHttpTransport, sizeLimitMessage } from "./http.js";
|
|
7
7
|
import { buildQueryString } from "./query.js";
|
|
8
|
-
import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, describeCkanError, echoedCredentialForms, redactCredentials, redactSecrets, redactUrl, } from "./errors.js";
|
|
8
|
+
import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, cutText, describeCkanError, echoedCredentialForms, redactCredentials, redactSecrets, redactUrl, TextBuilder, } from "./errors.js";
|
|
9
9
|
import { assertValid, baseUrlProblem, headerValueProblem, intRangeProblem } from "./validate.js";
|
|
10
10
|
// quoteValue lives with the other message helpers (validate.ts quotes refused values with it).
|
|
11
11
|
export { quoteValue } from "./errors.js";
|
|
@@ -80,35 +80,57 @@ export function isBidiControl(code) {
|
|
|
80
80
|
(code >= 0x202a && code <= 0x202e) ||
|
|
81
81
|
(code >= 0x2066 && code <= 0x2069));
|
|
82
82
|
}
|
|
83
|
+
/**
|
|
84
|
+
* The most server text (in UTF-16 units) `sanitizeServerText` reads; a longer text is cut first.
|
|
85
|
+
* Every message keeps far less of it (MAX_MESSAGE_VALUE_LENGTH, 500 characters), and cleaning all
|
|
86
|
+
* of a 16 MB error detail, one string node per character, exhausted a small heap.
|
|
87
|
+
*/
|
|
88
|
+
const MAX_SERVER_TEXT = 16 * 1024;
|
|
89
|
+
/**
|
|
90
|
+
* A whole terminal escape sequence: a CSI (`ESC [` or the 8-bit U+009B, parameters,
|
|
91
|
+
* intermediates, final byte), a terminated control string (OSC/DCS/SOS/PM/APC, `ESC ]` …
|
|
92
|
+
* up to BEL or ST, or their 8-bit forms) or a short `ESC` sequence. Dropping only the
|
|
93
|
+
* control bytes left the printable rest behind (`ESC ] 0 ; title BEL` became `]0;title`).
|
|
94
|
+
* An unterminated control string is left to the per-character filter, so it cannot
|
|
95
|
+
* swallow the rest of the message. A control string's body ends at an ESC or at another
|
|
96
|
+
* introducer: with the introducers allowed inside it, an unterminated run of them was matched
|
|
97
|
+
* from each one to the end of the text, quadratic (100 000 × U+009D took 16 s). Written with
|
|
98
|
+
* escapes, so the source holds no control bytes.
|
|
99
|
+
*/
|
|
100
|
+
const ESCAPE_SEQUENCE = /(?:\u001b\[|\u009b)[0-?]*[ -/]*[@-~]|(?:\u001b[\]PX^_]|[\u0090\u0098\u009d-\u009f])[^\u0007\u001b\u0090\u0098\u009c-\u009f]*(?:\u0007|\u001b\\|\u009c)|\u001b[ -/]*[0-~]/g;
|
|
101
|
+
/**
|
|
102
|
+
* What `sanitizeServerText` drops once the escape sequences are gone: the C0 controls but the
|
|
103
|
+
* whitespace ones (TAB to CR), DEL, the C1 controls, and every format character (General
|
|
104
|
+
* Category Cf: bidi controls, zero-width characters, U+00AD, U+FEFF, …).
|
|
105
|
+
*/
|
|
106
|
+
const DROPPED_CHAR = /[\u0000-\u0008\u000e-\u001f\u007f-\u009f\p{Cf}]/gu;
|
|
83
107
|
/**
|
|
84
108
|
* Make a string that originates in an attacker-controlled response — the error
|
|
85
109
|
* `detail`, a CKAN `success:false` error, the echoed Content-Type — safe to print
|
|
86
110
|
* into an error message on stderr:
|
|
87
111
|
*
|
|
88
|
-
* -
|
|
112
|
+
* - Whole escape sequences (CSI, OSC and other control strings) are dropped first,
|
|
113
|
+
* then any remaining C0 and C1 controls and DEL. A JSON error body can encode an escape
|
|
89
114
|
* (U+001B) that JSON.parse turns into a real control byte; printed raw, a hostile
|
|
90
115
|
* or MITM'd endpoint could drive ANSI/OSC sequences into the terminal (display
|
|
91
116
|
* spoofing, title changes, OSC 52 clipboard writes).
|
|
92
|
-
* -
|
|
93
|
-
* reorder the visible message
|
|
117
|
+
* - Unicode format characters (General Category Cf) are dropped: the bidi controls
|
|
118
|
+
* (isBidiControl), so server text cannot reorder the visible message, and the
|
|
119
|
+
* invisible ones — zero-width space/joiners, word joiner, soft hyphen, BOM — so
|
|
120
|
+
* two messages that look identical are identical.
|
|
94
121
|
* - Every run of whitespace — newlines, tabs, U+2028/U+2029 included — becomes one
|
|
95
122
|
* space and the ends are trimmed, so the text stays on one line and a server
|
|
96
123
|
* cannot forge an `Error:` line of its own.
|
|
124
|
+
* - Only the first MAX_SERVER_TEXT (16 384) units are read, and every step is a linear
|
|
125
|
+
* regex replace: a hostile error body costs neither seconds nor the heap. A caller that
|
|
126
|
+
* scrubs credentials does so first, so a cut never splits one.
|
|
97
127
|
*
|
|
98
128
|
* The CLI's JSON output is escaped separately (`escapeControlChars` in
|
|
99
129
|
* cli/shared.ts): `JSON.stringify` alone leaves DEL, C1 and bidi characters raw.
|
|
100
|
-
* Written
|
|
130
|
+
* Written with escapes so no raw control byte appears in this source.
|
|
101
131
|
*/
|
|
102
132
|
export function sanitizeServerText(text) {
|
|
103
|
-
|
|
104
|
-
for (const ch of text) {
|
|
105
|
-
const n = ch.codePointAt(0) ?? 0;
|
|
106
|
-
const whitespaceControl = n >= 0x09 && n <= 0x0d;
|
|
107
|
-
if (!whitespaceControl && (n <= 0x1f || (n >= 0x7f && n <= 0x9f) || isBidiControl(n)))
|
|
108
|
-
continue;
|
|
109
|
-
out += ch;
|
|
110
|
-
}
|
|
111
|
-
return out.replace(/\s+/g, " ").trim();
|
|
133
|
+
return cutText(text, MAX_SERVER_TEXT).replace(ESCAPE_SEQUENCE, "").replace(DROPPED_CHAR, "").replace(/\s+/g, " ").trim();
|
|
112
134
|
}
|
|
113
135
|
/**
|
|
114
136
|
* Check a base URL (baseUrlProblem: not blank, no whitespace or control
|
|
@@ -246,6 +268,8 @@ export class RequestEngine {
|
|
|
246
268
|
// JSON.stringify of a client never show it, so a password in the base URL can't be
|
|
247
269
|
// logged by accident. Messages use redactUrl.
|
|
248
270
|
#baseUrl;
|
|
271
|
+
/** `#baseUrl` without its userinfo: what every request URL starts with. */
|
|
272
|
+
#requestBase;
|
|
249
273
|
/** The base URL's userinfo, raw and percent-decoded, for scrubbing server and transport text. */
|
|
250
274
|
#credentials;
|
|
251
275
|
/**
|
|
@@ -272,8 +296,18 @@ export class RequestEngine {
|
|
|
272
296
|
// consumer that injects a custom transport would otherwise get no gating at
|
|
273
297
|
// all, and could be steered to a non-http(s) scheme. Only undefined selects
|
|
274
298
|
// the default.
|
|
275
|
-
|
|
276
|
-
|
|
299
|
+
const baseUrl = validateBaseUrl(options.baseUrl ?? DEFAULT_BASE_URL);
|
|
300
|
+
// Requests go to the base URL as WHATWG parses it, the reading every check of it makes (the
|
|
301
|
+
// host, the cleartext warning, the Authorization header), never to a text-based reading of
|
|
302
|
+
// the string: a transport that reads the URL as text reaches the same host.
|
|
303
|
+
const parsed = new URL(baseUrl);
|
|
304
|
+
this.#baseUrl = parsed.href.replace(/\/+$/, "");
|
|
305
|
+
parsed.username = "";
|
|
306
|
+
parsed.password = "";
|
|
307
|
+
this.#requestBase = parsed.href.replace(/\/+$/, "");
|
|
308
|
+
// The userinfo as typed and as sent (WHATWG percent-encodes some of its characters).
|
|
309
|
+
const userinfos = [...new Set([...credentialsIn(baseUrl), ...credentialsIn(this.#baseUrl)])];
|
|
310
|
+
this.#credentials = userinfos.flatMap((raw) => {
|
|
277
311
|
try {
|
|
278
312
|
return [raw, decodeURIComponent(raw)];
|
|
279
313
|
}
|
|
@@ -281,8 +315,7 @@ export class RequestEngine {
|
|
|
281
315
|
return [raw];
|
|
282
316
|
}
|
|
283
317
|
});
|
|
284
|
-
this.#echoed =
|
|
285
|
-
.flatMap(echoedCredentialForms)
|
|
318
|
+
this.#echoed = [...new Set(userinfos.flatMap(echoedCredentialForms).flatMap(urlSpellings))]
|
|
286
319
|
.sort((a, b) => b.length - a.length);
|
|
287
320
|
this.transport = functionOption("transport", options.transport, nodeHttpTransport);
|
|
288
321
|
// Only undefined selects the default; a blank or unsendable value is refused.
|
|
@@ -338,6 +371,34 @@ export class RequestEngine {
|
|
|
338
371
|
scrub(text) {
|
|
339
372
|
return this.#credentials.length === 0 ? text : redactSecrets(redactCredentials(text, this.#credentials), this.#echoed);
|
|
340
373
|
}
|
|
374
|
+
/**
|
|
375
|
+
* An error body, already scrubbed as sent, as `CkanApiError.body` keeps it: when it is JSON
|
|
376
|
+
* with an escape in it, checked decoded too. An escape (`\"`, `\u00e4`) hides an echoed
|
|
377
|
+
* credential from the scrub of the text as sent, while a caller who parses the body sees it. Such
|
|
378
|
+
* a body is kept as the decoded JSON written again (`JSON.stringify`, one spelling per character)
|
|
379
|
+
* with the credential scrubbed in that spelling. A body that doesn't parse is text nobody
|
|
380
|
+
* decodes, and stays as sent; one too deeply nested to write again is left out.
|
|
381
|
+
*/
|
|
382
|
+
scrubBody(scrubbed) {
|
|
383
|
+
if (this.#echoed.length === 0 || !scrubbed.includes("\\"))
|
|
384
|
+
return scrubbed;
|
|
385
|
+
let decoded;
|
|
386
|
+
try {
|
|
387
|
+
decoded = JSON.parse(scrubbed);
|
|
388
|
+
}
|
|
389
|
+
catch {
|
|
390
|
+
return scrubbed;
|
|
391
|
+
}
|
|
392
|
+
let rewritten;
|
|
393
|
+
try {
|
|
394
|
+
rewritten = JSON.stringify(decoded);
|
|
395
|
+
}
|
|
396
|
+
catch {
|
|
397
|
+
return "";
|
|
398
|
+
}
|
|
399
|
+
const spelled = this.#echoed.map((form) => JSON.stringify(form).slice(1, -1));
|
|
400
|
+
return spelled.some((form) => rewritten.includes(form)) ? redactSecrets(rewritten, spelled) : scrubbed;
|
|
401
|
+
}
|
|
341
402
|
/**
|
|
342
403
|
* A failure as the `cause` of the error the engine raises: the original when its
|
|
343
404
|
* text carries no credentials, otherwise a copy with them scrubbed (message, `code`
|
|
@@ -375,7 +436,7 @@ export class RequestEngine {
|
|
|
375
436
|
composeUrl(path, query, withUserinfo) {
|
|
376
437
|
const normalizedPath = path.startsWith("/") ? path : `/${path}`;
|
|
377
438
|
const qs = query ? buildQueryString(query) : "";
|
|
378
|
-
const base = withUserinfo ? this.#baseUrl :
|
|
439
|
+
const base = withUserinfo ? this.#baseUrl : this.#requestBase;
|
|
379
440
|
return `${base}${normalizedPath}${qs ? `?${qs}` : ""}`;
|
|
380
441
|
}
|
|
381
442
|
/**
|
|
@@ -405,6 +466,24 @@ export class RequestEngine {
|
|
|
405
466
|
clearTimeout(timer);
|
|
406
467
|
}
|
|
407
468
|
}
|
|
469
|
+
/**
|
|
470
|
+
* The URL a redirect leads to as messages and errors show it: the Location with the base URL's
|
|
471
|
+
* credentials scrubbed as sent, before resolving percent-encodes what is left of them in a new
|
|
472
|
+
* spelling (`'` as `%27` in a query), then resolved against the URL shown so far, its userinfo
|
|
473
|
+
* left out. A Location the scrub made unparseable (a credential in its host) is shown as scrubbed.
|
|
474
|
+
*/
|
|
475
|
+
shownTarget(location, shown) {
|
|
476
|
+
const sent = this.scrub(location);
|
|
477
|
+
try {
|
|
478
|
+
const target = new URL(sent, shown);
|
|
479
|
+
target.username = "";
|
|
480
|
+
target.password = "";
|
|
481
|
+
return target.href;
|
|
482
|
+
}
|
|
483
|
+
catch {
|
|
484
|
+
return sent;
|
|
485
|
+
}
|
|
486
|
+
}
|
|
408
487
|
/** Perform a request with Accept negotiation and transient-error retries. */
|
|
409
488
|
async request(method, path, options = { accept: "application/json" }) {
|
|
410
489
|
// The transport never sees the base URL's userinfo: the engine sends it as an
|
|
@@ -412,6 +491,9 @@ export class RequestEngine {
|
|
|
412
491
|
// absolute) keeps it and one to another origin or scheme drops it. A transport
|
|
413
492
|
// such as fetch also refuses a URL with credentials outright.
|
|
414
493
|
let url = this.composeUrl(path, options.query, false);
|
|
494
|
+
// The URL as messages, errors and `RawResponse.url` show it: after a redirect, the Location
|
|
495
|
+
// with the base URL's credentials scrubbed (shownTarget); `url` is what is requested.
|
|
496
|
+
let shown = url;
|
|
415
497
|
let headers = {
|
|
416
498
|
Accept: options.accept,
|
|
417
499
|
"User-Agent": this.userAgent,
|
|
@@ -445,7 +527,7 @@ export class RequestEngine {
|
|
|
445
527
|
if (cause instanceof CkanError && !(cause instanceof CkanNetworkError))
|
|
446
528
|
throw cause;
|
|
447
529
|
const reason = cause instanceof Error ? cause.message : String(cause);
|
|
448
|
-
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(
|
|
530
|
+
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(shown))} failed: ${sanitizeServerText(this.scrub(reason))}`, {
|
|
449
531
|
cause: this.scrubCause(cause),
|
|
450
532
|
});
|
|
451
533
|
}
|
|
@@ -453,14 +535,14 @@ export class RequestEngine {
|
|
|
453
535
|
// would otherwise surface below as a raw TypeError, outside the CkanError contract.
|
|
454
536
|
const invalid = responseProblem(response);
|
|
455
537
|
if (invalid !== undefined) {
|
|
456
|
-
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(
|
|
538
|
+
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(shown))} failed: the transport returned an invalid response (${invalid}).`);
|
|
457
539
|
}
|
|
458
540
|
// A transport must not follow redirects itself (`redirect: "manual"`): one that
|
|
459
541
|
// did (fetch's default) may have carried the Authorization header to another
|
|
460
542
|
// host, and the answer is not the one asked for. Reject it when it says so (`url`).
|
|
461
543
|
const finalUrl = response.url;
|
|
462
544
|
if (typeof finalUrl === "string" && finalUrl !== "" && originOf(finalUrl) !== originOf(url)) {
|
|
463
|
-
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(
|
|
545
|
+
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(shown))} failed: the transport followed a redirect to another origin ` +
|
|
464
546
|
`(${cutForMessage(sanitizeServerText(redactUrl(this.scrub(finalUrl))))}); a transport must not follow redirects ` +
|
|
465
547
|
`(HttpRequest.redirect is "manual").`);
|
|
466
548
|
}
|
|
@@ -471,7 +553,7 @@ export class RequestEngine {
|
|
|
471
553
|
// The size cap holds whatever the transport did: the default one aborts early, a
|
|
472
554
|
// custom one may have read everything.
|
|
473
555
|
if (this.maxResponseBytes > 0 && body.byteLength > this.maxResponseBytes) {
|
|
474
|
-
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(
|
|
556
|
+
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(shown))} failed: ${sizeLimitMessage(this.maxResponseBytes)}`);
|
|
475
557
|
}
|
|
476
558
|
const retryable = status === 429 || status === 503;
|
|
477
559
|
// A Retry-After beyond MAX_RETRY_AFTER_MS is not retried: the error below
|
|
@@ -484,7 +566,7 @@ export class RequestEngine {
|
|
|
484
566
|
// for less: `Retry-After: 0` or a date in the past turned the retries into a
|
|
485
567
|
// zero-delay burst against a server that had just asked for less load.
|
|
486
568
|
const backoff = this.retryDelayMs * attempt;
|
|
487
|
-
await this.backOff(attempt, retryAfter === undefined ? backoff : Math.max(retryAfter, backoff),
|
|
569
|
+
await this.backOff(attempt, retryAfter === undefined ? backoff : Math.max(retryAfter, backoff), shown, status);
|
|
488
570
|
continue;
|
|
489
571
|
}
|
|
490
572
|
// Follow redirects, resolving the Location relative to the current URL.
|
|
@@ -500,7 +582,7 @@ export class RequestEngine {
|
|
|
500
582
|
// never reaches a transport (a custom one may not check the scheme).
|
|
501
583
|
if (nextUrl.protocol !== "http:" && nextUrl.protocol !== "https:") {
|
|
502
584
|
throw new CkanNetworkError(`Refusing to follow redirect to unsupported protocol "${cutForMessage(sanitizeServerText(nextUrl.protocol))}" ` +
|
|
503
|
-
`for ${method} ${cutForMessage(redactUrl(
|
|
585
|
+
`for ${method} ${cutForMessage(redactUrl(shown))}`);
|
|
504
586
|
}
|
|
505
587
|
// Credential-strip guard: if the redirect crosses origin (scheme + host +
|
|
506
588
|
// port, so an https->http downgrade counts), keep only the engine's own
|
|
@@ -521,6 +603,7 @@ export class RequestEngine {
|
|
|
521
603
|
headers = engineHeadersOnly(headers);
|
|
522
604
|
}
|
|
523
605
|
url = nextUrl.toString();
|
|
606
|
+
shown = this.shownTarget(location, shown);
|
|
524
607
|
redirects += 1;
|
|
525
608
|
continue;
|
|
526
609
|
}
|
|
@@ -535,16 +618,16 @@ export class RequestEngine {
|
|
|
535
618
|
const detail = target !== undefined
|
|
536
619
|
? `stopped after ${this.maxRedirects} redirects (a redirect loop?)`
|
|
537
620
|
: location
|
|
538
|
-
? `redirect to ${cutForMessage(sanitizeServerText(location))} not followed`
|
|
621
|
+
? `redirect to ${cutForMessage(sanitizeServerText(this.scrub(location)))} not followed`
|
|
539
622
|
: "redirect not followed (no Location header)";
|
|
540
|
-
throw new CkanApiError({ status, url, method, body: this.scrub(errorBodyText(body)), detail });
|
|
623
|
+
throw new CkanApiError({ status, url: shown, method, body: this.scrubBody(this.scrub(errorBodyText(body))), detail });
|
|
541
624
|
}
|
|
542
|
-
throw this.toApiError(method,
|
|
625
|
+
throw this.toApiError(method, shown, status, body, status === 401 || status === 403 ? dropped : undefined, {
|
|
543
626
|
retries: attempt,
|
|
544
627
|
...(tooLong ? { retryAfterMs: retryAfter } : {}),
|
|
545
628
|
});
|
|
546
629
|
}
|
|
547
|
-
return { data: body, contentType, status, url };
|
|
630
|
+
return { data: body, contentType, status, url: shown };
|
|
548
631
|
}
|
|
549
632
|
}
|
|
550
633
|
/** Perform a GET expecting JSON and parse it into `T` (see {@link getAnswer}). */
|
|
@@ -627,28 +710,24 @@ export class RequestEngine {
|
|
|
627
710
|
catch {
|
|
628
711
|
// Non-JSON error body; leave detail undefined.
|
|
629
712
|
}
|
|
630
|
-
// `detail` came from the response body
|
|
713
|
+
// `detail` came from the response body, decoded: scrubbed again (a JSON escape hid the
|
|
714
|
+
// credential from the scrub above), then stripped of control characters so a hostile
|
|
631
715
|
// endpoint cannot inject terminal escape sequences via the stderr error message.
|
|
632
716
|
if (detail !== undefined)
|
|
633
|
-
detail = sanitizeServerText(detail);
|
|
717
|
+
detail = sanitizeServerText(this.scrub(detail));
|
|
634
718
|
if (hint !== undefined)
|
|
635
719
|
detail = detail === undefined ? hint : `${detail}; ${hint}`;
|
|
636
720
|
return new CkanApiError({
|
|
637
721
|
status,
|
|
638
722
|
url,
|
|
639
723
|
method,
|
|
640
|
-
body: text,
|
|
724
|
+
body: this.scrubBody(text),
|
|
641
725
|
detail,
|
|
642
726
|
retries: retry.retries,
|
|
643
727
|
...(retry.retryAfterMs === undefined ? {} : { retryAfterMs: retry.retryAfterMs, maxRetryAfterMs: MAX_RETRY_AFTER_MS }),
|
|
644
728
|
});
|
|
645
729
|
}
|
|
646
730
|
}
|
|
647
|
-
/** `url` without its userinfo (`https://user:pw@host/x` → `https://host/x`), as written otherwise. */
|
|
648
|
-
function withoutUserinfo(url) {
|
|
649
|
-
const [userinfo] = credentialsIn(url);
|
|
650
|
-
return userinfo === undefined ? url : url.replace(`://${userinfo}@`, "://");
|
|
651
|
-
}
|
|
652
731
|
/**
|
|
653
732
|
* The `Authorization` header for a URL's userinfo (`Basic base64(user:password)`,
|
|
654
733
|
* both percent-decoded, as Node's own http client builds it), or undefined without
|
|
@@ -770,7 +849,7 @@ function escapeAt(json, at) {
|
|
|
770
849
|
function wellFormedEscapes(json) {
|
|
771
850
|
if (!SURROGATE_ESCAPE.test(json))
|
|
772
851
|
return { json, replaced: 0 };
|
|
773
|
-
|
|
852
|
+
const out = new TextBuilder();
|
|
774
853
|
let copied = 0;
|
|
775
854
|
let replaced = 0;
|
|
776
855
|
for (let i = 0; i < json.length; i++) {
|
|
@@ -786,16 +865,32 @@ function wellFormedEscapes(json) {
|
|
|
786
865
|
i += 11;
|
|
787
866
|
continue;
|
|
788
867
|
}
|
|
789
|
-
out
|
|
868
|
+
out.push(json.slice(copied, i), "\\ufffd");
|
|
790
869
|
copied = i + 6;
|
|
791
870
|
replaced++;
|
|
792
871
|
i += 5;
|
|
793
872
|
}
|
|
794
|
-
|
|
873
|
+
if (replaced === 0)
|
|
874
|
+
return { json, replaced };
|
|
875
|
+
out.push(json.slice(copied));
|
|
876
|
+
return { json: out.toString(), replaced };
|
|
795
877
|
}
|
|
796
878
|
function isLowSurrogate(code) {
|
|
797
879
|
return code >= 0xdc00 && code <= 0xdfff;
|
|
798
880
|
}
|
|
881
|
+
/**
|
|
882
|
+
* `form` and its spelling in a URL (`encodeURIComponent`): a server that puts a credential into
|
|
883
|
+
* a URL, a redirect's Location, percent-encodes it. A form that has no such spelling (a lone
|
|
884
|
+
* surrogate) is kept as it is.
|
|
885
|
+
*/
|
|
886
|
+
function urlSpellings(form) {
|
|
887
|
+
try {
|
|
888
|
+
return [form, encodeURIComponent(form)];
|
|
889
|
+
}
|
|
890
|
+
catch {
|
|
891
|
+
return [form];
|
|
892
|
+
}
|
|
893
|
+
}
|
|
799
894
|
/** Resolve a Location header against the current URL; undefined if missing or malformed. */
|
|
800
895
|
function resolveLocation(location, base) {
|
|
801
896
|
if (location === undefined || location === "")
|
|
@@ -833,9 +928,17 @@ function exactAtTheEdges(literal, value) {
|
|
|
833
928
|
/** A decimal literal as sign, significant digits (no leading or trailing zeros) and exponent: digits × 10^exponent. */
|
|
834
929
|
function decimalOf(text) {
|
|
835
930
|
const [, sign = "", whole = "", fraction = "", exp = "0"] = /^(-?)(\d+)(?:\.(\d+))?(?:[eE]([+-]?\d+))?$/.exec(text) ?? [];
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
931
|
+
// The zeros are counted with index loops: `/0+$/` tried a match at every zero of a run that
|
|
932
|
+
// doesn't end the text, quadratic in the run, and one `1.<64 000 zeros>1e300` in an answer held
|
|
933
|
+
// the CLI for seconds, past --timeout (the scan runs once the answer is in).
|
|
934
|
+
const all = whole + fraction;
|
|
935
|
+
let start = 0;
|
|
936
|
+
while (start < all.length && all.charCodeAt(start) === 0x30)
|
|
937
|
+
start++;
|
|
938
|
+
let end = all.length;
|
|
939
|
+
while (end > start && all.charCodeAt(end - 1) === 0x30)
|
|
940
|
+
end--;
|
|
941
|
+
return { negative: sign === "-", digits: all.slice(start, end), exponent: Number(exp) - fraction.length + (all.length - end) };
|
|
839
942
|
}
|
|
840
943
|
/** A JSON number token as the grammar allows it, matched at a given position (sticky). */
|
|
841
944
|
const STRICT_JSON_NUMBER = /-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/y;
|
|
@@ -902,7 +1005,8 @@ export function scanNumbers(json, bigIntegerKeys = NO_KEYS) {
|
|
|
902
1005
|
const literal = STRICT_JSON_NUMBER.exec(json)?.[0];
|
|
903
1006
|
if (literal === undefined)
|
|
904
1007
|
continue;
|
|
905
|
-
out
|
|
1008
|
+
out ??= new TextBuilder();
|
|
1009
|
+
out.push(json.slice(copied, at), '"', literal, '"');
|
|
906
1010
|
copied = at + literal.length;
|
|
907
1011
|
i = copied - 1;
|
|
908
1012
|
}
|
|
@@ -920,7 +1024,10 @@ export function scanNumbers(json, bigIntegerKeys = NO_KEYS) {
|
|
|
920
1024
|
i += literal.length - 1;
|
|
921
1025
|
}
|
|
922
1026
|
}
|
|
923
|
-
|
|
1027
|
+
if (out === undefined)
|
|
1028
|
+
return { json, inexact };
|
|
1029
|
+
out.push(json.slice(copied));
|
|
1030
|
+
return { json: out.toString(), inexact };
|
|
924
1031
|
}
|
|
925
1032
|
/** A member name written with an escape, decoded; a name JSON.parse can't read matches nothing. */
|
|
926
1033
|
function decodedName(quoted) {
|
|
@@ -9,20 +9,60 @@ export declare function redactUrl(url: string): string;
|
|
|
9
9
|
/**
|
|
10
10
|
* The userinfo a URL carries, exactly as written — `["alice:pa#ss"]` for
|
|
11
11
|
* `https://alice:pa#ss@host` — or `[]` when it carries none. Only a value that starts
|
|
12
|
-
* with a scheme (`^[A-Za-z][A-Za-z0-9+.-]
|
|
12
|
+
* with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`, or a special one such as `http:` with any
|
|
13
|
+
* slashes or backslashes after it, as WHATWG reads it) counts: a bare `a:b@c` is a dataset id, a
|
|
13
14
|
* search text or a User-Agent as often as a credential, and the base URL always has a
|
|
14
15
|
* scheme. It works on URLs that don't parse too: the userinfo is everything between
|
|
15
16
|
* `://` and the last `@` before the host. Used to redact those exact strings from text
|
|
16
17
|
* that echoes the value (usage errors, help), whatever characters the password contains.
|
|
17
18
|
*/
|
|
18
19
|
export declare function credentialsIn(value: string): string[];
|
|
20
|
+
/**
|
|
21
|
+
* The userinfo of a value meant as a URL (a base URL, a request URL), however its scheme is
|
|
22
|
+
* typed: as `credentialsIn` reads it when the value has a scheme; else as if it had `http://`
|
|
23
|
+
* (`user:password@host`, `TOKEN:@host`); else, for a scheme mistyped before slashes
|
|
24
|
+
* (`http//user:pw@host` with the colon left out, `http/…`, `htp:/…`), everything between the
|
|
25
|
+
* first run of slashes or backslashes and the last `@`, as `credentialsIn` reads a URL that
|
|
26
|
+
* doesn't parse. Such a value is refused, but the refusal (commander's, the transport's) quotes
|
|
27
|
+
* it, so its userinfo has to be found to be redacted. Only for a value known to be a URL: in
|
|
28
|
+
* other text a bare `a/b@c` is no credential.
|
|
29
|
+
*/
|
|
30
|
+
export declare function typedUrlCredentials(value: string): string[];
|
|
31
|
+
/**
|
|
32
|
+
* `text` with the userinfo of every URL in it replaced by `***` (`https://user:secret@host`
|
|
33
|
+
* becomes `https://***@host`): the pattern backstop for text no exact secret matches. Text-based,
|
|
34
|
+
* so it also covers a URL that does not parse; the userinfo ends at the last `@` before the first
|
|
35
|
+
* whitespace, `/`, `?`, `#` or `'` after `://`. Linear in `text`, however many URLs it holds: each
|
|
36
|
+
* `://` is found with `indexOf`, its scheme by walking back no further than the previous `://`,
|
|
37
|
+
* and the search goes on right after each `://`, not after the authority it read: a URL glued
|
|
38
|
+
* onto the one before it (`https://host://u:p@h`, `https://u:p@x://v:q@w`) starts inside that
|
|
39
|
+
* authority, which ends at its `/`. An authority holds no `/`, so no `://` lies wholly inside one,
|
|
40
|
+
* and every character is read a bounded number of times: by the authority loop and `indexOf`
|
|
41
|
+
* forwards, by `schemeStart` backwards. The regex it replaces (a scheme tried at every
|
|
42
|
+
* word boundary) was quadratic: an 80 KB answer of `a.a.…` took about 5 s to print.
|
|
43
|
+
*/
|
|
44
|
+
export declare function redactUrlUserinfo(text: string): string;
|
|
45
|
+
/**
|
|
46
|
+
* A text put together from many pieces (slices of a server answer and their replacements),
|
|
47
|
+
* joined in batches of BUILDER_BATCH into flat strings, so memory stays proportional to the text,
|
|
48
|
+
* not to the number of pieces. One `+=` per piece made a rope node each, and one array of every
|
|
49
|
+
* piece a slot each: millions of lone-surrogate repairs in a large answer exhausted a small heap
|
|
50
|
+
* either way.
|
|
51
|
+
*/
|
|
52
|
+
export declare class TextBuilder {
|
|
53
|
+
#private;
|
|
54
|
+
push(...pieces: string[]): void;
|
|
55
|
+
toString(): string;
|
|
56
|
+
}
|
|
19
57
|
/**
|
|
20
58
|
* The forms in which a server may echo the credentials of a userinfo (`user:password`,
|
|
21
|
-
* as `credentialsIn` returns it) back in an error body
|
|
22
|
-
* (base64 of the decoded `user:password`, UTF-8 as the engine
|
|
23
|
-
* `user:password` itself, and the
|
|
24
|
-
*
|
|
25
|
-
*
|
|
59
|
+
* as `credentialsIn` returns it) back in an error body, in this order: the
|
|
60
|
+
* `Authorization: Basic` value (base64 of the decoded `user:password`, UTF-8 as the engine
|
|
61
|
+
* sends it), the decoded `user:password` itself, and the secret alone when it is at least 4
|
|
62
|
+
* characters long. The secret is the password; in a userinfo without one (`TOKEN@host`,
|
|
63
|
+
* `TOKEN:@host`) it is the user name, a token, which the engine sends as Basic `TOKEN:`. `[]`
|
|
64
|
+
* for an empty userinfo. None of them has an `@` to anchor on, so they are replaced as exact
|
|
65
|
+
* strings (`redactSecrets`).
|
|
26
66
|
*/
|
|
27
67
|
export declare function echoedCredentialForms(userinfo: string): string[];
|
|
28
68
|
/** `text` with every occurrence of each of `secrets` (exact strings, no anchor) replaced by `***`. */
|
|
@@ -83,6 +123,12 @@ export declare class CkanApiError extends CkanError {
|
|
|
83
123
|
readonly detail: string | undefined;
|
|
84
124
|
readonly url: string;
|
|
85
125
|
readonly method: string;
|
|
126
|
+
/**
|
|
127
|
+
* The answer's body as text, the base URL's credentials scrubbed. As sent, unless it is JSON
|
|
128
|
+
* whose decoded text still holds an echoed credential behind an escape: then the decoded JSON
|
|
129
|
+
* written again (other escapes, no insignificant whitespace) with it replaced by `***`, and `""`
|
|
130
|
+
* for such a body nested too deeply to write again.
|
|
131
|
+
*/
|
|
86
132
|
readonly body: string;
|
|
87
133
|
/** How many times the engine retried the request before giving up (0 when it did not). */
|
|
88
134
|
readonly retries: number;
|